blob: de0481aa585ef5e63139e3ec2c999e4d4734500b [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Alex Klein87531182019-08-12 15:23:37 -060036# Registered handlers for uprevving versioned packages.
37_UPREV_FUNCS = {}
38
Alex Kleineb77ffa2019-05-28 14:47:44 -060039
40class Error(Exception):
41 """Module's base error class."""
42
43
Alex Klein4de25e82019-08-05 15:58:39 -060044class UnknownPackageError(Error):
45 """Uprev attempted for a package without a registered handler."""
46
47
Alex Kleineb77ffa2019-05-28 14:47:44 -060048class UprevError(Error):
49 """An error occurred while uprevving packages."""
50
51
Michael Mortensenb70e8a82019-10-10 18:43:41 -060052class NoAndroidVersionError(Error):
53 """An error occurred while trying to determine the android version."""
54
55
56class NoAndroidBranchError(Error):
57 """An error occurred while trying to determine the android branch."""
58
59
60class NoAndroidTargetError(Error):
61 """An error occurred while trying to determine the android target."""
62
63
Alex Klein4de25e82019-08-05 15:58:39 -060064class AndroidIsPinnedUprevError(UprevError):
65 """Raised when we try to uprev while Android is pinned."""
66
67 def __init__(self, new_android_atom):
68 """Initialize a AndroidIsPinnedUprevError.
69
70 Args:
71 new_android_atom: The Android atom that we failed to
72 uprev to, due to Android being pinned.
73 """
74 assert new_android_atom
75 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
76 new_android_atom)
77 super(AndroidIsPinnedUprevError, self).__init__(msg)
78 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060079
80
Yaakov Shaul1eafe832019-09-10 16:50:26 -060081class EbuildManifestError(Error):
82 """Error when running ebuild manifest."""
83
84
Andrew Lamb9563a152019-12-04 11:42:18 -070085class GeneratedCrosConfigFilesError(Error):
86 """Error when cros_config_schema does not produce expected files"""
87
88 def __init__(self, expected_files, found_files):
89 msg = ('Expected to find generated C files: %s. Actually found: %s' %
90 (expected_files, found_files))
91 super(GeneratedCrosConfigFilesError, self).__init__(msg)
92
Alex Klein7a3a7dd2020-01-08 16:44:38 -070093
Yaakov Shaul730814a2019-09-10 13:58:25 -060094UprevVersionedPackageModifications = collections.namedtuple(
95 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -060096
Yaakov Shaul730814a2019-09-10 13:58:25 -060097
98class UprevVersionedPackageResult(object):
99 """Data object for uprev_versioned_package."""
100
101 def __init__(self):
102 self.modified = []
103
104 def add_result(self, new_version, modified_files):
105 """Adds version/ebuilds tuple to result.
106
107 Args:
108 new_version: New version number of package.
109 modified_files: List of files modified for the given version.
110 """
111 result = UprevVersionedPackageModifications(new_version, modified_files)
112 self.modified.append(result)
113 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600114
115 @property
116 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600117 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600118
119
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600120def patch_ebuild_vars(ebuild_path, variables):
121 """Updates variables in ebuild.
122
123 Use this function rather than portage_util.EBuild.UpdateEBuild when you
124 want to preserve the variable position and quotes within the ebuild.
125
126 Args:
127 ebuild_path: The path of the ebuild.
128 variables: Dictionary of variables to update in ebuild.
129 """
130 try:
131 for line in fileinput.input(ebuild_path, inplace=1):
132 varname, eq, _ = line.partition('=')
133 if eq == '=' and varname.strip() in variables:
134 value = variables[varname]
135 sys.stdout.write('%s="%s"\n' % (varname, value))
136 else:
137 sys.stdout.write(line)
138 finally:
139 fileinput.close()
140
141
Alex Klein87531182019-08-12 15:23:37 -0600142def uprevs_versioned_package(package):
143 """Decorator to register package uprev handlers."""
144 assert package
145
146 def register(func):
147 """Registers |func| as a handler for |package|."""
148 _UPREV_FUNCS[package] = func
149
150 @functools.wraps(func)
151 def pass_through(*args, **kwargs):
152 return func(*args, **kwargs)
153
154 return pass_through
155
156 return register
157
158
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700159def uprev_android(tracking_branch,
160 android_package,
161 android_build_branch,
162 chroot,
163 build_targets=None,
164 android_version=None,
Alex Klein4de25e82019-08-05 15:58:39 -0600165 android_gts_build_branch=None):
166 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700167 command = [
168 'cros_mark_android_as_stable',
169 '--tracking_branch=%s' % tracking_branch,
170 '--android_package=%s' % android_package,
171 '--android_build_branch=%s' % android_build_branch,
172 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600173 if build_targets:
174 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
175 if android_version:
176 command.append('--force_version=%s' % android_version)
177 if android_gts_build_branch:
178 command.append('--android_gts_build_branch=%s' % android_gts_build_branch)
179
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700180 result = cros_build_lib.run(
181 command,
182 stdout=True,
183 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500184 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700185 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600186
Mike Frysinger88d96362020-02-14 19:05:45 -0500187 portage_atom_string = result.stdout.strip()
188 android_atom = None
189 if portage_atom_string:
190 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600191 if not android_atom:
192 logging.info('Found nothing to rev.')
193 return None
194
195 for target in build_targets or []:
196 # Sanity check: We should always be able to merge the version of
197 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700198 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600199 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700200 cros_build_lib.run(
201 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600202 except cros_build_lib.RunCommandError:
203 logging.error(
204 'Cannot emerge-%s =%s\nIs Android pinned to an older '
205 'version?', target, android_atom)
206 raise AndroidIsPinnedUprevError(android_atom)
207
208 return android_atom
209
210
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700211def uprev_build_targets(build_targets,
212 overlay_type,
213 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600214 output_dir=None):
215 """Uprev the set provided build targets, or all if not specified.
216
217 Args:
218 build_targets (list[build_target_util.BuildTarget]|None): The build targets
219 whose overlays should be uprevved, empty or None for all.
220 overlay_type (str): One of the valid overlay types except None (see
221 constants.VALID_OVERLAYS).
222 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
223 output_dir (str|None): The path to optionally dump result files.
224 """
225 # Need a valid overlay, but exclude None.
226 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
227
228 if build_targets:
229 overlays = portage_util.FindOverlaysForBoards(
230 overlay_type, boards=[t.name for t in build_targets])
231 else:
232 overlays = portage_util.FindOverlays(overlay_type)
233
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700234 return uprev_overlays(
235 overlays,
236 build_targets=build_targets,
237 chroot=chroot,
238 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600239
240
241def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
242 """Uprev the given overlays.
243
244 Args:
245 overlays (list[str]): The list of overlay paths.
246 build_targets (list[build_target_util.BuildTarget]|None): The build targets
247 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
248 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
249 output_dir (str|None): The path to optionally dump result files.
250
251 Returns:
252 list[str] - The paths to all of the modified ebuild files. This includes the
253 new files that were added (i.e. the new versions) and all of the removed
254 files (i.e. the old versions).
255 """
256 assert overlays
257
258 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
259
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700260 uprev_manager = uprev_lib.UprevOverlayManager(
261 overlays,
262 manifest,
263 build_targets=build_targets,
264 chroot=chroot,
265 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600266 uprev_manager.uprev()
267
268 return uprev_manager.modified_ebuilds
269
270
Alex Klein87531182019-08-12 15:23:37 -0600271def uprev_versioned_package(package, build_targets, refs, chroot):
272 """Call registered uprev handler function for the package.
273
274 Args:
275 package (portage_util.CPV): The package being uprevved.
276 build_targets (list[build_target_util.BuildTarget]): The build targets to
277 clean on a successful uprev.
278 refs (list[uprev_lib.GitRef]):
279 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
280
281 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600282 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600283 """
284 assert package
285
286 if package.cp not in _UPREV_FUNCS:
287 raise UnknownPackageError(
288 'Package "%s" does not have a registered handler.' % package.cp)
289
Andrew Lambea9a8a22019-12-12 14:03:43 -0700290 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600291
292
Evan Hernandezb51f1522019-08-15 11:29:40 -0600293# TODO(evanhernandez): Remove this. Only a quick hack for testing.
294@uprevs_versioned_package('sample/sample')
295def uprev_sample(*_args, **_kwargs):
296 """Mimics an uprev by changing files in sandbox repos.
297
298 See: uprev_versioned_package.
299 """
300 paths = [
301 os.path.join(constants.SOURCE_ROOT, 'infra/dummies', repo, 'sample.txt')
302 for repo in ('general-sandbox', 'merge-sandbox')
303 ]
304
Yaakov Shaul730814a2019-09-10 13:58:25 -0600305 return UprevVersionedPackageResult().add_result('1.2.3', paths)
Evan Hernandezb51f1522019-08-15 11:29:40 -0600306
307
Yaakov Shaul395ae832019-09-09 14:45:32 -0600308@uprevs_versioned_package('afdo/kernel-profiles')
309def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600310 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600311
312 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600313
314 Raises:
315 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600316 """
317 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
318 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
319
David Burger92485342019-09-10 17:52:45 -0600320 with open(path, 'r') as f:
321 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600322
Yaakov Shaul730814a2019-09-10 13:58:25 -0600323 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600324 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600325 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
326 'sys-kernel', version)
327 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
328 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600329 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
330 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600331 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600332 patch_ebuild_vars(ebuild_path,
333 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600334
335 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600336 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400337 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600338 except cros_build_lib.RunCommandError as e:
339 raise EbuildManifestError(
340 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600341 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600342
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600343 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600344
Yaakov Shaul730814a2019-09-10 13:58:25 -0600345 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
346
347 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600348
349
Trent Begin315d9d92019-12-03 21:55:53 -0700350@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Begin6daa8702020-01-29 14:58:12 -0700351def uprev_sludge(build_targets, refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700352 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
353
354 See: uprev_versioned_package.
355 """
Trent Begin6daa8702020-01-29 14:58:12 -0700356 # Unused by uprev_sludge
357 del build_targets, refs
358
Trent Begin315d9d92019-12-03 21:55:53 -0700359 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700360 package_path = os.path.join('src', 'private-overlays',
361 'project-wilco-private', 'chromeos-base', package)
362 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700363
Trent Begin6daa8702020-01-29 14:58:12 -0700364 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700365
366
Trent Begin6daa8702020-01-29 14:58:12 -0700367def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700368 """Changes the package ebuild's version to match the version pin file.
369
370 Args:
371 package_path: The path of the package relative to the src root. This path
372 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700373 version_pin_path: The path of the version_pin file that contains only a
374 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700375 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700376 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700377
378 Returns:
379 UprevVersionedPackageResult: The result.
380 """
381 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700382
383 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
384 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700385 if not ebuild_paths:
386 raise UprevError('No ebuilds found for %s' % package)
387 elif len(ebuild_paths) > 1:
388 raise UprevError('Multiple ebuilds found for %s' % package)
389 else:
390 ebuild_path = ebuild_paths[0]
391
Trent Begind943df92020-02-25 10:30:10 -0700392 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
393 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700394 new_ebuild_path = os.path.join(package_path,
395 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700396 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
397 os.rename(ebuild_path, new_ebuild_src_path)
Trent Begin4a11a632020-02-28 12:59:58 -0700398 manifest_src_path = os.path.join(package_src_path, 'Manifest')
Trent Begind943df92020-02-25 10:30:10 -0700399 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
400 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700401
Trent Begin6daa8702020-01-29 14:58:12 -0700402 try:
Trent Begind943df92020-02-25 10:30:10 -0700403 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700404 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700405 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700406 'Unable to update manifest for %s: %s' % (package, e.stderr))
407
Trent Begin315d9d92019-12-03 21:55:53 -0700408 result = UprevVersionedPackageResult()
Trent Begin4a11a632020-02-28 12:59:58 -0700409 result.add_result(version,
410 [new_ebuild_src_path, ebuild_path, manifest_src_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700411 return result
412
413
Alex Klein87531182019-08-12 15:23:37 -0600414@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700415def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600416 """Uprev chrome and its related packages.
417
418 See: uprev_versioned_package.
419 """
420 # Determine the version from the refs (tags), i.e. the chrome versions are the
421 # tag names.
422 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
423
424 uprev_manager = uprev_lib.UprevChromeManager(
425 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600426 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600427 # Start with chrome itself, as we can't do anything else unless chrome
428 # uprevs successfully.
429 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600430 return result
Alex Klein87531182019-08-12 15:23:37 -0600431
432 # With a successful chrome rev, also uprev related packages.
433 for package in constants.OTHER_CHROME_PACKAGES:
434 uprev_manager.uprev(package)
435
David Burger37f48672019-09-18 17:07:56 -0600436 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600437
438
Andrew Lamb9563a152019-12-04 11:42:18 -0700439def _generate_platform_c_files(replication_config, chroot):
440 """Generates platform C files from a platform JSON payload.
441
442 Args:
443 replication_config (replication_config_pb2.ReplicationConfig): A
444 ReplicationConfig that has already been run. If it produced a
445 build_config.json file, that file will be used to generate platform C
446 files. Otherwise, nothing will be generated.
447 chroot (chroot_lib.Chroot): The chroot to use to generate.
448
449 Returns:
450 A list of generated files.
451 """
452 # Generate the platform C files from the build config. Note that it would be
453 # more intuitive to generate the platform C files from the platform config;
454 # however, cros_config_schema does not allow this, because the platform config
455 # payload is not always valid input. For example, if a property is both
456 # 'required' and 'build-only', it will fail schema validation. Thus, use the
457 # build config, and use '-f' to filter.
458 build_config_path = [
459 rule.destination_path
460 for rule in replication_config.file_replication_rules
461 if rule.destination_path.endswith('build_config.json')
462 ]
463
464 if not build_config_path:
465 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700466 'No build_config.json found, will not generate platform C files. '
467 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700468 return []
469
470 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700471 raise ValueError('Expected at most one build_config.json destination path. '
472 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700473
474 build_config_path = build_config_path[0]
475
476 # Paths to the build_config.json and dir to output C files to, in the
477 # chroot.
478 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
479 build_config_path)
480 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
481 os.path.dirname(build_config_path))
482
483 command = [
484 'cros_config_schema', '-m', build_config_chroot_path, '-g',
485 generated_output_chroot_dir, '-f', '"TRUE"'
486 ]
487
488 cros_build_lib.run(
489 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
490
491 # A relative (to the source root) path to the generated C files.
492 generated_output_dir = os.path.dirname(build_config_path)
493 generated_files = []
494 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
495 for f in expected_c_files:
496 if os.path.exists(
497 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
498 generated_files.append(os.path.join(generated_output_dir, f))
499
500 if len(expected_c_files) != len(generated_files):
501 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
502
503 return generated_files
504
505
Andrew Lambe836f222019-12-09 12:27:38 -0700506def _get_private_overlay_package_root(ref, package):
507 """Returns the absolute path to the root of a given private overlay.
508
509 Args:
510 ref (uprev_lib.GitRef): GitRef for the private overlay.
511 package (str): Path to the package in the overlay.
512 """
513 # There might be a cleaner way to map from package -> path within the source
514 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700515 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700516 match = re.match(private_overlay_ref_pattern, ref.path)
517 if not match:
518 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
519 (private_overlay_ref_pattern, ref))
520
521 overlay = match.group(1)
522
523 return os.path.join(constants.SOURCE_ROOT,
524 'src/private-overlays/overlay-%s-private' % overlay,
525 package)
526
527
Andrew Lambea9a8a22019-12-12 14:03:43 -0700528@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
529def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700530 """Replicate a private cros_config change to the corresponding public config.
531
Alex Kleinad6b48a2020-01-08 16:57:41 -0700532 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700533 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700534 package = 'chromeos-base/chromeos-config-bsp'
535
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700536 if len(refs) != 1:
537 raise ValueError('Expected exactly one ref, actual %s' % refs)
538
539 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700540 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700541 replication_config_path = os.path.join(package_root,
542 'replication_config.jsonpb')
543
544 try:
545 replication_config = json_format.Parse(
546 osutils.ReadFile(replication_config_path),
547 replication_config_pb2.ReplicationConfig())
548 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700549 raise ValueError(
550 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700551
552 replication_lib.Replicate(replication_config)
553
554 modified_files = [
555 rule.destination_path
556 for rule in replication_config.file_replication_rules
557 ]
558
Andrew Lamb9563a152019-12-04 11:42:18 -0700559 # The generated platform C files are not easily filtered by replication rules,
560 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
561 # files cannot. Therefore, replicate and filter the JSON payloads, and then
562 # generate filtered C files from the JSON payload.
563 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700564
565 # Use the private repo's commit hash as the new version.
566 new_private_version = refs[0].revision
567
Andrew Lamb988f4da2019-12-10 10:16:43 -0700568 # modified_files should contain only relative paths at this point, but the
569 # returned UprevVersionedPackageResult must contain only absolute paths.
570 for i, modified_file in enumerate(modified_files):
571 assert not os.path.isabs(modified_file)
572 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
573
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700574 return UprevVersionedPackageResult().add_result(new_private_version,
575 modified_files)
576
577
Alex Kleinbbef2b32019-08-27 10:38:50 -0600578def get_best_visible(atom, build_target=None):
579 """Returns the best visible CPV for the given atom.
580
581 Args:
582 atom (str): The atom to look up.
583 build_target (build_target_util.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600584 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700585
586 Returns:
587 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600588 """
David Burger1e0fe232019-07-01 14:52:07 -0600589 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600590
591 board = build_target.name if build_target else None
592 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600593
594
Alex Klein149fd3b2019-12-16 16:01:05 -0700595def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600596 """Check if a prebuilt exists.
597
598 Args:
599 atom (str): The package whose prebuilt is being queried.
600 build_target (build_target_util.BuildTarget): The build target whose
601 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700602 useflags: Any additional USE flags that should be set. May be a string
603 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700604
605 Returns:
606 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600607 """
608 assert atom
609
610 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700611 extra_env = None
612 if useflags:
613 new_flags = useflags
614 if not isinstance(useflags, six.string_types):
615 new_flags = ' '.join(useflags)
616
617 existing = os.environ.get('USE', '')
618 final_flags = '%s %s' % (existing, new_flags)
619 extra_env = {'USE': final_flags.strip()}
620 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600621
622
David Burger0f9dd4e2019-10-08 12:33:42 -0600623def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600624 """Check if |build_target| builds |atom| (has it in its depgraph)."""
625 cros_build_lib.AssertInsideChroot()
626
Chris McDonalda22b74f2019-11-22 13:55:06 -0700627 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600628 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600629
630
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600631def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600632 """Returns the current Chrome version for the board (or in buildroot).
633
634 Args:
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600635 build_target (build_target_util.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700636
637 Returns:
638 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600639 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600640 # TODO(crbug/1019770): Long term we should not need the try/catch here once
641 # the builds function above only returns True for chrome when
642 # determine_chrome_version will succeed.
643 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700644 cpv = portage_util.PortageqBestVisible(
645 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600646 except cros_build_lib.RunCommandError as e:
647 # Return None because portage failed when trying to determine the chrome
648 # version.
649 logging.warning('Caught exception in determine_chrome_package: %s', e)
650 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600651 # Something like 78.0.3877.4_rc -> 78.0.3877.4
652 return cpv.version_no_rev.partition('_')[0]
653
654
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600655def determine_android_package(board):
656 """Returns the active Android container package in use by the board.
657
658 Args:
659 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700660
661 Returns:
662 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600663 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600664 try:
665 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600666 except cros_build_lib.RunCommandError as e:
667 # Return None because a command (likely portage) failed when trying to
668 # determine the package.
669 logging.warning('Caught exception in determine_android_package: %s', e)
670 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600671
Alex Kleinad6b48a2020-01-08 16:57:41 -0700672 # We assume there is only one Android package in the depgraph.
673 for package in packages:
674 if package.startswith('chromeos-base/android-container-') or \
675 package.startswith('chromeos-base/android-vm-'):
676 return package
677 return None
678
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600679
680def determine_android_version(boards=None):
681 """Determine the current Android version in buildroot now and return it.
682
683 This uses the typical portage logic to determine which version of Android
684 is active right now in the buildroot.
685
686 Args:
687 boards: List of boards to check version of.
688
689 Returns:
690 The Android build ID of the container for the boards.
691
692 Raises:
693 NoAndroidVersionError: if no unique Android version can be determined.
694 """
695 if not boards:
696 return None
697 # Verify that all boards have the same version.
698 version = None
699 for board in boards:
700 package = determine_android_package(board)
701 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600702 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600703 cpv = portage_util.SplitCPV(package)
704 if not cpv:
705 raise NoAndroidVersionError(
706 'Android version could not be determined for %s' % board)
707 if not version:
708 version = cpv.version_no_rev
709 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700710 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
711 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600712 return version
713
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700714
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600715def determine_android_branch(board):
716 """Returns the Android branch in use by the active container ebuild."""
717 try:
718 android_package = determine_android_package(board)
719 except cros_build_lib.RunCommandError:
720 raise NoAndroidBranchError(
721 'Android branch could not be determined for %s' % board)
722 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600723 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600724 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
725 # We assume all targets pull from the same branch and that we always
726 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
727 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
728 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
729 for target in targets:
730 if target in ebuild_content:
731 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
732 if branch is not None:
733 return branch.group(1)
734 raise NoAndroidBranchError(
735 'Android branch could not be determined for %s (ebuild empty?)' % board)
736
737
738def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600739 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600740 try:
741 android_package = determine_android_package(board)
742 except cros_build_lib.RunCommandError:
743 raise NoAndroidTargetError(
744 'Android Target could not be determined for %s' % board)
745 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600746 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600747 if android_package.startswith('chromeos-base/android-vm-'):
748 return 'bertha'
749 elif android_package.startswith('chromeos-base/android-container-'):
750 return 'cheets'
751
752 raise NoAndroidTargetError(
753 'Android Target cannot be determined for the package: %s' %
754 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600755
756
757def determine_platform_version():
758 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600759 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600760 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
761 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600762
763
764def determine_milestone_version():
765 """Returns the platform version from the source root."""
766 # Milestone version is something like '79'.
767 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
768 return version.chrome_branch
769
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700770
Michael Mortensen009cb662019-10-21 11:38:43 -0600771def determine_full_version():
772 """Returns the full version from the source root."""
773 # Full version is something like 'R79-12575.0.0'.
774 milestone_version = determine_milestone_version()
775 platform_version = determine_platform_version()
776 full_version = ('R%s-%s' % (milestone_version, platform_version))
777 return full_version