blob: 64eb97a08ec15181889e785e458a844fd495ac8c [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Mike Frysingerbafb3182020-02-21 03:15:43 -050036
37assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
38
39
Alex Klein87531182019-08-12 15:23:37 -060040# Registered handlers for uprevving versioned packages.
41_UPREV_FUNCS = {}
42
Alex Kleineb77ffa2019-05-28 14:47:44 -060043
44class Error(Exception):
45 """Module's base error class."""
46
47
Alex Klein4de25e82019-08-05 15:58:39 -060048class UnknownPackageError(Error):
49 """Uprev attempted for a package without a registered handler."""
50
51
Alex Kleineb77ffa2019-05-28 14:47:44 -060052class UprevError(Error):
53 """An error occurred while uprevving packages."""
54
55
Michael Mortensenb70e8a82019-10-10 18:43:41 -060056class NoAndroidVersionError(Error):
57 """An error occurred while trying to determine the android version."""
58
59
60class NoAndroidBranchError(Error):
61 """An error occurred while trying to determine the android branch."""
62
63
64class NoAndroidTargetError(Error):
65 """An error occurred while trying to determine the android target."""
66
67
Alex Klein4de25e82019-08-05 15:58:39 -060068class AndroidIsPinnedUprevError(UprevError):
69 """Raised when we try to uprev while Android is pinned."""
70
71 def __init__(self, new_android_atom):
72 """Initialize a AndroidIsPinnedUprevError.
73
74 Args:
75 new_android_atom: The Android atom that we failed to
76 uprev to, due to Android being pinned.
77 """
78 assert new_android_atom
79 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
80 new_android_atom)
81 super(AndroidIsPinnedUprevError, self).__init__(msg)
82 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060083
84
Yaakov Shaul1eafe832019-09-10 16:50:26 -060085class EbuildManifestError(Error):
86 """Error when running ebuild manifest."""
87
88
Andrew Lamb9563a152019-12-04 11:42:18 -070089class GeneratedCrosConfigFilesError(Error):
90 """Error when cros_config_schema does not produce expected files"""
91
92 def __init__(self, expected_files, found_files):
93 msg = ('Expected to find generated C files: %s. Actually found: %s' %
94 (expected_files, found_files))
95 super(GeneratedCrosConfigFilesError, self).__init__(msg)
96
Alex Klein7a3a7dd2020-01-08 16:44:38 -070097
Yaakov Shaul730814a2019-09-10 13:58:25 -060098UprevVersionedPackageModifications = collections.namedtuple(
99 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -0600100
Yaakov Shaul730814a2019-09-10 13:58:25 -0600101
102class UprevVersionedPackageResult(object):
103 """Data object for uprev_versioned_package."""
104
105 def __init__(self):
106 self.modified = []
107
108 def add_result(self, new_version, modified_files):
109 """Adds version/ebuilds tuple to result.
110
111 Args:
112 new_version: New version number of package.
113 modified_files: List of files modified for the given version.
114 """
115 result = UprevVersionedPackageModifications(new_version, modified_files)
116 self.modified.append(result)
117 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600118
119 @property
120 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600121 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600122
123
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600124def patch_ebuild_vars(ebuild_path, variables):
125 """Updates variables in ebuild.
126
127 Use this function rather than portage_util.EBuild.UpdateEBuild when you
128 want to preserve the variable position and quotes within the ebuild.
129
130 Args:
131 ebuild_path: The path of the ebuild.
132 variables: Dictionary of variables to update in ebuild.
133 """
134 try:
135 for line in fileinput.input(ebuild_path, inplace=1):
136 varname, eq, _ = line.partition('=')
137 if eq == '=' and varname.strip() in variables:
138 value = variables[varname]
139 sys.stdout.write('%s="%s"\n' % (varname, value))
140 else:
141 sys.stdout.write(line)
142 finally:
143 fileinput.close()
144
145
Alex Klein87531182019-08-12 15:23:37 -0600146def uprevs_versioned_package(package):
147 """Decorator to register package uprev handlers."""
148 assert package
149
150 def register(func):
151 """Registers |func| as a handler for |package|."""
152 _UPREV_FUNCS[package] = func
153
154 @functools.wraps(func)
155 def pass_through(*args, **kwargs):
156 return func(*args, **kwargs)
157
158 return pass_through
159
160 return register
161
162
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700163def uprev_android(tracking_branch,
164 android_package,
165 android_build_branch,
166 chroot,
167 build_targets=None,
168 android_version=None,
Alex Klein4de25e82019-08-05 15:58:39 -0600169 android_gts_build_branch=None):
170 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700171 command = [
172 'cros_mark_android_as_stable',
173 '--tracking_branch=%s' % tracking_branch,
174 '--android_package=%s' % android_package,
175 '--android_build_branch=%s' % android_build_branch,
176 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600177 if build_targets:
178 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
179 if android_version:
180 command.append('--force_version=%s' % android_version)
181 if android_gts_build_branch:
182 command.append('--android_gts_build_branch=%s' % android_gts_build_branch)
183
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700184 result = cros_build_lib.run(
185 command,
186 stdout=True,
187 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500188 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700189 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600190
Mike Frysinger88d96362020-02-14 19:05:45 -0500191 portage_atom_string = result.stdout.strip()
192 android_atom = None
193 if portage_atom_string:
194 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600195 if not android_atom:
196 logging.info('Found nothing to rev.')
197 return None
198
199 for target in build_targets or []:
200 # Sanity check: We should always be able to merge the version of
201 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700202 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600203 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700204 cros_build_lib.run(
205 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600206 except cros_build_lib.RunCommandError:
207 logging.error(
208 'Cannot emerge-%s =%s\nIs Android pinned to an older '
209 'version?', target, android_atom)
210 raise AndroidIsPinnedUprevError(android_atom)
211
212 return android_atom
213
214
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700215def uprev_build_targets(build_targets,
216 overlay_type,
217 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600218 output_dir=None):
219 """Uprev the set provided build targets, or all if not specified.
220
221 Args:
222 build_targets (list[build_target_util.BuildTarget]|None): The build targets
223 whose overlays should be uprevved, empty or None for all.
224 overlay_type (str): One of the valid overlay types except None (see
225 constants.VALID_OVERLAYS).
226 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
227 output_dir (str|None): The path to optionally dump result files.
228 """
229 # Need a valid overlay, but exclude None.
230 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
231
232 if build_targets:
233 overlays = portage_util.FindOverlaysForBoards(
234 overlay_type, boards=[t.name for t in build_targets])
235 else:
236 overlays = portage_util.FindOverlays(overlay_type)
237
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700238 return uprev_overlays(
239 overlays,
240 build_targets=build_targets,
241 chroot=chroot,
242 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600243
244
245def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
246 """Uprev the given overlays.
247
248 Args:
249 overlays (list[str]): The list of overlay paths.
250 build_targets (list[build_target_util.BuildTarget]|None): The build targets
251 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
252 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
253 output_dir (str|None): The path to optionally dump result files.
254
255 Returns:
256 list[str] - The paths to all of the modified ebuild files. This includes the
257 new files that were added (i.e. the new versions) and all of the removed
258 files (i.e. the old versions).
259 """
260 assert overlays
261
262 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
263
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700264 uprev_manager = uprev_lib.UprevOverlayManager(
265 overlays,
266 manifest,
267 build_targets=build_targets,
268 chroot=chroot,
269 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600270 uprev_manager.uprev()
271
272 return uprev_manager.modified_ebuilds
273
274
Alex Klein87531182019-08-12 15:23:37 -0600275def uprev_versioned_package(package, build_targets, refs, chroot):
276 """Call registered uprev handler function for the package.
277
278 Args:
279 package (portage_util.CPV): The package being uprevved.
280 build_targets (list[build_target_util.BuildTarget]): The build targets to
281 clean on a successful uprev.
282 refs (list[uprev_lib.GitRef]):
283 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
284
285 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600286 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600287 """
288 assert package
289
290 if package.cp not in _UPREV_FUNCS:
291 raise UnknownPackageError(
292 'Package "%s" does not have a registered handler.' % package.cp)
293
Andrew Lambea9a8a22019-12-12 14:03:43 -0700294 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600295
296
Evan Hernandezb51f1522019-08-15 11:29:40 -0600297# TODO(evanhernandez): Remove this. Only a quick hack for testing.
298@uprevs_versioned_package('sample/sample')
299def uprev_sample(*_args, **_kwargs):
300 """Mimics an uprev by changing files in sandbox repos.
301
302 See: uprev_versioned_package.
303 """
304 paths = [
305 os.path.join(constants.SOURCE_ROOT, 'infra/dummies', repo, 'sample.txt')
306 for repo in ('general-sandbox', 'merge-sandbox')
307 ]
308
Yaakov Shaul730814a2019-09-10 13:58:25 -0600309 return UprevVersionedPackageResult().add_result('1.2.3', paths)
Evan Hernandezb51f1522019-08-15 11:29:40 -0600310
311
Yaakov Shaul395ae832019-09-09 14:45:32 -0600312@uprevs_versioned_package('afdo/kernel-profiles')
313def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600314 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600315
316 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600317
318 Raises:
319 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600320 """
321 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
322 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
323
David Burger92485342019-09-10 17:52:45 -0600324 with open(path, 'r') as f:
325 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600326
Yaakov Shaul730814a2019-09-10 13:58:25 -0600327 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600328 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600329 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
330 'sys-kernel', version)
331 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
332 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600333 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
334 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600335 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600336 patch_ebuild_vars(ebuild_path,
337 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600338
339 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600340 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400341 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600342 except cros_build_lib.RunCommandError as e:
343 raise EbuildManifestError(
344 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600345 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600346
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600347 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600348
Yaakov Shaul730814a2019-09-10 13:58:25 -0600349 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
350
351 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600352
353
Trent Begin315d9d92019-12-03 21:55:53 -0700354@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Begin6daa8702020-01-29 14:58:12 -0700355def uprev_sludge(build_targets, refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700356 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
357
358 See: uprev_versioned_package.
359 """
Trent Begin6daa8702020-01-29 14:58:12 -0700360 # Unused by uprev_sludge
361 del build_targets, refs
362
Trent Begin315d9d92019-12-03 21:55:53 -0700363 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700364 package_path = os.path.join('src', 'private-overlays',
365 'project-wilco-private', 'chromeos-base', package)
366 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700367
Trent Begin6daa8702020-01-29 14:58:12 -0700368 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700369
370
Trent Begin6daa8702020-01-29 14:58:12 -0700371def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700372 """Changes the package ebuild's version to match the version pin file.
373
374 Args:
375 package_path: The path of the package relative to the src root. This path
376 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700377 version_pin_path: The path of the version_pin file that contains only a
378 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700379 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700380 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700381
382 Returns:
383 UprevVersionedPackageResult: The result.
384 """
385 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700386
387 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
388 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700389 if not ebuild_paths:
390 raise UprevError('No ebuilds found for %s' % package)
391 elif len(ebuild_paths) > 1:
392 raise UprevError('Multiple ebuilds found for %s' % package)
393 else:
394 ebuild_path = ebuild_paths[0]
395
Trent Begind943df92020-02-25 10:30:10 -0700396 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
397 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700398 new_ebuild_path = os.path.join(package_path,
399 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700400 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
401 os.rename(ebuild_path, new_ebuild_src_path)
Trent Begin4a11a632020-02-28 12:59:58 -0700402 manifest_src_path = os.path.join(package_src_path, 'Manifest')
Trent Begind943df92020-02-25 10:30:10 -0700403 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
404 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700405
Trent Begin6daa8702020-01-29 14:58:12 -0700406 try:
Trent Begind943df92020-02-25 10:30:10 -0700407 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700408 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700409 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700410 'Unable to update manifest for %s: %s' % (package, e.stderr))
411
Trent Begin315d9d92019-12-03 21:55:53 -0700412 result = UprevVersionedPackageResult()
Trent Begin4a11a632020-02-28 12:59:58 -0700413 result.add_result(version,
414 [new_ebuild_src_path, ebuild_path, manifest_src_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700415 return result
416
417
Alex Klein87531182019-08-12 15:23:37 -0600418@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700419def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600420 """Uprev chrome and its related packages.
421
422 See: uprev_versioned_package.
423 """
424 # Determine the version from the refs (tags), i.e. the chrome versions are the
425 # tag names.
426 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
427
428 uprev_manager = uprev_lib.UprevChromeManager(
429 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600430 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600431 # Start with chrome itself, as we can't do anything else unless chrome
432 # uprevs successfully.
433 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600434 return result
Alex Klein87531182019-08-12 15:23:37 -0600435
436 # With a successful chrome rev, also uprev related packages.
437 for package in constants.OTHER_CHROME_PACKAGES:
438 uprev_manager.uprev(package)
439
David Burger37f48672019-09-18 17:07:56 -0600440 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600441
442
Andrew Lamb9563a152019-12-04 11:42:18 -0700443def _generate_platform_c_files(replication_config, chroot):
444 """Generates platform C files from a platform JSON payload.
445
446 Args:
447 replication_config (replication_config_pb2.ReplicationConfig): A
448 ReplicationConfig that has already been run. If it produced a
449 build_config.json file, that file will be used to generate platform C
450 files. Otherwise, nothing will be generated.
451 chroot (chroot_lib.Chroot): The chroot to use to generate.
452
453 Returns:
454 A list of generated files.
455 """
456 # Generate the platform C files from the build config. Note that it would be
457 # more intuitive to generate the platform C files from the platform config;
458 # however, cros_config_schema does not allow this, because the platform config
459 # payload is not always valid input. For example, if a property is both
460 # 'required' and 'build-only', it will fail schema validation. Thus, use the
461 # build config, and use '-f' to filter.
462 build_config_path = [
463 rule.destination_path
464 for rule in replication_config.file_replication_rules
465 if rule.destination_path.endswith('build_config.json')
466 ]
467
468 if not build_config_path:
469 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700470 'No build_config.json found, will not generate platform C files. '
471 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700472 return []
473
474 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700475 raise ValueError('Expected at most one build_config.json destination path. '
476 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700477
478 build_config_path = build_config_path[0]
479
480 # Paths to the build_config.json and dir to output C files to, in the
481 # chroot.
482 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
483 build_config_path)
484 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
485 os.path.dirname(build_config_path))
486
487 command = [
488 'cros_config_schema', '-m', build_config_chroot_path, '-g',
489 generated_output_chroot_dir, '-f', '"TRUE"'
490 ]
491
492 cros_build_lib.run(
493 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
494
495 # A relative (to the source root) path to the generated C files.
496 generated_output_dir = os.path.dirname(build_config_path)
497 generated_files = []
498 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
499 for f in expected_c_files:
500 if os.path.exists(
501 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
502 generated_files.append(os.path.join(generated_output_dir, f))
503
504 if len(expected_c_files) != len(generated_files):
505 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
506
507 return generated_files
508
509
Andrew Lambe836f222019-12-09 12:27:38 -0700510def _get_private_overlay_package_root(ref, package):
511 """Returns the absolute path to the root of a given private overlay.
512
513 Args:
514 ref (uprev_lib.GitRef): GitRef for the private overlay.
515 package (str): Path to the package in the overlay.
516 """
517 # There might be a cleaner way to map from package -> path within the source
518 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700519 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700520 match = re.match(private_overlay_ref_pattern, ref.path)
521 if not match:
522 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
523 (private_overlay_ref_pattern, ref))
524
525 overlay = match.group(1)
526
527 return os.path.join(constants.SOURCE_ROOT,
528 'src/private-overlays/overlay-%s-private' % overlay,
529 package)
530
531
Andrew Lambea9a8a22019-12-12 14:03:43 -0700532@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
533def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700534 """Replicate a private cros_config change to the corresponding public config.
535
Alex Kleinad6b48a2020-01-08 16:57:41 -0700536 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700537 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700538 package = 'chromeos-base/chromeos-config-bsp'
539
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700540 if len(refs) != 1:
541 raise ValueError('Expected exactly one ref, actual %s' % refs)
542
543 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700544 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700545 replication_config_path = os.path.join(package_root,
546 'replication_config.jsonpb')
547
548 try:
549 replication_config = json_format.Parse(
550 osutils.ReadFile(replication_config_path),
551 replication_config_pb2.ReplicationConfig())
552 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700553 raise ValueError(
554 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700555
556 replication_lib.Replicate(replication_config)
557
558 modified_files = [
559 rule.destination_path
560 for rule in replication_config.file_replication_rules
561 ]
562
Andrew Lamb9563a152019-12-04 11:42:18 -0700563 # The generated platform C files are not easily filtered by replication rules,
564 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
565 # files cannot. Therefore, replicate and filter the JSON payloads, and then
566 # generate filtered C files from the JSON payload.
567 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700568
569 # Use the private repo's commit hash as the new version.
570 new_private_version = refs[0].revision
571
Andrew Lamb988f4da2019-12-10 10:16:43 -0700572 # modified_files should contain only relative paths at this point, but the
573 # returned UprevVersionedPackageResult must contain only absolute paths.
574 for i, modified_file in enumerate(modified_files):
575 assert not os.path.isabs(modified_file)
576 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
577
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700578 return UprevVersionedPackageResult().add_result(new_private_version,
579 modified_files)
580
581
Alex Kleinbbef2b32019-08-27 10:38:50 -0600582def get_best_visible(atom, build_target=None):
583 """Returns the best visible CPV for the given atom.
584
585 Args:
586 atom (str): The atom to look up.
587 build_target (build_target_util.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600588 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700589
590 Returns:
591 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600592 """
David Burger1e0fe232019-07-01 14:52:07 -0600593 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600594
595 board = build_target.name if build_target else None
596 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600597
598
Alex Klein149fd3b2019-12-16 16:01:05 -0700599def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600600 """Check if a prebuilt exists.
601
602 Args:
603 atom (str): The package whose prebuilt is being queried.
604 build_target (build_target_util.BuildTarget): The build target whose
605 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700606 useflags: Any additional USE flags that should be set. May be a string
607 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700608
609 Returns:
610 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600611 """
612 assert atom
613
614 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700615 extra_env = None
616 if useflags:
617 new_flags = useflags
618 if not isinstance(useflags, six.string_types):
619 new_flags = ' '.join(useflags)
620
621 existing = os.environ.get('USE', '')
622 final_flags = '%s %s' % (existing, new_flags)
623 extra_env = {'USE': final_flags.strip()}
624 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600625
626
David Burger0f9dd4e2019-10-08 12:33:42 -0600627def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600628 """Check if |build_target| builds |atom| (has it in its depgraph)."""
629 cros_build_lib.AssertInsideChroot()
630
Chris McDonalda22b74f2019-11-22 13:55:06 -0700631 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600632 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600633
634
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600635def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600636 """Returns the current Chrome version for the board (or in buildroot).
637
638 Args:
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600639 build_target (build_target_util.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700640
641 Returns:
642 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600643 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600644 # TODO(crbug/1019770): Long term we should not need the try/catch here once
645 # the builds function above only returns True for chrome when
646 # determine_chrome_version will succeed.
647 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700648 cpv = portage_util.PortageqBestVisible(
649 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600650 except cros_build_lib.RunCommandError as e:
651 # Return None because portage failed when trying to determine the chrome
652 # version.
653 logging.warning('Caught exception in determine_chrome_package: %s', e)
654 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600655 # Something like 78.0.3877.4_rc -> 78.0.3877.4
656 return cpv.version_no_rev.partition('_')[0]
657
658
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600659def determine_android_package(board):
660 """Returns the active Android container package in use by the board.
661
662 Args:
663 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700664
665 Returns:
666 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600667 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600668 try:
669 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600670 except cros_build_lib.RunCommandError as e:
671 # Return None because a command (likely portage) failed when trying to
672 # determine the package.
673 logging.warning('Caught exception in determine_android_package: %s', e)
674 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600675
Alex Kleinad6b48a2020-01-08 16:57:41 -0700676 # We assume there is only one Android package in the depgraph.
677 for package in packages:
678 if package.startswith('chromeos-base/android-container-') or \
679 package.startswith('chromeos-base/android-vm-'):
680 return package
681 return None
682
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600683
684def determine_android_version(boards=None):
685 """Determine the current Android version in buildroot now and return it.
686
687 This uses the typical portage logic to determine which version of Android
688 is active right now in the buildroot.
689
690 Args:
691 boards: List of boards to check version of.
692
693 Returns:
694 The Android build ID of the container for the boards.
695
696 Raises:
697 NoAndroidVersionError: if no unique Android version can be determined.
698 """
699 if not boards:
700 return None
701 # Verify that all boards have the same version.
702 version = None
703 for board in boards:
704 package = determine_android_package(board)
705 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600706 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600707 cpv = portage_util.SplitCPV(package)
708 if not cpv:
709 raise NoAndroidVersionError(
710 'Android version could not be determined for %s' % board)
711 if not version:
712 version = cpv.version_no_rev
713 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700714 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
715 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600716 return version
717
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700718
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600719def determine_android_branch(board):
720 """Returns the Android branch in use by the active container ebuild."""
721 try:
722 android_package = determine_android_package(board)
723 except cros_build_lib.RunCommandError:
724 raise NoAndroidBranchError(
725 'Android branch could not be determined for %s' % board)
726 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600727 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600728 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
729 # We assume all targets pull from the same branch and that we always
730 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
731 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
732 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
733 for target in targets:
734 if target in ebuild_content:
735 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
736 if branch is not None:
737 return branch.group(1)
738 raise NoAndroidBranchError(
739 'Android branch could not be determined for %s (ebuild empty?)' % board)
740
741
742def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600743 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600744 try:
745 android_package = determine_android_package(board)
746 except cros_build_lib.RunCommandError:
747 raise NoAndroidTargetError(
748 'Android Target could not be determined for %s' % board)
749 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600750 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600751 if android_package.startswith('chromeos-base/android-vm-'):
752 return 'bertha'
753 elif android_package.startswith('chromeos-base/android-container-'):
754 return 'cheets'
755
756 raise NoAndroidTargetError(
757 'Android Target cannot be determined for the package: %s' %
758 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600759
760
761def determine_platform_version():
762 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600763 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600764 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
765 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600766
767
768def determine_milestone_version():
769 """Returns the platform version from the source root."""
770 # Milestone version is something like '79'.
771 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
772 return version.chrome_branch
773
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700774
Michael Mortensen009cb662019-10-21 11:38:43 -0600775def determine_full_version():
776 """Returns the full version from the source root."""
777 # Full version is something like 'R79-12575.0.0'.
778 milestone_version = determine_milestone_version()
779 platform_version = determine_platform_version()
780 full_version = ('R%s-%s' % (milestone_version, platform_version))
781 return full_version