blob: 94eea19fae1acc81414d582d0157a6ad3a45e6a3 [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Mike Frysingerbafb3182020-02-21 03:15:43 -050036
37assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
38
39
Alex Klein87531182019-08-12 15:23:37 -060040# Registered handlers for uprevving versioned packages.
41_UPREV_FUNCS = {}
42
Alex Kleineb77ffa2019-05-28 14:47:44 -060043
44class Error(Exception):
45 """Module's base error class."""
46
47
Alex Klein4de25e82019-08-05 15:58:39 -060048class UnknownPackageError(Error):
49 """Uprev attempted for a package without a registered handler."""
50
51
Alex Kleineb77ffa2019-05-28 14:47:44 -060052class UprevError(Error):
53 """An error occurred while uprevving packages."""
54
55
Michael Mortensenb70e8a82019-10-10 18:43:41 -060056class NoAndroidVersionError(Error):
57 """An error occurred while trying to determine the android version."""
58
59
60class NoAndroidBranchError(Error):
61 """An error occurred while trying to determine the android branch."""
62
63
64class NoAndroidTargetError(Error):
65 """An error occurred while trying to determine the android target."""
66
67
Alex Klein4de25e82019-08-05 15:58:39 -060068class AndroidIsPinnedUprevError(UprevError):
69 """Raised when we try to uprev while Android is pinned."""
70
71 def __init__(self, new_android_atom):
72 """Initialize a AndroidIsPinnedUprevError.
73
74 Args:
75 new_android_atom: The Android atom that we failed to
76 uprev to, due to Android being pinned.
77 """
78 assert new_android_atom
79 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
80 new_android_atom)
81 super(AndroidIsPinnedUprevError, self).__init__(msg)
82 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060083
84
Yaakov Shaul1eafe832019-09-10 16:50:26 -060085class EbuildManifestError(Error):
86 """Error when running ebuild manifest."""
87
88
Andrew Lamb9563a152019-12-04 11:42:18 -070089class GeneratedCrosConfigFilesError(Error):
90 """Error when cros_config_schema does not produce expected files"""
91
92 def __init__(self, expected_files, found_files):
93 msg = ('Expected to find generated C files: %s. Actually found: %s' %
94 (expected_files, found_files))
95 super(GeneratedCrosConfigFilesError, self).__init__(msg)
96
Alex Klein7a3a7dd2020-01-08 16:44:38 -070097
Yaakov Shaul730814a2019-09-10 13:58:25 -060098UprevVersionedPackageModifications = collections.namedtuple(
99 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -0600100
Yaakov Shaul730814a2019-09-10 13:58:25 -0600101
102class UprevVersionedPackageResult(object):
103 """Data object for uprev_versioned_package."""
104
105 def __init__(self):
106 self.modified = []
107
108 def add_result(self, new_version, modified_files):
109 """Adds version/ebuilds tuple to result.
110
111 Args:
112 new_version: New version number of package.
113 modified_files: List of files modified for the given version.
114 """
115 result = UprevVersionedPackageModifications(new_version, modified_files)
116 self.modified.append(result)
117 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600118
119 @property
120 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600121 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600122
123
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600124def patch_ebuild_vars(ebuild_path, variables):
125 """Updates variables in ebuild.
126
127 Use this function rather than portage_util.EBuild.UpdateEBuild when you
128 want to preserve the variable position and quotes within the ebuild.
129
130 Args:
131 ebuild_path: The path of the ebuild.
132 variables: Dictionary of variables to update in ebuild.
133 """
134 try:
135 for line in fileinput.input(ebuild_path, inplace=1):
136 varname, eq, _ = line.partition('=')
137 if eq == '=' and varname.strip() in variables:
138 value = variables[varname]
139 sys.stdout.write('%s="%s"\n' % (varname, value))
140 else:
141 sys.stdout.write(line)
142 finally:
143 fileinput.close()
144
145
Alex Klein87531182019-08-12 15:23:37 -0600146def uprevs_versioned_package(package):
147 """Decorator to register package uprev handlers."""
148 assert package
149
150 def register(func):
151 """Registers |func| as a handler for |package|."""
152 _UPREV_FUNCS[package] = func
153
154 @functools.wraps(func)
155 def pass_through(*args, **kwargs):
156 return func(*args, **kwargs)
157
158 return pass_through
159
160 return register
161
162
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700163def uprev_android(tracking_branch,
164 android_package,
165 android_build_branch,
166 chroot,
167 build_targets=None,
168 android_version=None,
Alex Klein4de25e82019-08-05 15:58:39 -0600169 android_gts_build_branch=None):
170 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700171 command = [
172 'cros_mark_android_as_stable',
173 '--tracking_branch=%s' % tracking_branch,
174 '--android_package=%s' % android_package,
175 '--android_build_branch=%s' % android_build_branch,
176 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600177 if build_targets:
178 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
179 if android_version:
180 command.append('--force_version=%s' % android_version)
181 if android_gts_build_branch:
182 command.append('--android_gts_build_branch=%s' % android_gts_build_branch)
183
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700184 result = cros_build_lib.run(
185 command,
186 stdout=True,
187 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500188 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700189 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600190
Mike Frysinger88d96362020-02-14 19:05:45 -0500191 portage_atom_string = result.stdout.strip()
192 android_atom = None
193 if portage_atom_string:
194 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600195 if not android_atom:
196 logging.info('Found nothing to rev.')
197 return None
198
199 for target in build_targets or []:
200 # Sanity check: We should always be able to merge the version of
201 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700202 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600203 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700204 cros_build_lib.run(
205 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600206 except cros_build_lib.RunCommandError:
207 logging.error(
208 'Cannot emerge-%s =%s\nIs Android pinned to an older '
209 'version?', target, android_atom)
210 raise AndroidIsPinnedUprevError(android_atom)
211
212 return android_atom
213
214
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700215def uprev_build_targets(build_targets,
216 overlay_type,
217 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600218 output_dir=None):
219 """Uprev the set provided build targets, or all if not specified.
220
221 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600222 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600223 whose overlays should be uprevved, empty or None for all.
224 overlay_type (str): One of the valid overlay types except None (see
225 constants.VALID_OVERLAYS).
226 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
227 output_dir (str|None): The path to optionally dump result files.
228 """
229 # Need a valid overlay, but exclude None.
230 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
231
232 if build_targets:
233 overlays = portage_util.FindOverlaysForBoards(
234 overlay_type, boards=[t.name for t in build_targets])
235 else:
236 overlays = portage_util.FindOverlays(overlay_type)
237
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700238 return uprev_overlays(
239 overlays,
240 build_targets=build_targets,
241 chroot=chroot,
242 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600243
244
245def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
246 """Uprev the given overlays.
247
248 Args:
249 overlays (list[str]): The list of overlay paths.
Alex Klein2960c752020-03-09 13:43:38 -0600250 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600251 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
252 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
253 output_dir (str|None): The path to optionally dump result files.
254
255 Returns:
256 list[str] - The paths to all of the modified ebuild files. This includes the
257 new files that were added (i.e. the new versions) and all of the removed
258 files (i.e. the old versions).
259 """
260 assert overlays
261
262 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
263
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700264 uprev_manager = uprev_lib.UprevOverlayManager(
265 overlays,
266 manifest,
267 build_targets=build_targets,
268 chroot=chroot,
269 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600270 uprev_manager.uprev()
271
272 return uprev_manager.modified_ebuilds
273
274
Alex Klein87531182019-08-12 15:23:37 -0600275def uprev_versioned_package(package, build_targets, refs, chroot):
276 """Call registered uprev handler function for the package.
277
278 Args:
279 package (portage_util.CPV): The package being uprevved.
Alex Klein2960c752020-03-09 13:43:38 -0600280 build_targets (list[build_target_lib.BuildTarget]): The build targets to
Alex Klein87531182019-08-12 15:23:37 -0600281 clean on a successful uprev.
282 refs (list[uprev_lib.GitRef]):
283 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
284
285 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600286 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600287 """
288 assert package
289
290 if package.cp not in _UPREV_FUNCS:
291 raise UnknownPackageError(
292 'Package "%s" does not have a registered handler.' % package.cp)
293
Andrew Lambea9a8a22019-12-12 14:03:43 -0700294 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600295
296
Evan Hernandezb51f1522019-08-15 11:29:40 -0600297# TODO(evanhernandez): Remove this. Only a quick hack for testing.
298@uprevs_versioned_package('sample/sample')
299def uprev_sample(*_args, **_kwargs):
300 """Mimics an uprev by changing files in sandbox repos.
301
302 See: uprev_versioned_package.
303 """
304 paths = [
305 os.path.join(constants.SOURCE_ROOT, 'infra/dummies', repo, 'sample.txt')
306 for repo in ('general-sandbox', 'merge-sandbox')
307 ]
308
Yaakov Shaul730814a2019-09-10 13:58:25 -0600309 return UprevVersionedPackageResult().add_result('1.2.3', paths)
Evan Hernandezb51f1522019-08-15 11:29:40 -0600310
311
Yaakov Shaul395ae832019-09-09 14:45:32 -0600312@uprevs_versioned_package('afdo/kernel-profiles')
313def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600314 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600315
316 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600317
318 Raises:
319 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600320 """
321 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
322 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
323
David Burger92485342019-09-10 17:52:45 -0600324 with open(path, 'r') as f:
325 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600326
Yaakov Shaul730814a2019-09-10 13:58:25 -0600327 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600328 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600329 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
330 'sys-kernel', version)
331 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
332 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600333 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
334 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600335 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600336 patch_ebuild_vars(ebuild_path,
337 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600338
339 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600340 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400341 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600342 except cros_build_lib.RunCommandError as e:
343 raise EbuildManifestError(
344 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600345 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600346
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600347 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600348
Yaakov Shaul730814a2019-09-10 13:58:25 -0600349 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
350
351 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600352
353
Trent Beginaf51f1b2020-03-09 17:35:31 -0600354@uprevs_versioned_package('chromeos-base/termina-image-amd64')
355def uprev_termina_amd64(_build_targets, _refs, chroot):
356 """Updates termina amd64 VM - chromeos-base/termina-image-amd64.
357
358 See: uprev_versioned_package.
359 """
360 return uprev_termina('termina-image-amd64', chroot)
361
362
363@uprevs_versioned_package('chromeos-base/termina-image-arm')
364def uprev_termina_arm(_build_targets, _refs, chroot):
365 """Updates termina arm VM - chromeos-base/termina-image-arm.
366
367 See: uprev_versioned_package.
368 """
369 return uprev_termina('termina-image-arm', chroot)
370
371
372def uprev_termina(package, chroot):
373 """Helper function to uprev termina VM.
374
375 Args:
376 package (string): name of the package
377 chroot (chroot_lib.Chroot): specify a chroot to enter.
378
379 Returns:
380 UprevVersionedPackageResult: The result.
381 """
382 package_path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos-base',
383 package)
384 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
385 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
386
387
Trent Begin315d9d92019-12-03 21:55:53 -0700388@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Beginaf51f1b2020-03-09 17:35:31 -0600389def uprev_sludge(_build_targets, _refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700390 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
391
392 See: uprev_versioned_package.
393 """
394 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700395 package_path = os.path.join('src', 'private-overlays',
396 'project-wilco-private', 'chromeos-base', package)
397 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700398
Trent Begin6daa8702020-01-29 14:58:12 -0700399 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700400
401
Trent Begin6daa8702020-01-29 14:58:12 -0700402def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700403 """Changes the package ebuild's version to match the version pin file.
404
405 Args:
406 package_path: The path of the package relative to the src root. This path
407 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700408 version_pin_path: The path of the version_pin file that contains only a
409 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700410 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700411 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700412
413 Returns:
414 UprevVersionedPackageResult: The result.
415 """
416 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700417
418 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
419 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700420 if not ebuild_paths:
421 raise UprevError('No ebuilds found for %s' % package)
422 elif len(ebuild_paths) > 1:
423 raise UprevError('Multiple ebuilds found for %s' % package)
424 else:
425 ebuild_path = ebuild_paths[0]
426
Trent Begind943df92020-02-25 10:30:10 -0700427 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
428 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700429 new_ebuild_path = os.path.join(package_path,
430 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700431 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
432 os.rename(ebuild_path, new_ebuild_src_path)
Trent Begin4a11a632020-02-28 12:59:58 -0700433 manifest_src_path = os.path.join(package_src_path, 'Manifest')
Trent Begind943df92020-02-25 10:30:10 -0700434 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
435 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700436
Trent Begin6daa8702020-01-29 14:58:12 -0700437 try:
Trent Begind943df92020-02-25 10:30:10 -0700438 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700439 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700440 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700441 'Unable to update manifest for %s: %s' % (package, e.stderr))
442
Trent Begin315d9d92019-12-03 21:55:53 -0700443 result = UprevVersionedPackageResult()
Trent Begin4a11a632020-02-28 12:59:58 -0700444 result.add_result(version,
445 [new_ebuild_src_path, ebuild_path, manifest_src_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700446 return result
447
448
Alex Klein87531182019-08-12 15:23:37 -0600449@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700450def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600451 """Uprev chrome and its related packages.
452
453 See: uprev_versioned_package.
454 """
455 # Determine the version from the refs (tags), i.e. the chrome versions are the
456 # tag names.
457 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
458
459 uprev_manager = uprev_lib.UprevChromeManager(
460 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600461 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600462 # Start with chrome itself, as we can't do anything else unless chrome
463 # uprevs successfully.
464 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600465 return result
Alex Klein87531182019-08-12 15:23:37 -0600466
467 # With a successful chrome rev, also uprev related packages.
468 for package in constants.OTHER_CHROME_PACKAGES:
469 uprev_manager.uprev(package)
470
David Burger37f48672019-09-18 17:07:56 -0600471 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600472
473
Andrew Lamb9563a152019-12-04 11:42:18 -0700474def _generate_platform_c_files(replication_config, chroot):
475 """Generates platform C files from a platform JSON payload.
476
477 Args:
478 replication_config (replication_config_pb2.ReplicationConfig): A
479 ReplicationConfig that has already been run. If it produced a
480 build_config.json file, that file will be used to generate platform C
481 files. Otherwise, nothing will be generated.
482 chroot (chroot_lib.Chroot): The chroot to use to generate.
483
484 Returns:
485 A list of generated files.
486 """
487 # Generate the platform C files from the build config. Note that it would be
488 # more intuitive to generate the platform C files from the platform config;
489 # however, cros_config_schema does not allow this, because the platform config
490 # payload is not always valid input. For example, if a property is both
491 # 'required' and 'build-only', it will fail schema validation. Thus, use the
492 # build config, and use '-f' to filter.
493 build_config_path = [
494 rule.destination_path
495 for rule in replication_config.file_replication_rules
496 if rule.destination_path.endswith('build_config.json')
497 ]
498
499 if not build_config_path:
500 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700501 'No build_config.json found, will not generate platform C files. '
502 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700503 return []
504
505 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700506 raise ValueError('Expected at most one build_config.json destination path. '
507 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700508
509 build_config_path = build_config_path[0]
510
511 # Paths to the build_config.json and dir to output C files to, in the
512 # chroot.
513 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
514 build_config_path)
515 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
516 os.path.dirname(build_config_path))
517
518 command = [
519 'cros_config_schema', '-m', build_config_chroot_path, '-g',
520 generated_output_chroot_dir, '-f', '"TRUE"'
521 ]
522
523 cros_build_lib.run(
524 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
525
526 # A relative (to the source root) path to the generated C files.
527 generated_output_dir = os.path.dirname(build_config_path)
528 generated_files = []
529 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
530 for f in expected_c_files:
531 if os.path.exists(
532 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
533 generated_files.append(os.path.join(generated_output_dir, f))
534
535 if len(expected_c_files) != len(generated_files):
536 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
537
538 return generated_files
539
540
Andrew Lambe836f222019-12-09 12:27:38 -0700541def _get_private_overlay_package_root(ref, package):
542 """Returns the absolute path to the root of a given private overlay.
543
544 Args:
545 ref (uprev_lib.GitRef): GitRef for the private overlay.
546 package (str): Path to the package in the overlay.
547 """
548 # There might be a cleaner way to map from package -> path within the source
549 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700550 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700551 match = re.match(private_overlay_ref_pattern, ref.path)
552 if not match:
553 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
554 (private_overlay_ref_pattern, ref))
555
556 overlay = match.group(1)
557
558 return os.path.join(constants.SOURCE_ROOT,
559 'src/private-overlays/overlay-%s-private' % overlay,
560 package)
561
562
Andrew Lambea9a8a22019-12-12 14:03:43 -0700563@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
564def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700565 """Replicate a private cros_config change to the corresponding public config.
566
Alex Kleinad6b48a2020-01-08 16:57:41 -0700567 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700568 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700569 package = 'chromeos-base/chromeos-config-bsp'
570
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700571 if len(refs) != 1:
572 raise ValueError('Expected exactly one ref, actual %s' % refs)
573
574 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700575 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700576 replication_config_path = os.path.join(package_root,
577 'replication_config.jsonpb')
578
579 try:
580 replication_config = json_format.Parse(
581 osutils.ReadFile(replication_config_path),
582 replication_config_pb2.ReplicationConfig())
583 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700584 raise ValueError(
585 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700586
587 replication_lib.Replicate(replication_config)
588
589 modified_files = [
590 rule.destination_path
591 for rule in replication_config.file_replication_rules
592 ]
593
Andrew Lamb9563a152019-12-04 11:42:18 -0700594 # The generated platform C files are not easily filtered by replication rules,
595 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
596 # files cannot. Therefore, replicate and filter the JSON payloads, and then
597 # generate filtered C files from the JSON payload.
598 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700599
600 # Use the private repo's commit hash as the new version.
601 new_private_version = refs[0].revision
602
Andrew Lamb988f4da2019-12-10 10:16:43 -0700603 # modified_files should contain only relative paths at this point, but the
604 # returned UprevVersionedPackageResult must contain only absolute paths.
605 for i, modified_file in enumerate(modified_files):
606 assert not os.path.isabs(modified_file)
607 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
608
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700609 return UprevVersionedPackageResult().add_result(new_private_version,
610 modified_files)
611
612
Alex Kleinbbef2b32019-08-27 10:38:50 -0600613def get_best_visible(atom, build_target=None):
614 """Returns the best visible CPV for the given atom.
615
616 Args:
617 atom (str): The atom to look up.
Alex Klein2960c752020-03-09 13:43:38 -0600618 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600619 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700620
621 Returns:
622 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600623 """
David Burger1e0fe232019-07-01 14:52:07 -0600624 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600625
626 board = build_target.name if build_target else None
627 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600628
629
Alex Klein149fd3b2019-12-16 16:01:05 -0700630def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600631 """Check if a prebuilt exists.
632
633 Args:
634 atom (str): The package whose prebuilt is being queried.
Alex Klein2960c752020-03-09 13:43:38 -0600635 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600636 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700637 useflags: Any additional USE flags that should be set. May be a string
638 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700639
640 Returns:
641 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600642 """
643 assert atom
644
645 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700646 extra_env = None
647 if useflags:
648 new_flags = useflags
649 if not isinstance(useflags, six.string_types):
650 new_flags = ' '.join(useflags)
651
652 existing = os.environ.get('USE', '')
653 final_flags = '%s %s' % (existing, new_flags)
654 extra_env = {'USE': final_flags.strip()}
655 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600656
657
David Burger0f9dd4e2019-10-08 12:33:42 -0600658def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600659 """Check if |build_target| builds |atom| (has it in its depgraph)."""
660 cros_build_lib.AssertInsideChroot()
661
Chris McDonalda22b74f2019-11-22 13:55:06 -0700662 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600663 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600664
665
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600666def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600667 """Returns the current Chrome version for the board (or in buildroot).
668
669 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600670 build_target (build_target_lib.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700671
672 Returns:
673 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600674 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600675 # TODO(crbug/1019770): Long term we should not need the try/catch here once
676 # the builds function above only returns True for chrome when
677 # determine_chrome_version will succeed.
678 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700679 cpv = portage_util.PortageqBestVisible(
680 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600681 except cros_build_lib.RunCommandError as e:
682 # Return None because portage failed when trying to determine the chrome
683 # version.
684 logging.warning('Caught exception in determine_chrome_package: %s', e)
685 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600686 # Something like 78.0.3877.4_rc -> 78.0.3877.4
687 return cpv.version_no_rev.partition('_')[0]
688
689
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600690def determine_android_package(board):
691 """Returns the active Android container package in use by the board.
692
693 Args:
694 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700695
696 Returns:
697 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600698 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600699 try:
700 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600701 except cros_build_lib.RunCommandError as e:
702 # Return None because a command (likely portage) failed when trying to
703 # determine the package.
704 logging.warning('Caught exception in determine_android_package: %s', e)
705 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600706
Alex Kleinad6b48a2020-01-08 16:57:41 -0700707 # We assume there is only one Android package in the depgraph.
708 for package in packages:
709 if package.startswith('chromeos-base/android-container-') or \
710 package.startswith('chromeos-base/android-vm-'):
711 return package
712 return None
713
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600714
715def determine_android_version(boards=None):
716 """Determine the current Android version in buildroot now and return it.
717
718 This uses the typical portage logic to determine which version of Android
719 is active right now in the buildroot.
720
721 Args:
722 boards: List of boards to check version of.
723
724 Returns:
725 The Android build ID of the container for the boards.
726
727 Raises:
728 NoAndroidVersionError: if no unique Android version can be determined.
729 """
730 if not boards:
731 return None
732 # Verify that all boards have the same version.
733 version = None
734 for board in boards:
735 package = determine_android_package(board)
736 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600737 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600738 cpv = portage_util.SplitCPV(package)
739 if not cpv:
740 raise NoAndroidVersionError(
741 'Android version could not be determined for %s' % board)
742 if not version:
743 version = cpv.version_no_rev
744 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700745 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
746 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600747 return version
748
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700749
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600750def determine_android_branch(board):
751 """Returns the Android branch in use by the active container ebuild."""
752 try:
753 android_package = determine_android_package(board)
754 except cros_build_lib.RunCommandError:
755 raise NoAndroidBranchError(
756 'Android branch could not be determined for %s' % board)
757 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600758 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600759 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
760 # We assume all targets pull from the same branch and that we always
761 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
762 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
763 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
764 for target in targets:
765 if target in ebuild_content:
766 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
767 if branch is not None:
768 return branch.group(1)
769 raise NoAndroidBranchError(
770 'Android branch could not be determined for %s (ebuild empty?)' % board)
771
772
773def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600774 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600775 try:
776 android_package = determine_android_package(board)
777 except cros_build_lib.RunCommandError:
778 raise NoAndroidTargetError(
779 'Android Target could not be determined for %s' % board)
780 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600781 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600782 if android_package.startswith('chromeos-base/android-vm-'):
783 return 'bertha'
784 elif android_package.startswith('chromeos-base/android-container-'):
785 return 'cheets'
786
787 raise NoAndroidTargetError(
788 'Android Target cannot be determined for the package: %s' %
789 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600790
791
792def determine_platform_version():
793 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600794 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600795 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
796 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600797
798
799def determine_milestone_version():
800 """Returns the platform version from the source root."""
801 # Milestone version is something like '79'.
802 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
803 return version.chrome_branch
804
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700805
Michael Mortensen009cb662019-10-21 11:38:43 -0600806def determine_full_version():
807 """Returns the full version from the source root."""
808 # Full version is something like 'R79-12575.0.0'.
809 milestone_version = determine_milestone_version()
810 platform_version = determine_platform_version()
811 full_version = ('R%s-%s' % (milestone_version, platform_version))
812 return full_version