blob: 5fd7369a60afbb68a774f30a83f6fe774c8b2f12 [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# Copyright 2019 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Package utility functionality."""
6
Yaakov Shaul730814a2019-09-10 13:58:25 -06007import collections
Ben Reiche779cf42020-12-15 03:21:31 +00008from distutils.version import LooseVersion
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -06009import fileinput
Alex Klein87531182019-08-12 15:23:37 -060010import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060011import json
Chris McDonaldf7c03d42021-07-21 11:54:26 -060012import logging
Evan Hernandezb51f1522019-08-15 11:29:40 -060013import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060014import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060015import sys
Alex Klein68a28712021-11-08 11:08:30 -070016from typing import Iterable, List, NamedTuple, Optional, TYPE_CHECKING, Union
Alex Klein87531182019-08-12 15:23:37 -060017
Mike Frysinger2c024062021-05-22 15:43:22 -040018from chromite.third_party.google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060019
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060021from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060022from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060023from chromite.lib import cros_build_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import git
Michael Mortensende716a12020-05-15 11:27:00 -060025from chromite.lib import image_lib
Michael Mortensenb70e8a82019-10-10 18:43:41 -060026from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070028from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060029from chromite.lib import uprev_lib
Alex Klein18a60af2020-06-11 12:08:47 -060030from chromite.lib.parser import package_info
Shao-Chuan Lee05e51142021-11-24 12:27:37 +090031from chromite.service import android
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Mike Frysinger68796b52019-08-25 00:04:27 -040033
Alex Klein5caab872021-09-10 11:44:37 -060034if TYPE_CHECKING:
35 from chromite.lib import build_target_lib
Alex Klein16ea1b32021-10-01 15:48:50 -060036 from chromite.lib import chroot_lib
Chris McDonaldf7c03d42021-07-21 11:54:26 -060037
Alex Klein36b117f2019-09-30 15:13:46 -060038if cros_build_lib.IsInsideChroot():
Alex Klein6becabc2020-09-11 14:03:05 -060039 from chromite.lib import depgraph
Alex Klein36b117f2019-09-30 15:13:46 -060040 from chromite.service import dependency
41
Alex Klein87531182019-08-12 15:23:37 -060042# Registered handlers for uprevving versioned packages.
43_UPREV_FUNCS = {}
44
Alex Kleineb77ffa2019-05-28 14:47:44 -060045
46class Error(Exception):
47 """Module's base error class."""
48
49
Alex Klein4de25e82019-08-05 15:58:39 -060050class UnknownPackageError(Error):
51 """Uprev attempted for a package without a registered handler."""
52
53
Alex Kleineb77ffa2019-05-28 14:47:44 -060054class UprevError(Error):
55 """An error occurred while uprevving packages."""
56
57
Michael Mortensenb70e8a82019-10-10 18:43:41 -060058class NoAndroidVersionError(Error):
59 """An error occurred while trying to determine the android version."""
60
61
62class NoAndroidBranchError(Error):
63 """An error occurred while trying to determine the android branch."""
64
65
66class NoAndroidTargetError(Error):
67 """An error occurred while trying to determine the android target."""
68
69
Benjamin Shai8e14eb12022-04-28 22:55:16 +000070class KernelVersionError(Error):
71 """An error occurred while trying to determine the kernel version."""
72
73
Alex Klein4de25e82019-08-05 15:58:39 -060074class AndroidIsPinnedUprevError(UprevError):
75 """Raised when we try to uprev while Android is pinned."""
76
77 def __init__(self, new_android_atom):
78 """Initialize a AndroidIsPinnedUprevError.
79
80 Args:
81 new_android_atom: The Android atom that we failed to
82 uprev to, due to Android being pinned.
83 """
84 assert new_android_atom
85 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
86 new_android_atom)
Jae Hoon Kimad176b82021-07-26 19:29:29 +000087 super().__init__(msg)
Alex Klein4de25e82019-08-05 15:58:39 -060088 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060089
90
Andrew Lamb9563a152019-12-04 11:42:18 -070091class GeneratedCrosConfigFilesError(Error):
92 """Error when cros_config_schema does not produce expected files"""
93
94 def __init__(self, expected_files, found_files):
95 msg = ('Expected to find generated C files: %s. Actually found: %s' %
96 (expected_files, found_files))
Jae Hoon Kimad176b82021-07-26 19:29:29 +000097 super().__init__(msg)
Andrew Lamb9563a152019-12-04 11:42:18 -070098
Alex Klein7a3a7dd2020-01-08 16:44:38 -070099
Alex Klein6becabc2020-09-11 14:03:05 -0600100NeedsChromeSourceResult = collections.namedtuple('NeedsChromeSourceResult', (
101 'needs_chrome_source',
102 'builds_chrome',
103 'packages',
104 'missing_chrome_prebuilt',
105 'missing_follower_prebuilt',
Alex Klein9ce3f682021-06-23 15:06:44 -0600106 'local_uprev',
Alex Klein6becabc2020-09-11 14:03:05 -0600107))
108
109
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600110def patch_ebuild_vars(ebuild_path, variables):
111 """Updates variables in ebuild.
112
113 Use this function rather than portage_util.EBuild.UpdateEBuild when you
114 want to preserve the variable position and quotes within the ebuild.
115
116 Args:
117 ebuild_path: The path of the ebuild.
118 variables: Dictionary of variables to update in ebuild.
119 """
120 try:
121 for line in fileinput.input(ebuild_path, inplace=1):
Madeleine Hardt8ae7f102022-03-24 20:26:11 +0000122 for var, value in variables.items():
123 line = re.sub(fr'\b{var}=\S+', f'{var}="{value}"', line)
124 sys.stdout.write(line)
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600125 finally:
126 fileinput.close()
127
128
Alex Klein87531182019-08-12 15:23:37 -0600129def uprevs_versioned_package(package):
130 """Decorator to register package uprev handlers."""
131 assert package
132
133 def register(func):
134 """Registers |func| as a handler for |package|."""
135 _UPREV_FUNCS[package] = func
136
137 @functools.wraps(func)
138 def pass_through(*args, **kwargs):
139 return func(*args, **kwargs)
140
141 return pass_through
142
143 return register
144
145
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900146class UprevAndroidResult(NamedTuple):
147 """Results of an Android uprev."""
148 revved: bool
149 android_atom: str = None
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900150 modified_files: List[str] = None
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900151
152
153def uprev_android(
154 android_package: str,
155 chroot: 'chroot_lib.Chroot',
156 build_targets: Optional[List['build_target_lib.BuildTarget']] = None,
157 android_build_branch: Optional[str] = None,
158 android_version: Optional[str] = None,
159 skip_commit: bool = False) -> UprevAndroidResult:
160 """Performs an Android uprev by calling cros_mark_android_as_stable.
161
162 Args:
163 android_package: The Android package to uprev.
164 chroot: The chroot to enter.
165 build_targets: List of build targets to cleanup after uprev.
166 android_build_branch: Override the default Android branch corresponding to
167 the package.
168 android_version: Uprev to the particular version. By default the latest
169 available version is used.
170 skip_commit: Whether to skip committing the change after a successful uprev.
171
172 Returns:
173 The uprev result containing:
174 revved: Whether an uprev happened.
175 android_atom: If revved, the portage atom for the revved Android ebuild.
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900176 modified_files: If revved, list of files being modified.
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900177 """
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700178 command = [
179 'cros_mark_android_as_stable',
Shao-Chuan Lee85ba7ce2021-02-09 13:50:11 +0900180 f'--android_package={android_package}',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700181 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600182 if build_targets:
Shao-Chuan Lee85ba7ce2021-02-09 13:50:11 +0900183 command.append(f'--boards={":".join(bt.name for bt in build_targets)}')
Shao-Chuan Leea4b4f302021-05-12 14:40:20 +0900184 if android_build_branch:
185 command.append(f'--android_build_branch={android_build_branch}')
Alex Klein4de25e82019-08-05 15:58:39 -0600186 if android_version:
Shao-Chuan Lee85ba7ce2021-02-09 13:50:11 +0900187 command.append(f'--force_version={android_version}')
188 if skip_commit:
189 command.append('--skip_commit')
Alex Klein4de25e82019-08-05 15:58:39 -0600190
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700191 result = cros_build_lib.run(
192 command,
193 stdout=True,
194 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500195 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700196 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600197
Shao-Chuan Leedea458f2021-11-25 23:46:53 +0900198 # cros_mark_android_as_stable prints the uprev result to stdout as JSON in a
199 # single line. We only take the last line from stdout to make sure no junk
200 # output is included (e.g. messages from bashrc scripts that run upon entering
201 # the chroot.)
202 output = json.loads(result.stdout.strip().splitlines()[-1])
203
204 if not output['revved']:
Alex Klein4de25e82019-08-05 15:58:39 -0600205 logging.info('Found nothing to rev.')
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900206 return UprevAndroidResult(revved=False)
207
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900208 android_atom = output['android_atom']
Alex Klein4de25e82019-08-05 15:58:39 -0600209
210 for target in build_targets or []:
211 # Sanity check: We should always be able to merge the version of
212 # Android we just unmasked.
Shao-Chuan Lee85ba7ce2021-02-09 13:50:11 +0900213 command = [f'emerge-{target.name}', '-p', '--quiet', f'={android_atom}']
Alex Klein4de25e82019-08-05 15:58:39 -0600214 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700215 cros_build_lib.run(
216 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600217 except cros_build_lib.RunCommandError:
218 logging.error(
219 'Cannot emerge-%s =%s\nIs Android pinned to an older '
220 'version?', target, android_atom)
221 raise AndroidIsPinnedUprevError(android_atom)
222
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900223 return UprevAndroidResult(revved=True,
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900224 android_atom=android_atom,
225 modified_files=output['modified_files'])
226
227
228def uprev_android_lkgb(android_package: str,
229 build_targets: List['build_target_lib.BuildTarget'],
230 chroot: 'chroot_lib.Chroot'
231 ) -> uprev_lib.UprevVersionedPackageResult:
232 """Uprevs an Android package to the version specified in the LKGB file.
233
234 This is the PUpr handler for Android packages, triggered whenever the
235 corresponding LKGB file is being updated.
236
237 PUpr for Android does not test the uprev change in CQ; instead we run separate
238 jobs to test new Android versions, and we write the latest vetted version to
239 the LKGB file. Find the design at go/android-uprev-recipes.
240
241 Args:
242 android_package: The Android package to uprev.
243 build_targets: List of build targets to cleanup after uprev.
244 chroot: The chroot to enter.
245
246 Returns:
247 An uprev_lib.UprevVersionedPackageResult containing the new version and a
248 list of modified files.
249 """
Shao-Chuan Lee0e787d22021-12-13 21:33:35 +0900250 android_package_dir = android.GetAndroidPackageDir(android_package)
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900251 android_version = android.ReadLKGB(android_package_dir)
252
253 result = uprev_lib.UprevVersionedPackageResult()
254 uprev_result = uprev_android(android_package, chroot,
255 build_targets=build_targets,
256 android_version=android_version,
257 skip_commit=True)
258 if not uprev_result.revved:
259 return result
260
Shao-Chuan Lee0e787d22021-12-13 21:33:35 +0900261 # cros_mark_android_as_stable returns paths relative to |android.OVERLAY_DIR|.
262 result.add_result(android_version,
263 [os.path.join(android.OVERLAY_DIR, f)
264 for f in uprev_result.modified_files])
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900265 return result
266
267
268def define_uprev_android_lkgb_handlers():
269 """Dynamically define uprev handlers for each Android package"""
270
271 def define_handler(android_package):
272 """Defines the uprev handler for an Android package."""
273 full_package_name = 'chromeos-base/' + android_package
274
275 @uprevs_versioned_package(full_package_name)
276 def _handler(build_targets, _refs, chroot):
277 return uprev_android_lkgb(android_package, build_targets, chroot)
278
279 for android_package in constants.ANDROID_ALL_PACKAGES:
280 define_handler(android_package)
281
282
283define_uprev_android_lkgb_handlers()
Alex Klein4de25e82019-08-05 15:58:39 -0600284
285
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700286def uprev_build_targets(
287 build_targets: Optional[List['build_target_lib.BuildTarget']],
288 overlay_type: str,
289 chroot: 'chroot_lib.Chroot' = None,
290 output_dir: Optional[str] = None):
Alex Kleineb77ffa2019-05-28 14:47:44 -0600291 """Uprev the set provided build targets, or all if not specified.
292
293 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700294 build_targets: The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600295 whose overlays should be uprevved, empty or None for all.
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700296 overlay_type: One of the valid overlay types except None (see
Alex Kleineb77ffa2019-05-28 14:47:44 -0600297 constants.VALID_OVERLAYS).
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700298 chroot: The chroot to clean, if desired.
299 output_dir: The path to optionally dump result files.
Alex Kleineb77ffa2019-05-28 14:47:44 -0600300 """
301 # Need a valid overlay, but exclude None.
302 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
303
304 if build_targets:
305 overlays = portage_util.FindOverlaysForBoards(
306 overlay_type, boards=[t.name for t in build_targets])
307 else:
308 overlays = portage_util.FindOverlays(overlay_type)
309
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700310 return uprev_overlays(
311 overlays,
312 build_targets=build_targets,
313 chroot=chroot,
314 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600315
316
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700317def uprev_overlays(
318 overlays: List[str],
319 build_targets: Optional[List['build_target_lib.BuildTarget']] = None,
320 chroot: Optional['chroot_lib.Chroot'] = None,
321 output_dir: Optional[str] = None) -> List[str]:
Alex Kleineb77ffa2019-05-28 14:47:44 -0600322 """Uprev the given overlays.
323
324 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700325 overlays: The list of overlay paths.
326 build_targets: The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600327 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700328 chroot: The chroot to clean, if desired.
329 output_dir: The path to optionally dump result files.
Alex Kleineb77ffa2019-05-28 14:47:44 -0600330
331 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700332 The paths to all of the modified ebuild files. This includes the new files
333 that were added (i.e. the new versions) and all of the removed files
334 (i.e. the old versions).
Alex Kleineb77ffa2019-05-28 14:47:44 -0600335 """
336 assert overlays
337
338 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
339
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700340 uprev_manager = uprev_lib.UprevOverlayManager(
341 overlays,
342 manifest,
343 build_targets=build_targets,
344 chroot=chroot,
345 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600346 uprev_manager.uprev()
347
Dhanya Ganeshd03a4162021-08-25 00:21:50 +0000348 return uprev_manager.modified_ebuilds, uprev_manager.revved_packages
Alex Kleineb77ffa2019-05-28 14:47:44 -0600349
350
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700351def uprev_versioned_package(
352 package: package_info.CPV,
353 build_targets: List['build_target_lib.BuildTarget'],
354 refs: List[uprev_lib.GitRef],
355 chroot: 'chroot_lib.Chroot') -> 'uprev_lib.UprevVersionedPackageResult':
Alex Klein87531182019-08-12 15:23:37 -0600356 """Call registered uprev handler function for the package.
357
358 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700359 package: The package being uprevved.
360 build_targets: The build targets to clean on a successful uprev.
361 refs:
362 chroot: The chroot to enter for cleaning.
Alex Klein87531182019-08-12 15:23:37 -0600363
364 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700365 The result.
Alex Klein87531182019-08-12 15:23:37 -0600366 """
367 assert package
368
369 if package.cp not in _UPREV_FUNCS:
370 raise UnknownPackageError(
371 'Package "%s" does not have a registered handler.' % package.cp)
372
Andrew Lambea9a8a22019-12-12 14:03:43 -0700373 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600374
375
Navil Perezf57ba872020-06-04 22:38:37 +0000376@uprevs_versioned_package('media-libs/virglrenderer')
377def uprev_virglrenderer(_build_targets, refs, _chroot):
378 """Updates virglrenderer ebuilds.
379
380 See: uprev_versioned_package.
381
382 Returns:
383 UprevVersionedPackageResult: The result of updating virglrenderer ebuilds.
384 """
Navil Perezf57ba872020-06-04 22:38:37 +0000385 overlay = os.path.join(constants.SOURCE_ROOT,
386 constants.CHROMIUMOS_OVERLAY_DIR)
George Engelbrechte73f2782020-06-10 14:10:46 -0600387 repo_path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
388 'virglrenderer')
389 manifest = git.ManifestCheckout.Cached(repo_path)
Navil Perezf57ba872020-06-04 22:38:37 +0000390
391 uprev_manager = uprev_lib.UprevOverlayManager([overlay], manifest)
392 # TODO(crbug.com/1066242): Ebuilds for virglrenderer are currently
Jose Magana03b5a842020-08-19 12:52:59 +1000393 # denylisted. Do not force uprevs after builder is stable and ebuilds are no
394 # longer denylisted.
Navil Perezf57ba872020-06-04 22:38:37 +0000395 uprev_manager.uprev(package_list=['media-libs/virglrenderer'], force=True)
396
George Engelbrechte73f2782020-06-10 14:10:46 -0600397 updated_files = uprev_manager.modified_ebuilds
Chris McDonald38409112020-09-24 11:24:51 -0600398 result = uprev_lib.UprevVersionedPackageResult()
Navil Perezf57ba872020-06-04 22:38:37 +0000399 result.add_result(refs[0].revision, updated_files)
400 return result
401
Jose Magana03b5a842020-08-19 12:52:59 +1000402@uprevs_versioned_package('chromeos-base/drivefs')
403def uprev_drivefs(_build_targets, refs, chroot):
404 """Updates drivefs ebuilds.
405
Harvey Yang3eee06c2021-03-18 15:47:56 +0800406 DriveFS versions follow the tag format of refs/tags/drivefs_1.2.3.
Jose Magana03b5a842020-08-19 12:52:59 +1000407 See: uprev_versioned_package.
408
409 Returns:
410 UprevVersionedPackageResult: The result of updating drivefs ebuilds.
411 """
412
Ben Reiche779cf42020-12-15 03:21:31 +0000413 DRIVEFS_PATH_PREFIX = 'src/private-overlays/chromeos-overlay/chromeos-base'
Ben Reich4f3fa1b2020-12-19 08:21:26 +0000414 result = uprev_lib.UprevVersionedPackageResult()
415 all_changed_files = []
Jose Magana03b5a842020-08-19 12:52:59 +1000416
Harvey Yang3eee06c2021-03-18 15:47:56 +0800417 DRIVEFS_REFS_PREFIX = 'refs/tags/drivefs_'
418 drivefs_version = _get_latest_version_from_refs(DRIVEFS_REFS_PREFIX, refs)
Ben Reiche779cf42020-12-15 03:21:31 +0000419 if not drivefs_version:
420 # No valid DriveFS version is identified.
Ben Reich4f3fa1b2020-12-19 08:21:26 +0000421 return result
Jose Magana03b5a842020-08-19 12:52:59 +1000422
Ben Reiche779cf42020-12-15 03:21:31 +0000423 logging.debug('DriveFS version determined from refs: %s', drivefs_version)
Jose Magana03b5a842020-08-19 12:52:59 +1000424
Ben Reich4f3fa1b2020-12-19 08:21:26 +0000425 # Attempt to uprev drivefs package.
Ben Reiche779cf42020-12-15 03:21:31 +0000426 pkg_path = os.path.join(DRIVEFS_PATH_PREFIX, 'drivefs')
Jose Magana03b5a842020-08-19 12:52:59 +1000427 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(pkg_path,
Ben Reiche779cf42020-12-15 03:21:31 +0000428 drivefs_version,
Harvey Yang3eee06c2021-03-18 15:47:56 +0800429 chroot,
430 allow_downrev=False)
Jose Magana03b5a842020-08-19 12:52:59 +1000431
432 if not uprev_result:
Ben Reich4f3fa1b2020-12-19 08:21:26 +0000433 return result
Jose Magana03b5a842020-08-19 12:52:59 +1000434 all_changed_files.extend(uprev_result.changed_files)
Ben Reiche779cf42020-12-15 03:21:31 +0000435 result.add_result(drivefs_version, all_changed_files)
Jose Magana03b5a842020-08-19 12:52:59 +1000436
437 return result
438
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800439@uprevs_versioned_package('chromeos-base/perfetto')
440def uprev_perfetto(_build_targets, refs, chroot):
441 """Updates Perfetto ebuilds.
442
Harvey Yang3eee06c2021-03-18 15:47:56 +0800443 Perfetto versions follow the tag format of refs/tags/v1.2.
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800444 See: uprev_versioned_package.
445
446 Returns:
447 UprevVersionedPackageResult: The result of updating Perfetto ebuilds.
448 """
449 result = uprev_lib.UprevVersionedPackageResult()
450
Harvey Yang3eee06c2021-03-18 15:47:56 +0800451 PERFETTO_REFS_PREFIX = 'refs/tags/v'
452 perfetto_version = _get_latest_version_from_refs(PERFETTO_REFS_PREFIX, refs)
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800453 if not perfetto_version:
454 # No valid Perfetto version is identified.
455 return result
456
457 logging.debug('Perfetto version determined from refs: %s', perfetto_version)
458
459 # Attempt to uprev perfetto package.
460 PERFETTO_PATH = 'src/third_party/chromiumos-overlay/chromeos-base/perfetto'
461
Harvey Yang3eee06c2021-03-18 15:47:56 +0800462 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(
463 PERFETTO_PATH,
464 perfetto_version,
465 chroot,
466 allow_downrev=False,
467 ref=PERFETTO_REFS_PREFIX + perfetto_version)
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800468
469 if not uprev_result:
470 return result
471
472 result.add_result(perfetto_version, uprev_result.changed_files)
473
474 return result
Navil Perezf57ba872020-06-04 22:38:37 +0000475
Yaakov Shaul395ae832019-09-09 14:45:32 -0600476@uprevs_versioned_package('afdo/kernel-profiles')
477def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600478 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600479
480 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600481
482 Raises:
483 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600484 """
485 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
486 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
487
David Burger92485342019-09-10 17:52:45 -0600488 with open(path, 'r') as f:
489 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600490
Chris McDonald38409112020-09-24 11:24:51 -0600491 result = uprev_lib.UprevVersionedPackageResult()
Alex Klein6edd4e62021-07-08 12:43:25 -0600492 for kernel_pkg, version_info in versions.items():
493 path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR,
494 'sys-kernel', kernel_pkg)
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600495 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
Alex Klein6edd4e62021-07-08 12:43:25 -0600496 '%s-9999.ebuild' % kernel_pkg)
Yaakov Shaula187b152019-09-11 12:41:32 -0600497 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
Alex Klein6edd4e62021-07-08 12:43:25 -0600498 '%s-9999.ebuild' % kernel_pkg)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600499 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600500 patch_ebuild_vars(ebuild_path,
501 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600502
503 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600504 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400505 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600506 except cros_build_lib.RunCommandError as e:
Chris McDonald38409112020-09-24 11:24:51 -0600507 raise uprev_lib.EbuildManifestError(
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600508 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600509 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600510
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600511 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600512
Yaakov Shaul730814a2019-09-10 13:58:25 -0600513 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
514
515 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600516
517
Trent Begineb624182020-07-14 10:09:45 -0600518@uprevs_versioned_package('chromeos-base/termina-dlc')
Maciek Swiech6b12f662022-01-25 16:51:19 +0000519@uprevs_versioned_package('chromeos-base/termina-tools-dlc')
520def uprev_termina_dlcs(_build_targets, _refs, chroot):
521 """Updates shared termina-dlc and termina-tools-dlc ebuilds.
522
523 termina-dlc - chromeos-base/termina-dlc
524 termina-tools-dlc - chromeos-base/termina-tools-dlc
Trent Beginaf51f1b2020-03-09 17:35:31 -0600525
526 See: uprev_versioned_package.
527 """
Maciek Swiech6b12f662022-01-25 16:51:19 +0000528 termina_dlc_pkg = 'termina-dlc'
529 termina_dlc_pkg_path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR,
530 'chromeos-base', termina_dlc_pkg)
531 tools_dlc_pkg = 'termina-tools-dlc'
532 tools_dlc_pkg_path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR,
533 'chromeos-base', tools_dlc_pkg)
Patrick Meiring5897add2020-09-16 16:30:17 +1000534
Maciek Swiech6b12f662022-01-25 16:51:19 +0000535 # termina-dlc and termina-tools-dlc are pinned to the same version.
536 version_pin_src_path = _get_version_pin_src_path(termina_dlc_pkg_path)
Patrick Meiring5897add2020-09-16 16:30:17 +1000537 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
538
Maciek Swiech6b12f662022-01-25 16:51:19 +0000539 result = uprev_lib.uprev_ebuild_from_pin(termina_dlc_pkg_path, version_no_rev,
540 chroot)
541 result += uprev_lib.uprev_ebuild_from_pin(tools_dlc_pkg_path, version_no_rev,
542 chroot)
Patrick Meiring5897add2020-09-16 16:30:17 +1000543
Maciek Swiech6b12f662022-01-25 16:51:19 +0000544 return result
Patrick Meiring5897add2020-09-16 16:30:17 +1000545
Julio Hurtadof1befec2021-05-05 21:34:26 +0000546@uprevs_versioned_package('chromeos-base/chromeos-lacros')
547def uprev_lacros(_build_targets, refs, chroot):
548 """Updates lacros ebuilds.
549
Julio Hurtadoa994e002021-07-07 17:57:45 +0000550 Version to uprev to is gathered from the QA qualified version tracking file
551 stored in chromium/src/chrome/LACROS_QA_QUALIFIED_VERSION. Uprev is triggered
552 on modification of this file across all chromium/src branches.
Julio Hurtadof1befec2021-05-05 21:34:26 +0000553
554 See: uprev_versioned_package.
555 """
Julio Hurtadoa994e002021-07-07 17:57:45 +0000556 result = uprev_lib.UprevVersionedPackageResult()
Julio Hurtadof1befec2021-05-05 21:34:26 +0000557 path = os.path.join(
Dennis Kempin257b74c2021-09-28 14:38:17 -0700558 constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos-base', 'chromeos-lacros')
Julio Hurtadoa994e002021-07-07 17:57:45 +0000559 lacros_version = refs[0].revision
560 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(path,
561 lacros_version,
562 chroot,
563 allow_downrev=False)
564
565 if not uprev_result:
566 return result
567
568 result.add_result(lacros_version, uprev_result.changed_files)
569 return result
Julio Hurtadof1befec2021-05-05 21:34:26 +0000570
571
Julio Hurtado870ed322021-12-03 18:22:40 +0000572@uprevs_versioned_package('chromeos-base/chromeos-lacros-parallel')
573def uprev_lacros_in_parallel(
574 _build_targets: Optional[List['build_target_lib.BuildTarget']],
575 refs: List[uprev_lib.GitRef],
576 chroot: 'chroot_lib.Chroot') -> 'uprev_lib.UprevVersionedPackageResult':
577 """Updates lacros ebuilds in parallel with ash-chrome.
578
579 This handler is going to be used temporarily while lacros transitions to being
580 uprevved atomically with ash-chrome. Unlike a standalone lacros uprev, this
581 handler will not need to look at the QA qualified file. Rather, it will
582 function identical to ash-chrome using git tags.
583
584 See: uprev_versioned_package.
585
586 Returns:
587 UprevVersionedPackageResult: The result.
588 """
589 result = uprev_lib.UprevVersionedPackageResult()
590 path = os.path.join(
591 constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos-base', 'chromeos-lacros')
592 lacros_version = uprev_lib.get_version_from_refs(refs)
593 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(path,
594 lacros_version,
595 chroot,
596 allow_downrev=False)
597
598 if not uprev_result:
599 return result
600
601 result.add_result(lacros_version, uprev_result.changed_files)
602 return result
603
604
Patrick Meiring5897add2020-09-16 16:30:17 +1000605@uprevs_versioned_package('app-emulation/parallels-desktop')
606def uprev_parallels_desktop(_build_targets, _refs, chroot):
607 """Updates Parallels Desktop ebuild - app-emulation/parallels-desktop.
608
609 See: uprev_versioned_package
610
611 Returns:
612 UprevVersionedPackageResult: The result.
613 """
614 package = 'parallels-desktop'
615 package_path = os.path.join(constants.CHROMEOS_PARTNER_OVERLAY_DIR,
616 'app-emulation', package)
617 version_pin_src_path = _get_version_pin_src_path(package_path)
618
619 # Expect a JSON blob like the following:
620 # {
621 # "version": "1.2.3",
622 # "test_image": { "url": "...", "size": 12345678,
623 # "sha256sum": "<32 bytes of hexadecimal>" }
624 # }
625 with open(version_pin_src_path, 'r') as f:
626 pinned = json.load(f)
627
628 if 'version' not in pinned or 'test_image' not in pinned:
629 raise UprevError('VERSION-PIN for %s missing version and/or '
630 'test_image field' % package)
631
632 version = pinned['version']
633 if not isinstance(version, str):
634 raise UprevError('version in VERSION-PIN for %s not a string' % package)
635
636 # Update the ebuild.
Chris McDonald38409112020-09-24 11:24:51 -0600637 result = uprev_lib.uprev_ebuild_from_pin(package_path, version, chroot)
Patrick Meiring5897add2020-09-16 16:30:17 +1000638
639 # Update the VM image used for testing.
Patrick Meiring1342def2020-10-30 17:09:13 +1100640 test_image_path = ('src/platform/tast-tests-private/src/chromiumos/tast/'
641 'local/bundles/crosint/pita/data/'
642 'pluginvm_image.zip.external')
Patrick Meiring5897add2020-09-16 16:30:17 +1000643 test_image_src_path = os.path.join(constants.SOURCE_ROOT, test_image_path)
644 with open(test_image_src_path, 'w') as f:
645 json.dump(pinned['test_image'], f, indent=2)
646 result.add_result(version, [test_image_src_path])
647
648 return result
Trent Beginaf51f1b2020-03-09 17:35:31 -0600649
650
Trent Begin315d9d92019-12-03 21:55:53 -0700651@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Beginaf51f1b2020-03-09 17:35:31 -0600652def uprev_sludge(_build_targets, _refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700653 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
654
655 See: uprev_versioned_package.
656 """
657 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700658 package_path = os.path.join('src', 'private-overlays',
659 'project-wilco-private', 'chromeos-base', package)
Patrick Meiring5897add2020-09-16 16:30:17 +1000660 version_pin_src_path = _get_version_pin_src_path(package_path)
661 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700662
Chris McDonald38409112020-09-24 11:24:51 -0600663 return uprev_lib.uprev_ebuild_from_pin(package_path, version_no_rev, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700664
665
David Riley8513c1f2021-10-14 17:07:41 -0700666@uprevs_versioned_package('chromeos-base/borealis-dlc')
667def uprev_borealis_dlc(_build_targets, _refs, chroot):
668 """Updates shared borealis-dlc ebuild - chromeos-base/borealis-dlc.
669
670 See: uprev_versioned_package.
671 """
David Rileybc100ba2022-01-10 10:06:59 -0800672 package_path = os.path.join('src', 'private-overlays',
673 'chromeos-partner-overlay', 'chromeos-base',
674 'borealis-dlc')
David Riley8513c1f2021-10-14 17:07:41 -0700675
676 version_pin_src_path = _get_version_pin_src_path(package_path)
677 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
678
679 return uprev_lib.uprev_ebuild_from_pin(package_path, version_no_rev, chroot)
680
681
Patrick Meiring5897add2020-09-16 16:30:17 +1000682def _get_version_pin_src_path(package_path):
683 """Returns the path to the VERSION-PIN file for the given package."""
684 return os.path.join(constants.SOURCE_ROOT, package_path, 'VERSION-PIN')
685
686
Alex Klein87531182019-08-12 15:23:37 -0600687@uprevs_versioned_package(constants.CHROME_CP)
Alex Klein4e839252022-01-06 13:29:18 -0700688def uprev_chrome_from_ref(build_targets, refs, _chroot):
Alex Klein87531182019-08-12 15:23:37 -0600689 """Uprev chrome and its related packages.
690
691 See: uprev_versioned_package.
692 """
693 # Determine the version from the refs (tags), i.e. the chrome versions are the
694 # tag names.
Julio Hurtado870ed322021-12-03 18:22:40 +0000695 chrome_version = uprev_lib.get_version_from_refs(refs)
Chris McDonald25881af2020-05-12 03:17:53 -0600696 logging.debug('Chrome version determined from refs: %s', chrome_version)
Alex Klein87531182019-08-12 15:23:37 -0600697
Alex Klein4e839252022-01-06 13:29:18 -0700698 return uprev_chrome(chrome_version, build_targets, None)
Alex Kleinf69bd802021-06-22 15:43:49 -0600699
700
Alex Klein9ce3f682021-06-23 15:06:44 -0600701def revbump_chrome(
Alex Klein16ea1b32021-10-01 15:48:50 -0600702 build_targets: List['build_target_lib.BuildTarget'] = None,
Alex Klein9ce3f682021-06-23 15:06:44 -0600703 chroot: Optional['chroot_lib.Chroot'] = None
704) -> uprev_lib.UprevVersionedPackageResult:
Alex Kleinf69bd802021-06-22 15:43:49 -0600705 """Attempt to revbump chrome.
706
707 Revbumps are done by executing an uprev using the current stable version.
708 E.g. if chrome is on 1.2.3.4 and has a 1.2.3.4_rc-r2.ebuild, performing an
709 uprev on version 1.2.3.4 when there are applicable changes (e.g. to the 9999
710 ebuild) will result in a revbump to 1.2.3.4_rc-r3.ebuild.
711 """
712 chrome_version = uprev_lib.get_stable_chrome_version()
Alex Klein16ea1b32021-10-01 15:48:50 -0600713 return uprev_chrome(chrome_version, build_targets, chroot)
Alex Kleinf69bd802021-06-22 15:43:49 -0600714
715
Alex Klein9ce3f682021-06-23 15:06:44 -0600716def uprev_chrome(
Alex Klein16ea1b32021-10-01 15:48:50 -0600717 chrome_version: str,
718 build_targets: Optional[List['build_target_lib.BuildTarget']],
719 chroot: Optional['chroot_lib.Chroot']
Alex Klein9ce3f682021-06-23 15:06:44 -0600720) -> uprev_lib.UprevVersionedPackageResult:
721 """Attempt to uprev chrome and its related packages to the given version."""
Alex Klein87531182019-08-12 15:23:37 -0600722 uprev_manager = uprev_lib.UprevChromeManager(
723 chrome_version, build_targets=build_targets, chroot=chroot)
Chris McDonald38409112020-09-24 11:24:51 -0600724 result = uprev_lib.UprevVersionedPackageResult()
Chris McDonald25881af2020-05-12 03:17:53 -0600725 # TODO(crbug.com/1080429): Handle all possible outcomes of a Chrome uprev
726 # attempt. The expected behavior is documented in the following table:
727 #
728 # Outcome of Chrome uprev attempt:
729 # NEWER_VERSION_EXISTS:
730 # Do nothing.
731 # SAME_VERSION_EXISTS or REVISION_BUMP:
732 # Uprev followers
733 # Assert not VERSION_BUMP (any other outcome is fine)
734 # VERSION_BUMP or NEW_EBUILD_CREATED:
735 # Uprev followers
736 # Assert that Chrome & followers are at same package version
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600737
738 # Start with chrome itself so we can proceed accordingly.
739 chrome_result = uprev_manager.uprev(constants.CHROME_CP)
740 if chrome_result.newer_version_exists:
741 # Cannot use the given version (newer version already exists).
David Burger37f48672019-09-18 17:07:56 -0600742 return result
Alex Klein87531182019-08-12 15:23:37 -0600743
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600744 # Also uprev related packages.
Alex Klein87531182019-08-12 15:23:37 -0600745 for package in constants.OTHER_CHROME_PACKAGES:
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600746 follower_result = uprev_manager.uprev(package)
747 if chrome_result.stable_version and follower_result.version_bump:
748 logging.warning('%s had a version bump, but no more than a revision bump '
749 'should have been possible.', package)
Alex Klein87531182019-08-12 15:23:37 -0600750
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600751 if uprev_manager.modified_ebuilds:
752 # Record changes when we have them.
753 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
754
755 return result
Alex Klein87531182019-08-12 15:23:37 -0600756
757
Harvey Yang3eee06c2021-03-18 15:47:56 +0800758def _get_latest_version_from_refs(refs_prefix: str,
759 refs: List[uprev_lib.GitRef]) -> str:
760 """Get the latest version from refs
Ben Reiche779cf42020-12-15 03:21:31 +0000761
Ben Reiche779cf42020-12-15 03:21:31 +0000762 Versions are compared using |distutils.version.LooseVersion| and
763 the latest version is returned.
764
765 Args:
Harvey Yang3eee06c2021-03-18 15:47:56 +0800766 refs_prefix: The refs prefix of the tag format.
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800767 refs: The tags to parse for the latest Perfetto version.
768
769 Returns:
770 The latest Perfetto version to use.
771 """
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800772 valid_refs = []
773 for gitiles in refs:
Harvey Yang3eee06c2021-03-18 15:47:56 +0800774 if gitiles.ref.startswith(refs_prefix):
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800775 valid_refs.append(gitiles.ref)
776
777 if not valid_refs:
778 return None
779
780 # Sort by version and take the latest version.
781 target_version_ref = sorted(valid_refs,
782 key=LooseVersion,
783 reverse=True)[0]
Harvey Yang3eee06c2021-03-18 15:47:56 +0800784 return target_version_ref.replace(refs_prefix, '')
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800785
786
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700787def _generate_platform_c_files(
788 replication_config: replication_config_pb2.ReplicationConfig,
789 chroot: 'chroot_lib.Chroot') -> List[str]:
Andrew Lamb9563a152019-12-04 11:42:18 -0700790 """Generates platform C files from a platform JSON payload.
791
792 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700793 replication_config: A ReplicationConfig that has already been run. If it
794 produced a build_config.json file, that file will be used to generate
795 platform C files. Otherwise, nothing will be generated.
796 chroot: The chroot to use to generate.
Andrew Lamb9563a152019-12-04 11:42:18 -0700797
798 Returns:
799 A list of generated files.
800 """
801 # Generate the platform C files from the build config. Note that it would be
802 # more intuitive to generate the platform C files from the platform config;
803 # however, cros_config_schema does not allow this, because the platform config
804 # payload is not always valid input. For example, if a property is both
805 # 'required' and 'build-only', it will fail schema validation. Thus, use the
806 # build config, and use '-f' to filter.
807 build_config_path = [
808 rule.destination_path
809 for rule in replication_config.file_replication_rules
810 if rule.destination_path.endswith('build_config.json')
811 ]
812
813 if not build_config_path:
814 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700815 'No build_config.json found, will not generate platform C files. '
816 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700817 return []
818
819 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700820 raise ValueError('Expected at most one build_config.json destination path. '
821 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700822
823 build_config_path = build_config_path[0]
824
825 # Paths to the build_config.json and dir to output C files to, in the
826 # chroot.
827 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
828 build_config_path)
829 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
830 os.path.dirname(build_config_path))
831
832 command = [
833 'cros_config_schema', '-m', build_config_chroot_path, '-g',
834 generated_output_chroot_dir, '-f', '"TRUE"'
835 ]
836
837 cros_build_lib.run(
838 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
839
840 # A relative (to the source root) path to the generated C files.
841 generated_output_dir = os.path.dirname(build_config_path)
842 generated_files = []
843 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
844 for f in expected_c_files:
845 if os.path.exists(
846 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
847 generated_files.append(os.path.join(generated_output_dir, f))
848
849 if len(expected_c_files) != len(generated_files):
850 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
851
852 return generated_files
853
854
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700855def _get_private_overlay_package_root(ref: uprev_lib.GitRef, package: str):
Andrew Lambe836f222019-12-09 12:27:38 -0700856 """Returns the absolute path to the root of a given private overlay.
857
858 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700859 ref: GitRef for the private overlay.
860 package: Path to the package in the overlay.
Andrew Lambe836f222019-12-09 12:27:38 -0700861 """
862 # There might be a cleaner way to map from package -> path within the source
863 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700864 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700865 match = re.match(private_overlay_ref_pattern, ref.path)
866 if not match:
867 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
868 (private_overlay_ref_pattern, ref))
869
870 overlay = match.group(1)
871
872 return os.path.join(constants.SOURCE_ROOT,
873 'src/private-overlays/overlay-%s-private' % overlay,
874 package)
875
876
Andrew Lambea9a8a22019-12-12 14:03:43 -0700877@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
878def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700879 """Replicate a private cros_config change to the corresponding public config.
880
Alex Kleinad6b48a2020-01-08 16:57:41 -0700881 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700882 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700883 package = 'chromeos-base/chromeos-config-bsp'
884
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700885 if len(refs) != 1:
886 raise ValueError('Expected exactly one ref, actual %s' % refs)
887
888 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700889 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700890 replication_config_path = os.path.join(package_root,
891 'replication_config.jsonpb')
892
893 try:
894 replication_config = json_format.Parse(
895 osutils.ReadFile(replication_config_path),
896 replication_config_pb2.ReplicationConfig())
897 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700898 raise ValueError(
899 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700900
901 replication_lib.Replicate(replication_config)
902
903 modified_files = [
904 rule.destination_path
905 for rule in replication_config.file_replication_rules
906 ]
907
Andrew Lamb9563a152019-12-04 11:42:18 -0700908 # The generated platform C files are not easily filtered by replication rules,
909 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
910 # files cannot. Therefore, replicate and filter the JSON payloads, and then
911 # generate filtered C files from the JSON payload.
912 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700913
914 # Use the private repo's commit hash as the new version.
915 new_private_version = refs[0].revision
916
Andrew Lamb988f4da2019-12-10 10:16:43 -0700917 # modified_files should contain only relative paths at this point, but the
918 # returned UprevVersionedPackageResult must contain only absolute paths.
919 for i, modified_file in enumerate(modified_files):
920 assert not os.path.isabs(modified_file)
921 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
922
Chris McDonald38409112020-09-24 11:24:51 -0600923 return uprev_lib.UprevVersionedPackageResult().add_result(
924 new_private_version, modified_files)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700925
926
Dennis Kempinef05f2b2021-09-08 16:36:49 -0700927@uprevs_versioned_package('chromeos-base/crosvm')
928def uprev_crosvm(_build_targets, refs, _chroot):
929 """Updates crosvm ebuilds to latest revision
930
931 crosvm is not versioned. We are updating to the latest commit on the main
932 branch.
933
934 See: uprev_versioned_package.
935
936 Returns:
937 UprevVersionedPackageResult: The result of updating crosvm ebuilds.
938 """
939 overlay = os.path.join(constants.SOURCE_ROOT,
940 constants.CHROMIUMOS_OVERLAY_DIR)
941 repo_path = os.path.join(constants.SOURCE_ROOT, 'src', 'crosvm')
942 manifest = git.ManifestCheckout.Cached(repo_path)
943
944 uprev_manager = uprev_lib.UprevOverlayManager([overlay], manifest)
Dennis Kempin257b74c2021-09-28 14:38:17 -0700945 uprev_manager.uprev(
946 package_list=[
947 'chromeos-base/crosvm',
948 'dev-rust/assertions',
949 'dev-rust/cros_async',
950 'dev-rust/cros_fuzz',
951 'dev-rust/data_model',
952 'dev-rust/enumn',
953 'dev-rust/io_uring',
954 'dev-rust/p9',
955 'dev-rust/sync',
956 'dev-rust/sys_util',
957 'dev-rust/tempfile',
958 'media-sound/audio_streams',
959 ],
960 force=True
961 )
Dennis Kempinef05f2b2021-09-08 16:36:49 -0700962
963 updated_files = uprev_manager.modified_ebuilds
964 result = uprev_lib.UprevVersionedPackageResult()
965 result.add_result(refs[0].revision, updated_files)
966 return result
967
968
Alex Klein5caab872021-09-10 11:44:37 -0600969def get_best_visible(
970 atom: str,
971 build_target: Optional['build_target_lib.BuildTarget'] = None
972) -> package_info.PackageInfo:
Alex Kleinbbef2b32019-08-27 10:38:50 -0600973 """Returns the best visible CPV for the given atom.
974
975 Args:
Alex Klein5caab872021-09-10 11:44:37 -0600976 atom: The atom to look up.
977 build_target: The build target whose sysroot should be searched, or the SDK
978 if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700979
980 Returns:
Alex Klein5caab872021-09-10 11:44:37 -0600981 The best visible package, or None if none are visible.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600982 """
David Burger1e0fe232019-07-01 14:52:07 -0600983 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600984
Alex Klein5caab872021-09-10 11:44:37 -0600985 return portage_util.PortageqBestVisible(
986 atom,
987 board=build_target.name if build_target else None,
988 sysroot=build_target.root if build_target else None)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600989
990
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700991def has_prebuilt(
992 atom: str,
993 build_target: 'build_target_lib.BuildTarget' = None,
994 useflags: Union[Iterable[str], str] = None) -> bool:
Alex Kleinda39c6d2019-09-16 14:36:36 -0600995 """Check if a prebuilt exists.
996
997 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700998 atom: The package whose prebuilt is being queried.
999 build_target: The build target whose sysroot should be searched, or the
1000 SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -07001001 useflags: Any additional USE flags that should be set. May be a string
1002 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001003
1004 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001005 True if there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -06001006 """
1007 assert atom
1008
1009 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -07001010 extra_env = None
1011 if useflags:
1012 new_flags = useflags
Mike Frysingercfecd6b2020-12-14 23:54:05 -05001013 if not isinstance(useflags, str):
Alex Klein149fd3b2019-12-16 16:01:05 -07001014 new_flags = ' '.join(useflags)
1015
1016 existing = os.environ.get('USE', '')
1017 final_flags = '%s %s' % (existing, new_flags)
1018 extra_env = {'USE': final_flags.strip()}
1019 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -06001020
1021
David Burger0f9dd4e2019-10-08 12:33:42 -06001022def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -06001023 """Check if |build_target| builds |atom| (has it in its depgraph)."""
1024 cros_build_lib.AssertInsideChroot()
1025
Alex Kleind8cd4c62020-09-14 13:37:47 -06001026 pkgs = tuple(packages) if packages else None
LaMont Jones4cbecba2020-05-12 11:54:27 -06001027 # TODO(crbug/1081828): Receive and use sysroot.
1028 graph, _sdk_graph = dependency.GetBuildDependency(
Alex Kleind8cd4c62020-09-14 13:37:47 -06001029 build_target.root, build_target.name, pkgs)
Alex Klein36b117f2019-09-30 15:13:46 -06001030 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001031
1032
Alex Klein6becabc2020-09-11 14:03:05 -06001033def needs_chrome_source(
1034 build_target: 'build_target_lib.BuildTarget',
1035 compile_source=False,
1036 packages: Optional[List[package_info.PackageInfo]] = None,
1037 useflags=None):
1038 """Check if the chrome source is needed.
1039
1040 The chrome source is needed if the build target builds chrome or any of its
1041 follower packages, and can't use a prebuilt for them either because it's not
1042 available, or because we can't use prebuilts because it must build from
1043 source.
1044 """
1045 cros_build_lib.AssertInsideChroot()
1046
1047 # Check if it builds chrome and/or a follower package.
Alex Kleina8052162021-02-03 14:28:22 -07001048 graph = depgraph.get_sysroot_dependency_graph(build_target.root, packages)
Alex Klein6becabc2020-09-11 14:03:05 -06001049 builds_chrome = constants.CHROME_CP in graph
1050 builds_follower = {
1051 pkg: pkg in graph for pkg in constants.OTHER_CHROME_PACKAGES
1052 }
1053
Alex Klein9ce3f682021-06-23 15:06:44 -06001054 local_uprev = builds_chrome and revbump_chrome([build_target])
1055
Alex Klein6becabc2020-09-11 14:03:05 -06001056 # When we are compiling source set False since we do not use prebuilts.
1057 # When not compiling from source, start with True, i.e. we have every prebuilt
1058 # we've checked for up to this point.
1059 has_chrome_prebuilt = not compile_source
1060 has_follower_prebuilts = not compile_source
1061 # Save packages that need prebuilts for reporting.
1062 pkgs_needing_prebuilts = []
1063 if compile_source:
1064 # Need everything.
1065 pkgs_needing_prebuilts.append(constants.CHROME_CP)
1066 pkgs_needing_prebuilts.extend(
1067 [pkg for pkg, builds_pkg in builds_follower.items() if builds_pkg])
1068 else:
1069 # Check chrome itself.
1070 if builds_chrome:
1071 has_chrome_prebuilt = has_prebuilt(
1072 constants.CHROME_CP, build_target=build_target, useflags=useflags)
1073 if not has_chrome_prebuilt:
1074 pkgs_needing_prebuilts.append(constants.CHROME_CP)
1075 # Check follower packages.
1076 for pkg, builds_pkg in builds_follower.items():
1077 if not builds_pkg:
1078 continue
1079 prebuilt = has_prebuilt(pkg, build_target=build_target, useflags=useflags)
1080 has_follower_prebuilts &= prebuilt
1081 if not prebuilt:
1082 pkgs_needing_prebuilts.append(pkg)
1083 # Postcondition: has_chrome_prebuilt and has_follower_prebuilts now correctly
1084 # reflect whether we actually have the corresponding prebuilts for the build.
1085
1086 needs_chrome = builds_chrome and not has_chrome_prebuilt
1087 needs_follower = any(builds_follower.values()) and not has_follower_prebuilts
1088
1089 return NeedsChromeSourceResult(
Sergey Frolov26ff97b2021-08-14 00:02:11 +00001090 needs_chrome_source=needs_chrome or needs_follower,
Alex Klein6becabc2020-09-11 14:03:05 -06001091 builds_chrome=builds_chrome,
1092 packages=[package_info.parse(p) for p in pkgs_needing_prebuilts],
1093 missing_chrome_prebuilt=not has_chrome_prebuilt,
Alex Klein9ce3f682021-06-23 15:06:44 -06001094 missing_follower_prebuilt=not has_follower_prebuilts,
1095 local_uprev=local_uprev,
1096 )
Alex Klein6becabc2020-09-11 14:03:05 -06001097
1098
Alex Klein68a28712021-11-08 11:08:30 -07001099class TargetVersions(NamedTuple):
1100 """Data class for the info that makes up the "target versions"."""
1101 android_version: str
1102 android_branch: str
1103 android_target: str
1104 chrome_version: str
1105 platform_version: str
1106 milestone_version: str
1107 full_version: str
1108
1109
1110def get_target_versions(
1111 build_target: 'build_target_lib.BuildTarget',
1112 packages: List[package_info.PackageInfo] = None
1113) -> TargetVersions:
1114 """Aggregate version info for a few key packages and the OS as a whole."""
1115 # Android version.
1116 android_version = determine_android_version(build_target.name)
1117 logging.info('Found android version: %s', android_version)
1118 # Android branch version.
1119 android_branch = determine_android_branch(build_target.name)
1120 logging.info('Found android branch version: %s', android_branch)
1121 # Android target version.
1122 android_target = determine_android_target(build_target.name)
1123 logging.info('Found android target version: %s', android_target)
1124
1125 # TODO(crbug/1019770): Investigate cases where builds_chrome is true but
1126 # chrome_version is None.
1127
1128 builds_chrome = builds(constants.CHROME_CP, build_target, packages=packages)
1129 chrome_version = None
1130 if builds_chrome:
1131 # Chrome version fetch.
1132 chrome_version = determine_chrome_version(build_target)
1133 logging.info('Found chrome version: %s', chrome_version)
1134
1135 # The ChromeOS version info.
1136 platform_version = determine_platform_version()
1137 milestone_version = determine_milestone_version()
1138 full_version = determine_full_version()
1139
1140 return TargetVersions(android_version, android_branch, android_target,
1141 chrome_version, platform_version, milestone_version,
1142 full_version)
1143
1144
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001145def determine_chrome_version(
1146 build_target: 'build_target_lib.BuildTarget') -> Optional[str]:
Michael Mortensenc2615b72019-10-15 08:12:24 -06001147 """Returns the current Chrome version for the board (or in buildroot).
1148
1149 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001150 build_target: The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001151
1152 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001153 The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -06001154 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -06001155 # TODO(crbug/1019770): Long term we should not need the try/catch here once
1156 # the builds function above only returns True for chrome when
1157 # determine_chrome_version will succeed.
1158 try:
Alex Klein5caab872021-09-10 11:44:37 -06001159 pkg_info = portage_util.PortageqBestVisible(
Alex Klein7a3a7dd2020-01-08 16:44:38 -07001160 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -06001161 except cros_build_lib.RunCommandError as e:
1162 # Return None because portage failed when trying to determine the chrome
1163 # version.
1164 logging.warning('Caught exception in determine_chrome_package: %s', e)
1165 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -06001166 # Something like 78.0.3877.4_rc -> 78.0.3877.4
Alex Klein5caab872021-09-10 11:44:37 -06001167 return pkg_info.version.partition('_')[0]
Michael Mortensenc2615b72019-10-15 08:12:24 -06001168
1169
Alex Klein68a28712021-11-08 11:08:30 -07001170@functools.lru_cache()
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001171def determine_android_package(board: str) -> Optional[str]:
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001172 """Returns the active Android container package in use by the board.
1173
1174 Args:
1175 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001176
1177 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001178 The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001179 """
Michael Mortensene0f4b542019-10-24 15:30:23 -06001180 try:
Mike Frysingerdfa223c2022-04-27 11:02:27 -04001181 packages = portage_util.GetPackageDependencies(
1182 'virtual/target-os', board=board)
Michael Mortensene0f4b542019-10-24 15:30:23 -06001183 except cros_build_lib.RunCommandError as e:
1184 # Return None because a command (likely portage) failed when trying to
1185 # determine the package.
1186 logging.warning('Caught exception in determine_android_package: %s', e)
1187 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001188
Alex Kleinad6b48a2020-01-08 16:57:41 -07001189 # We assume there is only one Android package in the depgraph.
1190 for package in packages:
Mike Frysingerfcca49e2021-03-17 01:09:20 -04001191 if (package.startswith('chromeos-base/android-container-') or
1192 package.startswith('chromeos-base/android-vm-')):
Alex Kleinad6b48a2020-01-08 16:57:41 -07001193 return package
1194 return None
1195
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001196
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001197def determine_android_version(board: str, package: str = None):
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001198 """Determine the current Android version in buildroot now and return it.
1199
1200 This uses the typical portage logic to determine which version of Android
1201 is active right now in the buildroot.
1202
1203 Args:
Mike Frysingerb53f1822021-03-05 00:44:50 -05001204 board: The board name this is specific to.
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001205 package: The Android package, if already computed.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001206
1207 Returns:
Mike Frysingerb53f1822021-03-05 00:44:50 -05001208 The Android build ID of the container for the board.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001209
1210 Raises:
1211 NoAndroidVersionError: if no unique Android version can be determined.
1212 """
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001213 if not package:
1214 package = determine_android_package(board)
Mike Frysingerb53f1822021-03-05 00:44:50 -05001215 if not package:
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001216 return None
Mike Frysingerb53f1822021-03-05 00:44:50 -05001217 cpv = package_info.SplitCPV(package)
1218 if not cpv:
1219 raise NoAndroidVersionError(
1220 'Android version could not be determined for %s' % board)
1221 return cpv.version_no_rev
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001222
Alex Klein7a3a7dd2020-01-08 16:44:38 -07001223
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001224def determine_android_branch(board, package=None):
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001225 """Returns the Android branch in use by the active container ebuild."""
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001226 if not package:
1227 package = determine_android_package(board)
1228 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -06001229 return None
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001230 ebuild_path = portage_util.FindEbuildForBoardPackage(package, board)
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001231 # We assume all targets pull from the same branch and that we always
Federico 'Morg' Pareschicd9165a2020-05-29 09:45:55 +09001232 # have at least one of the following targets.
Shao-Chuan Lee73bba612020-06-17 11:47:04 +09001233 targets = constants.ANDROID_ALL_BUILD_TARGETS
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001234 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
1235 for target in targets:
1236 if target in ebuild_content:
1237 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
1238 if branch is not None:
1239 return branch.group(1)
1240 raise NoAndroidBranchError(
1241 'Android branch could not be determined for %s (ebuild empty?)' % board)
1242
1243
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001244def determine_android_target(board, package=None):
Michael Mortensen14960d02019-10-18 07:53:59 -06001245 """Returns the Android target in use by the active container ebuild."""
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001246 if not package:
1247 package = determine_android_package(board)
1248 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -06001249 return None
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001250 if package.startswith('chromeos-base/android-vm-'):
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001251 return 'bertha'
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001252 elif package.startswith('chromeos-base/android-container-'):
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001253 return 'cheets'
1254
1255 raise NoAndroidTargetError(
1256 'Android Target cannot be determined for the package: %s' %
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001257 package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -06001258
1259
1260def determine_platform_version():
1261 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -06001262 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -06001263 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
1264 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -06001265
1266
1267def determine_milestone_version():
1268 """Returns the platform version from the source root."""
1269 # Milestone version is something like '79'.
1270 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
1271 return version.chrome_branch
1272
Alex Klein7a3a7dd2020-01-08 16:44:38 -07001273
Michael Mortensen009cb662019-10-21 11:38:43 -06001274def determine_full_version():
1275 """Returns the full version from the source root."""
1276 # Full version is something like 'R79-12575.0.0'.
1277 milestone_version = determine_milestone_version()
1278 platform_version = determine_platform_version()
1279 full_version = ('R%s-%s' % (milestone_version, platform_version))
1280 return full_version
Michael Mortensen71ef5682020-05-07 14:29:24 -06001281
1282
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001283def find_fingerprints(
1284 build_target: 'build_target_lib.BuildTarget') -> List[str]:
Michael Mortensende716a12020-05-15 11:27:00 -06001285 """Returns a list of fingerprints for this build.
1286
1287 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001288 build_target: The build target.
Michael Mortensende716a12020-05-15 11:27:00 -06001289
1290 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001291 List of fingerprint strings.
Michael Mortensende716a12020-05-15 11:27:00 -06001292 """
1293 cros_build_lib.AssertInsideChroot()
1294 fp_file = 'cheets-fingerprint.txt'
1295 fp_path = os.path.join(
1296 image_lib.GetLatestImageLink(build_target.name),
1297 fp_file)
1298 if not os.path.isfile(fp_path):
1299 logging.info('Fingerprint file not found: %s', fp_path)
Michael Mortensend81d81e2020-06-09 14:20:59 -06001300 return []
Michael Mortensende716a12020-05-15 11:27:00 -06001301 logging.info('Reading fingerprint file: %s', fp_path)
1302 fingerprints = osutils.ReadFile(fp_path).splitlines()
1303 return fingerprints
1304
1305
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001306def get_all_firmware_versions(build_target: 'build_target_lib.BuildTarget'):
Michael Mortensen59e30872020-05-18 14:12:49 -06001307 """Extract firmware version for all models present.
1308
1309 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001310 build_target: The build target.
Michael Mortensen59e30872020-05-18 14:12:49 -06001311
1312 Returns:
1313 A dict of FirmwareVersions namedtuple instances by model.
1314 Each element will be populated based on whether it was present in the
1315 command output.
1316 """
1317 cros_build_lib.AssertInsideChroot()
1318 result = {}
1319 # Note that example output for _get_firmware_version_cmd_result is available
1320 # in the packages_unittest.py for testing get_all_firmware_versions.
1321 cmd_result = _get_firmware_version_cmd_result(build_target)
1322
Benjamin Shai12c767e2022-01-12 15:17:44 +00001323 if cmd_result:
1324 # There is a blank line between the version info for each model.
1325 firmware_version_payloads = cmd_result.split('\n\n')
1326 for firmware_version_payload in firmware_version_payloads:
1327 if 'BIOS' in firmware_version_payload:
1328 firmware_version = _find_firmware_versions(firmware_version_payload)
1329 result[firmware_version.model] = firmware_version
Michael Mortensen59e30872020-05-18 14:12:49 -06001330 return result
1331
1332
Benjamin Shai0858cd32022-01-10 20:23:49 +00001333class FirmwareVersions(NamedTuple):
1334 """Tuple to hold firmware versions, with truthiness."""
1335 model: Optional[str]
1336 main: Optional[str]
1337 main_rw: Optional[str]
1338 ec: Optional[str]
1339 ec_rw: Optional[str]
1340
1341 def __bool__(self):
1342 return bool(
1343 self.model or self.main or self.main_rw or self.ec or self.ec_rw)
Michael Mortensen71ef5682020-05-07 14:29:24 -06001344
1345
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001346def get_firmware_versions(build_target: 'build_target_lib.BuildTarget'):
Michael Mortensen71ef5682020-05-07 14:29:24 -06001347 """Extract version information from the firmware updater, if one exists.
1348
1349 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001350 build_target: The build target.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001351
1352 Returns:
1353 A FirmwareVersions namedtuple instance.
1354 Each element will either be set to the string output by the firmware
1355 updater shellball, or None if there is no firmware updater.
1356 """
1357 cros_build_lib.AssertInsideChroot()
1358 cmd_result = _get_firmware_version_cmd_result(build_target)
1359 if cmd_result:
1360 return _find_firmware_versions(cmd_result)
1361 else:
1362 return FirmwareVersions(None, None, None, None, None)
1363
1364
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001365def _get_firmware_version_cmd_result(
Alex Klein89b58732021-12-23 10:16:03 -07001366 build_target: 'build_target_lib.BuildTarget') -> Optional[str]:
Michael Mortensen71ef5682020-05-07 14:29:24 -06001367 """Gets the raw result output of the firmware updater version command.
1368
1369 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001370 build_target: The build target.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001371
1372 Returns:
1373 Command execution result.
1374 """
1375 updater = os.path.join(build_target.root,
1376 'usr/sbin/chromeos-firmwareupdate')
1377 logging.info('Calling updater %s', updater)
1378 # Call the updater using the chroot-based path.
Alex Klein89b58732021-12-23 10:16:03 -07001379 try:
1380 return cros_build_lib.run([updater, '-V'],
1381 capture_output=True, log_output=True,
1382 encoding='utf-8').stdout
1383 except cros_build_lib.RunCommandError:
1384 # Updater probably doesn't exist (e.g. betty).
1385 return None
Michael Mortensen71ef5682020-05-07 14:29:24 -06001386
1387
1388def _find_firmware_versions(cmd_output):
1389 """Finds firmware version output via regex matches against the cmd_output.
1390
1391 Args:
1392 cmd_output: The raw output to search against.
1393
1394 Returns:
1395 FirmwareVersions namedtuple with results.
1396 Each element will either be set to the string output by the firmware
1397 updater shellball, or None if there is no match.
1398 """
1399
1400 # Sometimes a firmware bundle includes a special combination of RO+RW
1401 # firmware. In this case, the RW firmware version is indicated with a "(RW)
1402 # version" field. In other cases, the "(RW) version" field is not present.
1403 # Therefore, search for the "(RW)" fields first and if they aren't present,
1404 # fallback to the other format. e.g. just "BIOS version:".
1405 # TODO(mmortensen): Use JSON once the firmware updater supports it.
1406 main = None
1407 main_rw = None
1408 ec = None
1409 ec_rw = None
1410 model = None
1411
1412 match = re.search(r'BIOS version:\s*(?P<version>.*)', cmd_output)
1413 if match:
1414 main = match.group('version')
1415
1416 match = re.search(r'BIOS \(RW\) version:\s*(?P<version>.*)', cmd_output)
1417 if match:
1418 main_rw = match.group('version')
1419
1420 match = re.search(r'EC version:\s*(?P<version>.*)', cmd_output)
1421 if match:
1422 ec = match.group('version')
1423
1424 match = re.search(r'EC \(RW\) version:\s*(?P<version>.*)', cmd_output)
1425 if match:
1426 ec_rw = match.group('version')
1427
1428 match = re.search(r'Model:\s*(?P<model>.*)', cmd_output)
1429 if match:
1430 model = match.group('model')
1431
1432 return FirmwareVersions(model, main, main_rw, ec, ec_rw)
Michael Mortensena4af79e2020-05-06 16:18:48 -06001433
1434
Benjamin Shai0858cd32022-01-10 20:23:49 +00001435class MainEcFirmwareVersions(NamedTuple):
1436 """Tuple to hold main and ec firmware versions, with truthiness."""
1437 main_fw_version: Optional[str]
1438 ec_fw_version: Optional[str]
1439
1440 def __bool__(self):
1441 return bool(self.main_fw_version or self.ec_fw_version)
1442
Michael Mortensena4af79e2020-05-06 16:18:48 -06001443
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001444def determine_firmware_versions(build_target: 'build_target_lib.BuildTarget'):
Michael Mortensena4af79e2020-05-06 16:18:48 -06001445 """Returns a namedtuple with main and ec firmware versions.
1446
1447 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001448 build_target: The build target.
Michael Mortensena4af79e2020-05-06 16:18:48 -06001449
1450 Returns:
1451 MainEcFirmwareVersions namedtuple with results.
1452 """
1453 fw_versions = get_firmware_versions(build_target)
1454 main_fw_version = fw_versions.main_rw or fw_versions.main
1455 ec_fw_version = fw_versions.ec_rw or fw_versions.ec
1456
1457 return MainEcFirmwareVersions(main_fw_version, ec_fw_version)
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001458
Benjamin Shai0858cd32022-01-10 20:23:49 +00001459
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001460def determine_kernel_version(
Benjamin Shai8e14eb12022-04-28 22:55:16 +00001461 build_target: 'build_target_lib.BuildTarget') -> str:
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001462 """Returns a string containing the kernel version for this build target.
1463
1464 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001465 build_target: The build target.
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001466
1467 Returns:
Benjamin Shai8e14eb12022-04-28 22:55:16 +00001468 The kernel versions, or empty string.
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001469 """
1470 try:
Benjamin Shai8e14eb12022-04-28 22:55:16 +00001471 packages = portage_util.GetFlattenedDepsForPackage(
1472 'virtual/linux-sources', board=build_target.name, depth=1)
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001473 except cros_build_lib.RunCommandError as e:
1474 logging.warning('Unable to get package list for metadata: %s', e)
Benjamin Shai8e14eb12022-04-28 22:55:16 +00001475 return ''
1476 if not packages:
1477 raise KernelVersionError('No package found in FlattenedDepsForPackage')
1478 if len(packages) != 1:
1479 raise KernelVersionError(
1480 'Too many packages found in FlattenedDepsForPackage (%s)' %
1481 ''.join(packages)
1482 )
1483 package = packages[0]
1484 kernel_version = package_info.SplitCPV(package).version
1485 logging.info('Found active kernel version: %s', kernel_version)
1486 return kernel_version
Michael Mortensen125bb012020-05-21 14:02:10 -06001487
1488
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001489def get_models(
1490 build_target: 'build_target_lib.BuildTarget',
1491 log_output: bool = True) -> Optional[List[str]]:
Michael Mortensen125bb012020-05-21 14:02:10 -06001492 """Obtain a list of models supported by a unified board.
1493
1494 This ignored whitelabel models since GoldenEye has no specific support for
1495 these at present.
1496
1497 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001498 build_target: The build target.
Michael Mortensen125bb012020-05-21 14:02:10 -06001499 log_output: Whether to log the output of the cros_config_host invocation.
1500
1501 Returns:
1502 A list of models supported by this board, if it is a unified build; None,
1503 if it is not a unified build.
1504 """
1505 return _run_cros_config_host(build_target, ['list-models'],
1506 log_output=log_output)
1507
1508
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001509def get_key_id(
1510 build_target: 'build_target_lib.BuildTarget',
1511 model: str) -> Optional[str]:
Michael Mortensen359c1f32020-05-28 19:35:42 -06001512 """Obtain the key_id for a model within the build_target.
1513
1514 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001515 build_target: The build target.
1516 model: The model name
Michael Mortensen359c1f32020-05-28 19:35:42 -06001517
1518 Returns:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001519 A key_id or None.
Michael Mortensen359c1f32020-05-28 19:35:42 -06001520 """
1521 model_arg = '--model=' + model
1522 key_id_list = _run_cros_config_host(
1523 build_target,
1524 [model_arg, 'get', '/firmware-signing', 'key-id'])
1525 key_id = None
1526 if len(key_id_list) == 1:
1527 key_id = key_id_list[0]
1528 return key_id
1529
1530
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001531def _run_cros_config_host(
1532 build_target: 'build_target_lib.BuildTarget',
1533 args: List[str],
1534 log_output: bool = True) -> Optional[List[str]]:
Michael Mortensen125bb012020-05-21 14:02:10 -06001535 """Run the cros_config_host tool.
1536
1537 Args:
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001538 build_target: The build target.
Michael Mortensen125bb012020-05-21 14:02:10 -06001539 args: List of arguments to pass.
1540 log_output: Whether to log the output of the cros_config_host.
1541
1542 Returns:
1543 Output of the tool
1544 """
1545 cros_build_lib.AssertInsideChroot()
1546 tool = '/usr/bin/cros_config_host'
1547 if not os.path.isfile(tool):
1548 return None
1549
1550 config_fname = build_target.full_path(
1551 'usr/share/chromeos-config/yaml/config.yaml')
1552
1553 result = cros_build_lib.run(
1554 [tool, '-c', config_fname] + args,
1555 capture_output=True,
1556 encoding='utf-8',
1557 log_output=log_output,
1558 check=False)
1559 if result.returncode:
1560 # Show the output for debugging purposes.
1561 if 'No such file or directory' not in result.error:
1562 logging.error('cros_config_host failed: %s\n', result.error)
1563 return None
1564 return result.output.strip().splitlines()