blob: af9ced59f0057b53bd9331458b3ea597c80694e4 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2019 The ChromiumOS Authors
Alex Kleineb77ffa2019-05-28 14:47:44 -06002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Package utility functionality."""
6
Yaakov Shaul730814a2019-09-10 13:58:25 -06007import collections
Ben Reiche779cf42020-12-15 03:21:31 +00008from distutils.version import LooseVersion
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -06009import fileinput
Alex Klein87531182019-08-12 15:23:37 -060010import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060011import json
Chris McDonaldf7c03d42021-07-21 11:54:26 -060012import logging
Evan Hernandezb51f1522019-08-15 11:29:40 -060013import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060014import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060015import sys
Alex Klein68a28712021-11-08 11:08:30 -070016from typing import Iterable, List, NamedTuple, Optional, TYPE_CHECKING, Union
Alex Klein87531182019-08-12 15:23:37 -060017
Mike Frysinger2c024062021-05-22 15:43:22 -040018from chromite.third_party.google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060019
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from chromite.api.gen.config import replication_config_pb2
Ram Chandrasekar60f69f32022-06-03 22:49:30 +000021from chromite.lib import chromeos_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060022from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060023from chromite.lib import cros_build_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import git
Michael Mortensende716a12020-05-15 11:27:00 -060025from chromite.lib import image_lib
Michael Mortensenb70e8a82019-10-10 18:43:41 -060026from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070028from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060029from chromite.lib import uprev_lib
Alex Klein18a60af2020-06-11 12:08:47 -060030from chromite.lib.parser import package_info
Shao-Chuan Lee05e51142021-11-24 12:27:37 +090031from chromite.service import android
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Mike Frysinger68796b52019-08-25 00:04:27 -040033
Alex Klein5caab872021-09-10 11:44:37 -060034if TYPE_CHECKING:
Alex Klein1699fab2022-09-08 08:46:06 -060035 from chromite.lib import build_target_lib
36 from chromite.lib import chroot_lib
Chris McDonaldf7c03d42021-07-21 11:54:26 -060037
Alex Klein36b117f2019-09-30 15:13:46 -060038if cros_build_lib.IsInsideChroot():
Alex Klein1699fab2022-09-08 08:46:06 -060039 from chromite.lib import depgraph
40 from chromite.service import dependency
Alex Klein36b117f2019-09-30 15:13:46 -060041
Alex Klein87531182019-08-12 15:23:37 -060042# Registered handlers for uprevving versioned packages.
43_UPREV_FUNCS = {}
44
Alex Kleineb77ffa2019-05-28 14:47:44 -060045
46class Error(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060047 """Module's base error class."""
Alex Kleineb77ffa2019-05-28 14:47:44 -060048
49
Alex Klein4de25e82019-08-05 15:58:39 -060050class UnknownPackageError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060051 """Uprev attempted for a package without a registered handler."""
Alex Klein4de25e82019-08-05 15:58:39 -060052
53
Alex Kleineb77ffa2019-05-28 14:47:44 -060054class UprevError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060055 """An error occurred while uprevving packages."""
Alex Kleineb77ffa2019-05-28 14:47:44 -060056
57
Michael Mortensenb70e8a82019-10-10 18:43:41 -060058class NoAndroidVersionError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060059 """An error occurred while trying to determine the android version."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -060060
61
62class NoAndroidBranchError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060063 """An error occurred while trying to determine the android branch."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -060064
65
66class NoAndroidTargetError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060067 """An error occurred while trying to determine the android target."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -060068
69
Alex Klein4de25e82019-08-05 15:58:39 -060070class AndroidIsPinnedUprevError(UprevError):
Alex Klein1699fab2022-09-08 08:46:06 -060071 """Raised when we try to uprev while Android is pinned."""
Alex Klein4de25e82019-08-05 15:58:39 -060072
Alex Klein1699fab2022-09-08 08:46:06 -060073 def __init__(self, new_android_atom):
74 """Initialize a AndroidIsPinnedUprevError.
Alex Klein4de25e82019-08-05 15:58:39 -060075
Alex Klein1699fab2022-09-08 08:46:06 -060076 Args:
77 new_android_atom: The Android atom that we failed to
78 uprev to, due to Android being pinned.
79 """
80 assert new_android_atom
81 msg = (
82 "Failed up uprev to Android version %s as Android was pinned."
83 % new_android_atom
84 )
85 super().__init__(msg)
86 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060087
88
Andrew Lamb9563a152019-12-04 11:42:18 -070089class GeneratedCrosConfigFilesError(Error):
Alex Klein1699fab2022-09-08 08:46:06 -060090 """Error when cros_config_schema does not produce expected files"""
Andrew Lamb9563a152019-12-04 11:42:18 -070091
Alex Klein1699fab2022-09-08 08:46:06 -060092 def __init__(self, expected_files, found_files):
93 msg = "Expected to find generated C files: %s. Actually found: %s" % (
94 expected_files,
95 found_files,
96 )
97 super().__init__(msg)
Andrew Lamb9563a152019-12-04 11:42:18 -070098
Alex Klein7a3a7dd2020-01-08 16:44:38 -070099
Alex Klein1699fab2022-09-08 08:46:06 -0600100NeedsChromeSourceResult = collections.namedtuple(
101 "NeedsChromeSourceResult",
102 (
103 "needs_chrome_source",
104 "builds_chrome",
105 "packages",
106 "missing_chrome_prebuilt",
107 "missing_follower_prebuilt",
108 "local_uprev",
109 ),
110)
Alex Klein6becabc2020-09-11 14:03:05 -0600111
112
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600113def patch_ebuild_vars(ebuild_path, variables):
Alex Klein1699fab2022-09-08 08:46:06 -0600114 """Updates variables in ebuild.
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600115
Alex Klein1699fab2022-09-08 08:46:06 -0600116 Use this function rather than portage_util.EBuild.UpdateEBuild when you
117 want to preserve the variable position and quotes within the ebuild.
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600118
Alex Klein1699fab2022-09-08 08:46:06 -0600119 Args:
120 ebuild_path: The path of the ebuild.
121 variables: Dictionary of variables to update in ebuild.
122 """
123 try:
124 for line in fileinput.input(ebuild_path, inplace=1):
125 for var, value in variables.items():
126 line = re.sub(rf"\b{var}=\S+", f'{var}="{value}"', line)
127 sys.stdout.write(line)
128 finally:
129 fileinput.close()
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600130
131
Alex Klein87531182019-08-12 15:23:37 -0600132def uprevs_versioned_package(package):
Alex Klein1699fab2022-09-08 08:46:06 -0600133 """Decorator to register package uprev handlers."""
134 assert package
Alex Klein87531182019-08-12 15:23:37 -0600135
Alex Klein1699fab2022-09-08 08:46:06 -0600136 def register(func):
137 """Registers |func| as a handler for |package|."""
138 _UPREV_FUNCS[package] = func
Alex Klein87531182019-08-12 15:23:37 -0600139
Alex Klein1699fab2022-09-08 08:46:06 -0600140 @functools.wraps(func)
141 def pass_through(*args, **kwargs):
142 return func(*args, **kwargs)
Alex Klein87531182019-08-12 15:23:37 -0600143
Alex Klein1699fab2022-09-08 08:46:06 -0600144 return pass_through
Alex Klein87531182019-08-12 15:23:37 -0600145
Alex Klein1699fab2022-09-08 08:46:06 -0600146 return register
Alex Klein87531182019-08-12 15:23:37 -0600147
148
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900149class UprevAndroidResult(NamedTuple):
Alex Klein1699fab2022-09-08 08:46:06 -0600150 """Results of an Android uprev."""
151
152 revved: bool
153 android_atom: str = None
154 modified_files: List[str] = None
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900155
156
157def uprev_android(
158 android_package: str,
Alex Klein1699fab2022-09-08 08:46:06 -0600159 chroot: "chroot_lib.Chroot",
160 build_targets: Optional[List["build_target_lib.BuildTarget"]] = None,
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900161 android_build_branch: Optional[str] = None,
162 android_version: Optional[str] = None,
Alex Klein1699fab2022-09-08 08:46:06 -0600163 skip_commit: bool = False,
164) -> UprevAndroidResult:
165 """Performs an Android uprev by calling cros_mark_android_as_stable.
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900166
Alex Klein1699fab2022-09-08 08:46:06 -0600167 Args:
168 android_package: The Android package to uprev.
169 chroot: The chroot to enter.
170 build_targets: List of build targets to cleanup after uprev.
171 android_build_branch: Override the default Android branch corresponding to
172 the package.
173 android_version: Uprev to the particular version. By default the latest
174 available version is used.
175 skip_commit: Whether to skip committing the change after a successful uprev.
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900176
Alex Klein1699fab2022-09-08 08:46:06 -0600177 Returns:
178 The uprev result containing:
179 revved: Whether an uprev happened.
180 android_atom: If revved, the portage atom for the revved Android ebuild.
181 modified_files: If revved, list of files being modified.
182 """
183 command = [
184 "cros_mark_android_as_stable",
185 f"--android_package={android_package}",
186 ]
187 if build_targets:
188 command.append(f'--boards={":".join(bt.name for bt in build_targets)}')
189 if android_build_branch:
190 command.append(f"--android_build_branch={android_build_branch}")
191 if android_version:
192 command.append(f"--force_version={android_version}")
193 if skip_commit:
194 command.append("--skip_commit")
Alex Klein4de25e82019-08-05 15:58:39 -0600195
Alex Klein1699fab2022-09-08 08:46:06 -0600196 result = cros_build_lib.run(
197 command,
198 stdout=True,
199 enter_chroot=True,
200 encoding="utf-8",
201 chroot_args=chroot.get_enter_args(),
202 )
Alex Klein4de25e82019-08-05 15:58:39 -0600203
Alex Klein1699fab2022-09-08 08:46:06 -0600204 # cros_mark_android_as_stable prints the uprev result to stdout as JSON in a
205 # single line. We only take the last line from stdout to make sure no junk
206 # output is included (e.g. messages from bashrc scripts that run upon entering
207 # the chroot.)
208 output = json.loads(result.stdout.strip().splitlines()[-1])
Shao-Chuan Leedea458f2021-11-25 23:46:53 +0900209
Alex Klein1699fab2022-09-08 08:46:06 -0600210 if not output["revved"]:
211 logging.info("Found nothing to rev.")
212 return UprevAndroidResult(revved=False)
Shao-Chuan Lee84bf9a22021-11-19 17:42:11 +0900213
Alex Klein1699fab2022-09-08 08:46:06 -0600214 android_atom = output["android_atom"]
Alex Klein4de25e82019-08-05 15:58:39 -0600215
Alex Klein1699fab2022-09-08 08:46:06 -0600216 for target in build_targets or []:
217 # Sanity check: We should always be able to merge the version of
218 # Android we just unmasked.
219 command = [f"emerge-{target.name}", "-p", "--quiet", f"={android_atom}"]
220 try:
221 cros_build_lib.run(
222 command, enter_chroot=True, chroot_args=chroot.get_enter_args()
223 )
224 except cros_build_lib.RunCommandError:
225 logging.error(
226 "Cannot emerge-%s =%s\nIs Android pinned to an older "
227 "version?",
228 target,
229 android_atom,
230 )
231 raise AndroidIsPinnedUprevError(android_atom)
Alex Klein4de25e82019-08-05 15:58:39 -0600232
Alex Klein1699fab2022-09-08 08:46:06 -0600233 return UprevAndroidResult(
234 revved=True,
235 android_atom=android_atom,
236 modified_files=output["modified_files"],
237 )
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900238
239
Alex Klein1699fab2022-09-08 08:46:06 -0600240def uprev_android_lkgb(
241 android_package: str,
242 build_targets: List["build_target_lib.BuildTarget"],
243 chroot: "chroot_lib.Chroot",
244) -> uprev_lib.UprevVersionedPackageResult:
245 """Uprevs an Android package to the version specified in the LKGB file.
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900246
Alex Klein1699fab2022-09-08 08:46:06 -0600247 This is the PUpr handler for Android packages, triggered whenever the
248 corresponding LKGB file is being updated.
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900249
Alex Klein1699fab2022-09-08 08:46:06 -0600250 PUpr for Android does not test the uprev change in CQ; instead we run separate
251 jobs to test new Android versions, and we write the latest vetted version to
252 the LKGB file. Find the design at go/android-uprev-recipes.
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900253
Alex Klein1699fab2022-09-08 08:46:06 -0600254 Args:
255 android_package: The Android package to uprev.
256 build_targets: List of build targets to cleanup after uprev.
257 chroot: The chroot to enter.
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900258
Alex Klein1699fab2022-09-08 08:46:06 -0600259 Returns:
260 An uprev_lib.UprevVersionedPackageResult containing the new version and a
261 list of modified files.
262 """
263 android_package_dir = android.GetAndroidPackageDir(android_package)
264 android_version = android.ReadLKGB(android_package_dir)
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900265
Alex Klein1699fab2022-09-08 08:46:06 -0600266 result = uprev_lib.UprevVersionedPackageResult()
267 uprev_result = uprev_android(
268 android_package,
269 chroot,
270 build_targets=build_targets,
271 android_version=android_version,
272 skip_commit=True,
273 )
274 if not uprev_result.revved:
275 return result
276
277 # cros_mark_android_as_stable returns paths relative to |android.OVERLAY_DIR|.
278 result.add_result(
279 android_version,
280 [
281 os.path.join(android.OVERLAY_DIR, f)
282 for f in uprev_result.modified_files
283 ],
284 )
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900285 return result
286
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900287
288def define_uprev_android_lkgb_handlers():
Alex Klein1699fab2022-09-08 08:46:06 -0600289 """Dynamically define uprev handlers for each Android package"""
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900290
Alex Klein1699fab2022-09-08 08:46:06 -0600291 def define_handler(android_package):
292 """Defines the uprev handler for an Android package."""
293 full_package_name = "chromeos-base/" + android_package
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900294
Alex Klein1699fab2022-09-08 08:46:06 -0600295 @uprevs_versioned_package(full_package_name)
296 def _handler(build_targets, _refs, chroot):
297 return uprev_android_lkgb(android_package, build_targets, chroot)
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900298
Alex Klein1699fab2022-09-08 08:46:06 -0600299 for android_package in constants.ANDROID_ALL_PACKAGES:
300 define_handler(android_package)
Shao-Chuan Lee05e51142021-11-24 12:27:37 +0900301
302
303define_uprev_android_lkgb_handlers()
Alex Klein4de25e82019-08-05 15:58:39 -0600304
305
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700306def uprev_build_targets(
Alex Klein1699fab2022-09-08 08:46:06 -0600307 build_targets: Optional[List["build_target_lib.BuildTarget"]],
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700308 overlay_type: str,
Alex Klein1699fab2022-09-08 08:46:06 -0600309 chroot: "chroot_lib.Chroot" = None,
310 output_dir: Optional[str] = None,
311):
312 """Uprev the set provided build targets, or all if not specified.
Alex Kleineb77ffa2019-05-28 14:47:44 -0600313
Alex Klein1699fab2022-09-08 08:46:06 -0600314 Args:
315 build_targets: The build targets
316 whose overlays should be uprevved, empty or None for all.
317 overlay_type: One of the valid overlay types except None (see
318 constants.VALID_OVERLAYS).
319 chroot: The chroot to clean, if desired.
320 output_dir: The path to optionally dump result files.
321 """
322 # Need a valid overlay, but exclude None.
323 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
Alex Kleineb77ffa2019-05-28 14:47:44 -0600324
Alex Klein1699fab2022-09-08 08:46:06 -0600325 if build_targets:
326 overlays = portage_util.FindOverlaysForBoards(
327 overlay_type, boards=[t.name for t in build_targets]
328 )
329 else:
330 overlays = portage_util.FindOverlays(overlay_type)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600331
Alex Klein1699fab2022-09-08 08:46:06 -0600332 return uprev_overlays(
333 overlays,
334 build_targets=build_targets,
335 chroot=chroot,
336 output_dir=output_dir,
337 )
Alex Kleineb77ffa2019-05-28 14:47:44 -0600338
339
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700340def uprev_overlays(
341 overlays: List[str],
Alex Klein1699fab2022-09-08 08:46:06 -0600342 build_targets: Optional[List["build_target_lib.BuildTarget"]] = None,
343 chroot: Optional["chroot_lib.Chroot"] = None,
344 output_dir: Optional[str] = None,
345) -> List[str]:
346 """Uprev the given overlays.
Alex Kleineb77ffa2019-05-28 14:47:44 -0600347
Alex Klein1699fab2022-09-08 08:46:06 -0600348 Args:
349 overlays: The list of overlay paths.
350 build_targets: The build targets
351 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
352 chroot: The chroot to clean, if desired.
353 output_dir: The path to optionally dump result files.
Alex Kleineb77ffa2019-05-28 14:47:44 -0600354
Alex Klein1699fab2022-09-08 08:46:06 -0600355 Returns:
356 The paths to all of the modified ebuild files. This includes the new files
357 that were added (i.e. the new versions) and all of the removed files
358 (i.e. the old versions).
359 """
360 assert overlays
Alex Kleineb77ffa2019-05-28 14:47:44 -0600361
Alex Klein1699fab2022-09-08 08:46:06 -0600362 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600363
Alex Klein1699fab2022-09-08 08:46:06 -0600364 uprev_manager = uprev_lib.UprevOverlayManager(
365 overlays,
366 manifest,
367 build_targets=build_targets,
368 chroot=chroot,
369 output_dir=output_dir,
370 )
371 uprev_manager.uprev()
Alex Kleineb77ffa2019-05-28 14:47:44 -0600372
Alex Klein1699fab2022-09-08 08:46:06 -0600373 return uprev_manager.modified_ebuilds, uprev_manager.revved_packages
Alex Kleineb77ffa2019-05-28 14:47:44 -0600374
375
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700376def uprev_versioned_package(
377 package: package_info.CPV,
Alex Klein1699fab2022-09-08 08:46:06 -0600378 build_targets: List["build_target_lib.BuildTarget"],
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700379 refs: List[uprev_lib.GitRef],
Alex Klein1699fab2022-09-08 08:46:06 -0600380 chroot: "chroot_lib.Chroot",
381) -> "uprev_lib.UprevVersionedPackageResult":
382 """Call registered uprev handler function for the package.
Alex Klein87531182019-08-12 15:23:37 -0600383
Alex Klein1699fab2022-09-08 08:46:06 -0600384 Args:
385 package: The package being uprevved.
386 build_targets: The build targets to clean on a successful uprev.
387 refs:
388 chroot: The chroot to enter for cleaning.
Alex Klein87531182019-08-12 15:23:37 -0600389
Alex Klein1699fab2022-09-08 08:46:06 -0600390 Returns:
391 The result.
392 """
393 assert package
Alex Klein87531182019-08-12 15:23:37 -0600394
Alex Klein1699fab2022-09-08 08:46:06 -0600395 if package.cp not in _UPREV_FUNCS:
396 raise UnknownPackageError(
397 'Package "%s" does not have a registered handler.' % package.cp
398 )
Alex Klein87531182019-08-12 15:23:37 -0600399
Alex Klein1699fab2022-09-08 08:46:06 -0600400 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600401
402
Alex Klein1699fab2022-09-08 08:46:06 -0600403@uprevs_versioned_package("media-libs/virglrenderer")
Navil Perezf57ba872020-06-04 22:38:37 +0000404def uprev_virglrenderer(_build_targets, refs, _chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600405 """Updates virglrenderer ebuilds.
Navil Perezf57ba872020-06-04 22:38:37 +0000406
Alex Klein1699fab2022-09-08 08:46:06 -0600407 See: uprev_versioned_package.
Navil Perezf57ba872020-06-04 22:38:37 +0000408
Alex Klein1699fab2022-09-08 08:46:06 -0600409 Returns:
410 UprevVersionedPackageResult: The result of updating virglrenderer ebuilds.
411 """
412 overlay = os.path.join(
413 constants.SOURCE_ROOT, constants.CHROMIUMOS_OVERLAY_DIR
414 )
415 repo_path = os.path.join(
416 constants.SOURCE_ROOT, "src", "third_party", "virglrenderer"
417 )
418 manifest = git.ManifestCheckout.Cached(repo_path)
Navil Perezf57ba872020-06-04 22:38:37 +0000419
Alex Klein1699fab2022-09-08 08:46:06 -0600420 uprev_manager = uprev_lib.UprevOverlayManager([overlay], manifest)
421 # TODO(crbug.com/1066242): Ebuilds for virglrenderer are currently
422 # denylisted. Do not force uprevs after builder is stable and ebuilds are no
423 # longer denylisted.
424 uprev_manager.uprev(package_list=["media-libs/virglrenderer"], force=True)
Navil Perezf57ba872020-06-04 22:38:37 +0000425
Alex Klein1699fab2022-09-08 08:46:06 -0600426 updated_files = uprev_manager.modified_ebuilds
427 result = uprev_lib.UprevVersionedPackageResult()
428 result.add_result(refs[-1].revision, updated_files)
429 return result
Navil Perezf57ba872020-06-04 22:38:37 +0000430
Alex Klein1699fab2022-09-08 08:46:06 -0600431
432@uprevs_versioned_package("chromeos-base/drivefs")
Jose Magana03b5a842020-08-19 12:52:59 +1000433def uprev_drivefs(_build_targets, refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600434 """Updates drivefs ebuilds.
Jose Magana03b5a842020-08-19 12:52:59 +1000435
Alex Klein1699fab2022-09-08 08:46:06 -0600436 DriveFS versions follow the tag format of refs/tags/drivefs_1.2.3.
437 See: uprev_versioned_package.
Jose Magana03b5a842020-08-19 12:52:59 +1000438
Alex Klein1699fab2022-09-08 08:46:06 -0600439 Returns:
440 UprevVersionedPackageResult: The result of updating drivefs ebuilds.
441 """
Jose Magana03b5a842020-08-19 12:52:59 +1000442
Alex Klein1699fab2022-09-08 08:46:06 -0600443 DRIVEFS_PATH_PREFIX = "src/private-overlays/chromeos-overlay/chromeos-base"
444 result = uprev_lib.UprevVersionedPackageResult()
445 all_changed_files = []
Jose Magana03b5a842020-08-19 12:52:59 +1000446
Alex Klein1699fab2022-09-08 08:46:06 -0600447 DRIVEFS_REFS_PREFIX = "refs/tags/drivefs_"
448 drivefs_version = _get_latest_version_from_refs(DRIVEFS_REFS_PREFIX, refs)
449 if not drivefs_version:
450 # No valid DriveFS version is identified.
451 return result
452
453 logging.debug("DriveFS version determined from refs: %s", drivefs_version)
454
455 # Attempt to uprev drivefs package.
456 pkg_path = os.path.join(DRIVEFS_PATH_PREFIX, "drivefs")
457 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(
458 pkg_path, drivefs_version, chroot, allow_downrev=False
459 )
460
461 if not uprev_result:
462 return result
463 all_changed_files.extend(uprev_result.changed_files)
464 result.add_result(drivefs_version, all_changed_files)
465
Ben Reich4f3fa1b2020-12-19 08:21:26 +0000466 return result
Jose Magana03b5a842020-08-19 12:52:59 +1000467
Jose Magana03b5a842020-08-19 12:52:59 +1000468
Alex Klein1699fab2022-09-08 08:46:06 -0600469@uprevs_versioned_package("chromeos-base/perfetto")
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800470def uprev_perfetto(_build_targets, refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600471 """Updates Perfetto ebuilds.
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800472
Alex Klein1699fab2022-09-08 08:46:06 -0600473 Perfetto versions follow the tag format of refs/tags/v1.2.
474 See: uprev_versioned_package.
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800475
Alex Klein1699fab2022-09-08 08:46:06 -0600476 Returns:
477 UprevVersionedPackageResult: The result of updating Perfetto ebuilds.
478 """
479 result = uprev_lib.UprevVersionedPackageResult()
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800480
Alex Klein1699fab2022-09-08 08:46:06 -0600481 PERFETTO_REFS_PREFIX = "refs/tags/v"
482 perfetto_version = _get_latest_version_from_refs(PERFETTO_REFS_PREFIX, refs)
483 if not perfetto_version:
484 # No valid Perfetto version is identified.
485 return result
486
487 logging.debug("Perfetto version determined from refs: %s", perfetto_version)
488
489 # Attempt to uprev perfetto package.
490 PERFETTO_PATH = "src/third_party/chromiumos-overlay/chromeos-base/perfetto"
491
492 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(
493 PERFETTO_PATH,
494 perfetto_version,
495 chroot,
496 allow_downrev=False,
497 ref=PERFETTO_REFS_PREFIX + perfetto_version,
498 )
499
500 if not uprev_result:
501 return result
502
503 result.add_result(perfetto_version, uprev_result.changed_files)
504
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800505 return result
506
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800507
Alex Klein1699fab2022-09-08 08:46:06 -0600508@uprevs_versioned_package("afdo/kernel-profiles")
Yaakov Shaul395ae832019-09-09 14:45:32 -0600509def uprev_kernel_afdo(*_args, **_kwargs):
Alex Klein1699fab2022-09-08 08:46:06 -0600510 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600511
Alex Klein1699fab2022-09-08 08:46:06 -0600512 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600513
Alex Klein1699fab2022-09-08 08:46:06 -0600514 Raises:
515 EbuildManifestError: When ebuild manifest does not complete successfuly.
516 """
517 path = os.path.join(
518 constants.SOURCE_ROOT,
519 "src",
520 "third_party",
521 "toolchain-utils",
522 "afdo_metadata",
523 "kernel_afdo.json",
524 )
Yaakov Shaul395ae832019-09-09 14:45:32 -0600525
Alex Klein1699fab2022-09-08 08:46:06 -0600526 with open(path, "r") as f:
527 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600528
Alex Klein1699fab2022-09-08 08:46:06 -0600529 result = uprev_lib.UprevVersionedPackageResult()
530 for kernel_pkg, version_info in versions.items():
531 path = os.path.join(
532 constants.CHROMIUMOS_OVERLAY_DIR, "sys-kernel", kernel_pkg
533 )
534 ebuild_path = os.path.join(
535 constants.SOURCE_ROOT, path, "%s-9999.ebuild" % kernel_pkg
536 )
537 chroot_ebuild_path = os.path.join(
538 constants.CHROOT_SOURCE_ROOT, path, "%s-9999.ebuild" % kernel_pkg
539 )
540 afdo_profile_version = version_info["name"]
541 patch_ebuild_vars(
542 ebuild_path, dict(AFDO_PROFILE_VERSION=afdo_profile_version)
543 )
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600544
Alex Klein1699fab2022-09-08 08:46:06 -0600545 try:
546 cmd = ["ebuild", chroot_ebuild_path, "manifest", "--force"]
547 cros_build_lib.run(cmd, enter_chroot=True)
548 except cros_build_lib.RunCommandError as e:
549 raise uprev_lib.EbuildManifestError(
550 "Error encountered when regenerating the manifest for ebuild: %s\n%s"
551 % (chroot_ebuild_path, e),
552 e,
553 )
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600554
Alex Klein1699fab2022-09-08 08:46:06 -0600555 manifest_path = os.path.join(constants.SOURCE_ROOT, path, "Manifest")
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600556
Alex Klein1699fab2022-09-08 08:46:06 -0600557 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
Yaakov Shaul730814a2019-09-10 13:58:25 -0600558
Alex Klein1699fab2022-09-08 08:46:06 -0600559 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600560
561
Alex Klein1699fab2022-09-08 08:46:06 -0600562@uprevs_versioned_package("chromeos-base/termina-dlc")
563@uprevs_versioned_package("chromeos-base/termina-tools-dlc")
Maciek Swiech6b12f662022-01-25 16:51:19 +0000564def uprev_termina_dlcs(_build_targets, _refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600565 """Updates shared termina-dlc and termina-tools-dlc ebuilds.
Maciek Swiech6b12f662022-01-25 16:51:19 +0000566
Alex Klein1699fab2022-09-08 08:46:06 -0600567 termina-dlc - chromeos-base/termina-dlc
568 termina-tools-dlc - chromeos-base/termina-tools-dlc
Trent Beginaf51f1b2020-03-09 17:35:31 -0600569
Alex Klein1699fab2022-09-08 08:46:06 -0600570 See: uprev_versioned_package.
571 """
572 termina_dlc_pkg = "termina-dlc"
573 termina_dlc_pkg_path = os.path.join(
574 constants.CHROMIUMOS_OVERLAY_DIR, "chromeos-base", termina_dlc_pkg
575 )
576 tools_dlc_pkg = "termina-tools-dlc"
577 tools_dlc_pkg_path = os.path.join(
578 constants.CHROMIUMOS_OVERLAY_DIR, "chromeos-base", tools_dlc_pkg
579 )
Patrick Meiring5897add2020-09-16 16:30:17 +1000580
Alex Klein1699fab2022-09-08 08:46:06 -0600581 # termina-dlc and termina-tools-dlc are pinned to the same version.
582 version_pin_src_path = _get_version_pin_src_path(termina_dlc_pkg_path)
583 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
Patrick Meiring5897add2020-09-16 16:30:17 +1000584
Alex Klein1699fab2022-09-08 08:46:06 -0600585 result = uprev_lib.uprev_ebuild_from_pin(
586 termina_dlc_pkg_path, version_no_rev, chroot
587 )
588 result += uprev_lib.uprev_ebuild_from_pin(
589 tools_dlc_pkg_path, version_no_rev, chroot
590 )
Patrick Meiring5897add2020-09-16 16:30:17 +1000591
Alex Klein1699fab2022-09-08 08:46:06 -0600592 return result
Patrick Meiring5897add2020-09-16 16:30:17 +1000593
Alex Klein1699fab2022-09-08 08:46:06 -0600594
595@uprevs_versioned_package("chromeos-base/chromeos-lacros")
Julio Hurtadof1befec2021-05-05 21:34:26 +0000596def uprev_lacros(_build_targets, refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600597 """Updates lacros ebuilds.
Julio Hurtadof1befec2021-05-05 21:34:26 +0000598
Alex Klein1699fab2022-09-08 08:46:06 -0600599 Version to uprev to is gathered from the QA qualified version tracking file
600 stored in chromium/src/chrome/LACROS_QA_QUALIFIED_VERSION. Uprev is triggered
601 on modification of this file across all chromium/src branches.
Julio Hurtadof1befec2021-05-05 21:34:26 +0000602
Alex Klein1699fab2022-09-08 08:46:06 -0600603 See: uprev_versioned_package.
604 """
605 result = uprev_lib.UprevVersionedPackageResult()
606 path = os.path.join(
607 constants.CHROMIUMOS_OVERLAY_DIR, "chromeos-base", "chromeos-lacros"
608 )
609 lacros_version = refs[0].revision
610 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(
611 path, lacros_version, chroot, allow_downrev=False
612 )
Julio Hurtadoa994e002021-07-07 17:57:45 +0000613
Alex Klein1699fab2022-09-08 08:46:06 -0600614 if not uprev_result:
615 return result
616
617 result.add_result(lacros_version, uprev_result.changed_files)
Julio Hurtadoa994e002021-07-07 17:57:45 +0000618 return result
619
Julio Hurtadof1befec2021-05-05 21:34:26 +0000620
Alex Klein1699fab2022-09-08 08:46:06 -0600621@uprevs_versioned_package("chromeos-base/chromeos-lacros-parallel")
Julio Hurtado870ed322021-12-03 18:22:40 +0000622def uprev_lacros_in_parallel(
Alex Klein1699fab2022-09-08 08:46:06 -0600623 _build_targets: Optional[List["build_target_lib.BuildTarget"]],
Julio Hurtado870ed322021-12-03 18:22:40 +0000624 refs: List[uprev_lib.GitRef],
Alex Klein1699fab2022-09-08 08:46:06 -0600625 chroot: "chroot_lib.Chroot",
626) -> "uprev_lib.UprevVersionedPackageResult":
627 """Updates lacros ebuilds in parallel with ash-chrome.
Julio Hurtado870ed322021-12-03 18:22:40 +0000628
Alex Klein1699fab2022-09-08 08:46:06 -0600629 This handler is going to be used temporarily while lacros transitions to being
630 uprevved atomically with ash-chrome. Unlike a standalone lacros uprev, this
631 handler will not need to look at the QA qualified file. Rather, it will
632 function identical to ash-chrome using git tags.
Julio Hurtado870ed322021-12-03 18:22:40 +0000633
Alex Klein1699fab2022-09-08 08:46:06 -0600634 See: uprev_versioned_package.
Julio Hurtado870ed322021-12-03 18:22:40 +0000635
Alex Klein1699fab2022-09-08 08:46:06 -0600636 Returns:
637 UprevVersionedPackageResult: The result.
638 """
639 result = uprev_lib.UprevVersionedPackageResult()
640 path = os.path.join(
641 constants.CHROMIUMOS_OVERLAY_DIR, "chromeos-base", "chromeos-lacros"
642 )
643 lacros_version = uprev_lib.get_version_from_refs(refs)
644 uprev_result = uprev_lib.uprev_workon_ebuild_to_version(
645 path, lacros_version, chroot, allow_downrev=False
646 )
Julio Hurtado870ed322021-12-03 18:22:40 +0000647
Alex Klein1699fab2022-09-08 08:46:06 -0600648 if not uprev_result:
649 return result
650
651 result.add_result(lacros_version, uprev_result.changed_files)
Julio Hurtado870ed322021-12-03 18:22:40 +0000652 return result
653
Julio Hurtado870ed322021-12-03 18:22:40 +0000654
Alex Klein1699fab2022-09-08 08:46:06 -0600655@uprevs_versioned_package("app-emulation/parallels-desktop")
Patrick Meiring5897add2020-09-16 16:30:17 +1000656def uprev_parallels_desktop(_build_targets, _refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600657 """Updates Parallels Desktop ebuild - app-emulation/parallels-desktop.
Patrick Meiring5897add2020-09-16 16:30:17 +1000658
Alex Klein1699fab2022-09-08 08:46:06 -0600659 See: uprev_versioned_package
Patrick Meiring5897add2020-09-16 16:30:17 +1000660
Alex Klein1699fab2022-09-08 08:46:06 -0600661 Returns:
662 UprevVersionedPackageResult: The result.
663 """
664 package = "parallels-desktop"
665 package_path = os.path.join(
666 constants.CHROMEOS_PARTNER_OVERLAY_DIR, "app-emulation", package
667 )
668 version_pin_src_path = _get_version_pin_src_path(package_path)
Patrick Meiring5897add2020-09-16 16:30:17 +1000669
Alex Klein1699fab2022-09-08 08:46:06 -0600670 # Expect a JSON blob like the following:
671 # {
672 # "version": "1.2.3",
673 # "test_image": { "url": "...", "size": 12345678,
674 # "sha256sum": "<32 bytes of hexadecimal>" }
675 # }
676 with open(version_pin_src_path, "r") as f:
677 pinned = json.load(f)
Patrick Meiring5897add2020-09-16 16:30:17 +1000678
Alex Klein1699fab2022-09-08 08:46:06 -0600679 if "version" not in pinned or "test_image" not in pinned:
680 raise UprevError(
681 "VERSION-PIN for %s missing version and/or "
682 "test_image field" % package
683 )
Patrick Meiring5897add2020-09-16 16:30:17 +1000684
Alex Klein1699fab2022-09-08 08:46:06 -0600685 version = pinned["version"]
686 if not isinstance(version, str):
687 raise UprevError("version in VERSION-PIN for %s not a string" % package)
Patrick Meiring5897add2020-09-16 16:30:17 +1000688
Alex Klein1699fab2022-09-08 08:46:06 -0600689 # Update the ebuild.
690 result = uprev_lib.uprev_ebuild_from_pin(package_path, version, chroot)
Patrick Meiring5897add2020-09-16 16:30:17 +1000691
Alex Klein1699fab2022-09-08 08:46:06 -0600692 # Update the VM image used for testing.
693 test_image_path = (
694 "src/platform/tast-tests-private/src/chromiumos/tast/"
695 "local/bundles/crosint/pita/data/"
696 "pluginvm_image.zip.external"
697 )
698 test_image_src_path = os.path.join(constants.SOURCE_ROOT, test_image_path)
699 with open(test_image_src_path, "w") as f:
700 json.dump(pinned["test_image"], f, indent=2)
701 result.add_result(version, [test_image_src_path])
Patrick Meiring5897add2020-09-16 16:30:17 +1000702
Alex Klein1699fab2022-09-08 08:46:06 -0600703 return result
Trent Beginaf51f1b2020-03-09 17:35:31 -0600704
705
Alex Klein1699fab2022-09-08 08:46:06 -0600706@uprevs_versioned_package("chromeos-base/chromeos-dtc-vm")
Trent Beginaf51f1b2020-03-09 17:35:31 -0600707def uprev_sludge(_build_targets, _refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600708 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
Trent Begin315d9d92019-12-03 21:55:53 -0700709
Alex Klein1699fab2022-09-08 08:46:06 -0600710 See: uprev_versioned_package.
711 """
712 package = "chromeos-dtc-vm"
713 package_path = os.path.join(
714 "src",
715 "private-overlays",
716 "project-wilco-private",
717 "chromeos-base",
718 package,
719 )
720 version_pin_src_path = _get_version_pin_src_path(package_path)
721 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700722
Alex Klein1699fab2022-09-08 08:46:06 -0600723 return uprev_lib.uprev_ebuild_from_pin(package_path, version_no_rev, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700724
725
Alex Klein1699fab2022-09-08 08:46:06 -0600726@uprevs_versioned_package("chromeos-base/borealis-dlc")
David Riley8513c1f2021-10-14 17:07:41 -0700727def uprev_borealis_dlc(_build_targets, _refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600728 """Updates shared borealis-dlc ebuild - chromeos-base/borealis-dlc.
David Riley8513c1f2021-10-14 17:07:41 -0700729
Alex Klein1699fab2022-09-08 08:46:06 -0600730 See: uprev_versioned_package.
731 """
732 package_path = os.path.join(
733 "src",
734 "private-overlays",
735 "chromeos-partner-overlay",
736 "chromeos-base",
737 "borealis-dlc",
738 )
David Riley8513c1f2021-10-14 17:07:41 -0700739
Alex Klein1699fab2022-09-08 08:46:06 -0600740 version_pin_src_path = _get_version_pin_src_path(package_path)
741 version_no_rev = osutils.ReadFile(version_pin_src_path).strip()
David Riley8513c1f2021-10-14 17:07:41 -0700742
Alex Klein1699fab2022-09-08 08:46:06 -0600743 return uprev_lib.uprev_ebuild_from_pin(package_path, version_no_rev, chroot)
David Riley8513c1f2021-10-14 17:07:41 -0700744
745
Patrick Meiring5897add2020-09-16 16:30:17 +1000746def _get_version_pin_src_path(package_path):
Alex Klein1699fab2022-09-08 08:46:06 -0600747 """Returns the path to the VERSION-PIN file for the given package."""
748 return os.path.join(constants.SOURCE_ROOT, package_path, "VERSION-PIN")
Patrick Meiring5897add2020-09-16 16:30:17 +1000749
750
Alex Klein87531182019-08-12 15:23:37 -0600751@uprevs_versioned_package(constants.CHROME_CP)
Alex Klein4e839252022-01-06 13:29:18 -0700752def uprev_chrome_from_ref(build_targets, refs, _chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600753 """Uprev chrome and its related packages.
Alex Klein87531182019-08-12 15:23:37 -0600754
Alex Klein1699fab2022-09-08 08:46:06 -0600755 See: uprev_versioned_package.
756 """
757 # Determine the version from the refs (tags), i.e. the chrome versions are the
758 # tag names.
759 chrome_version = uprev_lib.get_version_from_refs(refs)
760 logging.debug("Chrome version determined from refs: %s", chrome_version)
Alex Klein87531182019-08-12 15:23:37 -0600761
Alex Klein1699fab2022-09-08 08:46:06 -0600762 return uprev_chrome(chrome_version, build_targets, None)
Alex Kleinf69bd802021-06-22 15:43:49 -0600763
764
Alex Klein9ce3f682021-06-23 15:06:44 -0600765def revbump_chrome(
Alex Klein1699fab2022-09-08 08:46:06 -0600766 build_targets: List["build_target_lib.BuildTarget"] = None,
767 chroot: Optional["chroot_lib.Chroot"] = None,
Alex Klein9ce3f682021-06-23 15:06:44 -0600768) -> uprev_lib.UprevVersionedPackageResult:
Alex Klein1699fab2022-09-08 08:46:06 -0600769 """Attempt to revbump chrome.
Alex Kleinf69bd802021-06-22 15:43:49 -0600770
Alex Klein1699fab2022-09-08 08:46:06 -0600771 Revbumps are done by executing an uprev using the current stable version.
772 E.g. if chrome is on 1.2.3.4 and has a 1.2.3.4_rc-r2.ebuild, performing an
773 uprev on version 1.2.3.4 when there are applicable changes (e.g. to the 9999
774 ebuild) will result in a revbump to 1.2.3.4_rc-r3.ebuild.
775 """
776 chrome_version = uprev_lib.get_stable_chrome_version()
777 return uprev_chrome(chrome_version, build_targets, chroot)
Alex Kleinf69bd802021-06-22 15:43:49 -0600778
779
Alex Klein9ce3f682021-06-23 15:06:44 -0600780def uprev_chrome(
Alex Klein16ea1b32021-10-01 15:48:50 -0600781 chrome_version: str,
Alex Klein1699fab2022-09-08 08:46:06 -0600782 build_targets: Optional[List["build_target_lib.BuildTarget"]],
783 chroot: Optional["chroot_lib.Chroot"],
Alex Klein9ce3f682021-06-23 15:06:44 -0600784) -> uprev_lib.UprevVersionedPackageResult:
Alex Klein1699fab2022-09-08 08:46:06 -0600785 """Attempt to uprev chrome and its related packages to the given version."""
786 uprev_manager = uprev_lib.UprevChromeManager(
787 chrome_version, build_targets=build_targets, chroot=chroot
788 )
789 result = uprev_lib.UprevVersionedPackageResult()
790 # TODO(crbug.com/1080429): Handle all possible outcomes of a Chrome uprev
791 # attempt. The expected behavior is documented in the following table:
792 #
793 # Outcome of Chrome uprev attempt:
794 # NEWER_VERSION_EXISTS:
795 # Do nothing.
796 # SAME_VERSION_EXISTS or REVISION_BUMP:
797 # Uprev followers
798 # Assert not VERSION_BUMP (any other outcome is fine)
799 # VERSION_BUMP or NEW_EBUILD_CREATED:
800 # Uprev followers
801 # Assert that Chrome & followers are at same package version
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600802
Alex Klein1699fab2022-09-08 08:46:06 -0600803 # Start with chrome itself so we can proceed accordingly.
804 chrome_result = uprev_manager.uprev(constants.CHROME_CP)
805 if chrome_result.newer_version_exists:
806 # Cannot use the given version (newer version already exists).
807 return result
808
809 # Also uprev related packages.
810 for package in constants.OTHER_CHROME_PACKAGES:
811 follower_result = uprev_manager.uprev(package)
812 if chrome_result.stable_version and follower_result.version_bump:
813 logging.warning(
814 "%s had a version bump, but no more than a revision bump "
815 "should have been possible.",
816 package,
817 )
818
819 if uprev_manager.modified_ebuilds:
820 # Record changes when we have them.
821 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
822
David Burger37f48672019-09-18 17:07:56 -0600823 return result
Alex Klein87531182019-08-12 15:23:37 -0600824
Alex Klein87531182019-08-12 15:23:37 -0600825
Alex Klein1699fab2022-09-08 08:46:06 -0600826def _get_latest_version_from_refs(
827 refs_prefix: str, refs: List[uprev_lib.GitRef]
828) -> str:
829 """Get the latest version from refs
Alex Klein0b2ec2d2021-06-23 15:56:45 -0600830
Alex Klein1699fab2022-09-08 08:46:06 -0600831 Versions are compared using |distutils.version.LooseVersion| and
832 the latest version is returned.
Alex Klein87531182019-08-12 15:23:37 -0600833
Alex Klein1699fab2022-09-08 08:46:06 -0600834 Args:
835 refs_prefix: The refs prefix of the tag format.
836 refs: The tags to parse for the latest Perfetto version.
Alex Klein87531182019-08-12 15:23:37 -0600837
Alex Klein1699fab2022-09-08 08:46:06 -0600838 Returns:
839 The latest Perfetto version to use.
840 """
841 valid_refs = []
842 for gitiles in refs:
843 if gitiles.ref.startswith(refs_prefix):
844 valid_refs.append(gitiles.ref)
Ben Reiche779cf42020-12-15 03:21:31 +0000845
Alex Klein1699fab2022-09-08 08:46:06 -0600846 if not valid_refs:
847 return None
Ben Reiche779cf42020-12-15 03:21:31 +0000848
Alex Klein1699fab2022-09-08 08:46:06 -0600849 # Sort by version and take the latest version.
850 target_version_ref = sorted(valid_refs, key=LooseVersion, reverse=True)[0]
851 return target_version_ref.replace(refs_prefix, "")
Harvey Yang9c61e9c2021-03-02 16:32:43 +0800852
853
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700854def _generate_platform_c_files(
855 replication_config: replication_config_pb2.ReplicationConfig,
Alex Klein1699fab2022-09-08 08:46:06 -0600856 chroot: "chroot_lib.Chroot",
857) -> List[str]:
858 """Generates platform C files from a platform JSON payload.
Andrew Lamb9563a152019-12-04 11:42:18 -0700859
Alex Klein1699fab2022-09-08 08:46:06 -0600860 Args:
861 replication_config: A ReplicationConfig that has already been run. If it
862 produced a build_config.json file, that file will be used to generate
863 platform C files. Otherwise, nothing will be generated.
864 chroot: The chroot to use to generate.
Andrew Lamb9563a152019-12-04 11:42:18 -0700865
Alex Klein1699fab2022-09-08 08:46:06 -0600866 Returns:
867 A list of generated files.
868 """
869 # Generate the platform C files from the build config. Note that it would be
870 # more intuitive to generate the platform C files from the platform config;
871 # however, cros_config_schema does not allow this, because the platform config
872 # payload is not always valid input. For example, if a property is both
873 # 'required' and 'build-only', it will fail schema validation. Thus, use the
874 # build config, and use '-f' to filter.
875 build_config_path = [
876 rule.destination_path
877 for rule in replication_config.file_replication_rules
878 if rule.destination_path.endswith("build_config.json")
879 ]
Andrew Lamb9563a152019-12-04 11:42:18 -0700880
Alex Klein1699fab2022-09-08 08:46:06 -0600881 if not build_config_path:
882 logging.info(
883 "No build_config.json found, will not generate platform C files. "
884 "Replication config: %s",
885 replication_config,
886 )
887 return []
Andrew Lamb9563a152019-12-04 11:42:18 -0700888
Alex Klein1699fab2022-09-08 08:46:06 -0600889 if len(build_config_path) > 1:
890 raise ValueError(
891 "Expected at most one build_config.json destination path. "
892 "Replication config: %s" % replication_config
893 )
Andrew Lamb9563a152019-12-04 11:42:18 -0700894
Alex Klein1699fab2022-09-08 08:46:06 -0600895 build_config_path = build_config_path[0]
Andrew Lamb9563a152019-12-04 11:42:18 -0700896
Alex Klein1699fab2022-09-08 08:46:06 -0600897 # Paths to the build_config.json and dir to output C files to, in the
898 # chroot.
899 build_config_chroot_path = os.path.join(
900 constants.CHROOT_SOURCE_ROOT, build_config_path
901 )
902 generated_output_chroot_dir = os.path.join(
903 constants.CHROOT_SOURCE_ROOT, os.path.dirname(build_config_path)
904 )
Andrew Lamb9563a152019-12-04 11:42:18 -0700905
Alex Klein1699fab2022-09-08 08:46:06 -0600906 command = [
907 "cros_config_schema",
908 "-m",
909 build_config_chroot_path,
910 "-g",
911 generated_output_chroot_dir,
912 "-f",
913 '"TRUE"',
914 ]
Andrew Lamb9563a152019-12-04 11:42:18 -0700915
Alex Klein1699fab2022-09-08 08:46:06 -0600916 cros_build_lib.run(
917 command, enter_chroot=True, chroot_args=chroot.get_enter_args()
918 )
Andrew Lamb9563a152019-12-04 11:42:18 -0700919
Alex Klein1699fab2022-09-08 08:46:06 -0600920 # A relative (to the source root) path to the generated C files.
921 generated_output_dir = os.path.dirname(build_config_path)
922 generated_files = []
923 expected_c_files = ["config.c", "ec_config.c", "ec_config.h"]
924 for f in expected_c_files:
925 if os.path.exists(
926 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)
927 ):
928 generated_files.append(os.path.join(generated_output_dir, f))
Andrew Lamb9563a152019-12-04 11:42:18 -0700929
Alex Klein1699fab2022-09-08 08:46:06 -0600930 if len(expected_c_files) != len(generated_files):
931 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
Andrew Lamb9563a152019-12-04 11:42:18 -0700932
Alex Klein1699fab2022-09-08 08:46:06 -0600933 return generated_files
Andrew Lamb9563a152019-12-04 11:42:18 -0700934
935
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -0700936def _get_private_overlay_package_root(ref: uprev_lib.GitRef, package: str):
Alex Klein1699fab2022-09-08 08:46:06 -0600937 """Returns the absolute path to the root of a given private overlay.
Andrew Lambe836f222019-12-09 12:27:38 -0700938
Alex Klein1699fab2022-09-08 08:46:06 -0600939 Args:
940 ref: GitRef for the private overlay.
941 package: Path to the package in the overlay.
942 """
943 # There might be a cleaner way to map from package -> path within the source
944 # tree. For now, just use string patterns.
945 private_overlay_ref_pattern = (
946 r"/chromeos\/overlays\/overlay-([\w-]+)-private"
947 )
948 match = re.match(private_overlay_ref_pattern, ref.path)
949 if not match:
950 raise ValueError(
951 "ref.path must match the pattern: %s. Actual ref: %s"
952 % (private_overlay_ref_pattern, ref)
953 )
Andrew Lambe836f222019-12-09 12:27:38 -0700954
Alex Klein1699fab2022-09-08 08:46:06 -0600955 overlay = match.group(1)
Andrew Lambe836f222019-12-09 12:27:38 -0700956
Alex Klein1699fab2022-09-08 08:46:06 -0600957 return os.path.join(
958 constants.SOURCE_ROOT,
959 "src/private-overlays/overlay-%s-private" % overlay,
960 package,
961 )
Andrew Lambe836f222019-12-09 12:27:38 -0700962
963
Alex Klein1699fab2022-09-08 08:46:06 -0600964@uprevs_versioned_package("chromeos-base/chromeos-config-bsp")
Andrew Lambea9a8a22019-12-12 14:03:43 -0700965def replicate_private_config(_build_targets, refs, chroot):
Alex Klein1699fab2022-09-08 08:46:06 -0600966 """Replicate a private cros_config change to the corresponding public config.
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700967
Alex Klein1699fab2022-09-08 08:46:06 -0600968 See uprev_versioned_package for args
969 """
970 package = "chromeos-base/chromeos-config-bsp"
Andrew Lambea9a8a22019-12-12 14:03:43 -0700971
Alex Klein1699fab2022-09-08 08:46:06 -0600972 if len(refs) != 1:
973 raise ValueError("Expected exactly one ref, actual %s" % refs)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700974
Alex Klein1699fab2022-09-08 08:46:06 -0600975 # Expect a replication_config.jsonpb in the package root.
976 package_root = _get_private_overlay_package_root(refs[0], package)
977 replication_config_path = os.path.join(
978 package_root, "replication_config.jsonpb"
979 )
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700980
Alex Klein1699fab2022-09-08 08:46:06 -0600981 try:
982 replication_config = json_format.Parse(
983 osutils.ReadFile(replication_config_path),
984 replication_config_pb2.ReplicationConfig(),
985 )
986 except IOError:
987 raise ValueError(
988 "Expected ReplicationConfig missing at %s" % replication_config_path
989 )
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700990
Alex Klein1699fab2022-09-08 08:46:06 -0600991 replication_lib.Replicate(replication_config)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700992
Alex Klein1699fab2022-09-08 08:46:06 -0600993 modified_files = [
994 rule.destination_path
995 for rule in replication_config.file_replication_rules
996 ]
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700997
Alex Klein1699fab2022-09-08 08:46:06 -0600998 # The generated platform C files are not easily filtered by replication rules,
999 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
1000 # files cannot. Therefore, replicate and filter the JSON payloads, and then
1001 # generate filtered C files from the JSON payload.
1002 modified_files.extend(
1003 _generate_platform_c_files(replication_config, chroot)
1004 )
Andrew Lamb2bde9e42019-11-04 13:24:09 -07001005
Alex Klein1699fab2022-09-08 08:46:06 -06001006 # Use the private repo's commit hash as the new version.
1007 new_private_version = refs[0].revision
Andrew Lamb2bde9e42019-11-04 13:24:09 -07001008
Alex Klein1699fab2022-09-08 08:46:06 -06001009 # modified_files should contain only relative paths at this point, but the
1010 # returned UprevVersionedPackageResult must contain only absolute paths.
1011 for i, modified_file in enumerate(modified_files):
1012 assert not os.path.isabs(modified_file)
1013 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
Andrew Lamb988f4da2019-12-10 10:16:43 -07001014
Alex Klein1699fab2022-09-08 08:46:06 -06001015 return uprev_lib.UprevVersionedPackageResult().add_result(
1016 new_private_version, modified_files
1017 )
Andrew Lamb2bde9e42019-11-04 13:24:09 -07001018
1019
Alex Klein1699fab2022-09-08 08:46:06 -06001020@uprevs_versioned_package("chromeos-base/crosvm")
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001021def uprev_crosvm(_build_targets, refs, _chroot):
Alex Klein1699fab2022-09-08 08:46:06 -06001022 """Updates crosvm ebuilds to latest revision
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001023
Alex Klein1699fab2022-09-08 08:46:06 -06001024 crosvm is not versioned. We are updating to the latest commit on the main
1025 branch.
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001026
Alex Klein1699fab2022-09-08 08:46:06 -06001027 See: uprev_versioned_package.
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001028
Alex Klein1699fab2022-09-08 08:46:06 -06001029 Returns:
1030 UprevVersionedPackageResult: The result of updating crosvm ebuilds.
1031 """
1032 overlay = os.path.join(
1033 constants.SOURCE_ROOT, constants.CHROMIUMOS_OVERLAY_DIR
1034 )
1035 repo_path = os.path.join(constants.SOURCE_ROOT, "src", "crosvm")
1036 manifest = git.ManifestCheckout.Cached(repo_path)
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001037
Alex Klein1699fab2022-09-08 08:46:06 -06001038 uprev_manager = uprev_lib.UprevOverlayManager([overlay], manifest)
1039 uprev_manager.uprev(
1040 package_list=[
1041 "chromeos-base/crosvm",
1042 "dev-rust/assertions",
1043 "dev-rust/cros_async",
1044 "dev-rust/cros_fuzz",
1045 "dev-rust/data_model",
1046 "dev-rust/enumn",
1047 "dev-rust/io_uring",
1048 "dev-rust/p9",
1049 "dev-rust/sync",
1050 "dev-rust/sys_util",
1051 "dev-rust/tempfile",
1052 "media-sound/audio_streams",
1053 ],
1054 force=True,
1055 )
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001056
Alex Klein1699fab2022-09-08 08:46:06 -06001057 updated_files = uprev_manager.modified_ebuilds
1058 result = uprev_lib.UprevVersionedPackageResult()
1059 result.add_result(refs[0].revision, updated_files)
1060 return result
Dennis Kempinef05f2b2021-09-08 16:36:49 -07001061
1062
Alex Klein5caab872021-09-10 11:44:37 -06001063def get_best_visible(
Alex Klein1699fab2022-09-08 08:46:06 -06001064 atom: str, build_target: Optional["build_target_lib.BuildTarget"] = None
Alex Klein5caab872021-09-10 11:44:37 -06001065) -> package_info.PackageInfo:
Alex Klein1699fab2022-09-08 08:46:06 -06001066 """Returns the best visible CPV for the given atom.
Alex Kleinbbef2b32019-08-27 10:38:50 -06001067
Alex Klein1699fab2022-09-08 08:46:06 -06001068 Args:
1069 atom: The atom to look up.
1070 build_target: The build target whose sysroot should be searched, or the SDK
1071 if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001072
Alex Klein1699fab2022-09-08 08:46:06 -06001073 Returns:
1074 The best visible package, or None if none are visible.
1075 """
1076 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -06001077
Alex Klein1699fab2022-09-08 08:46:06 -06001078 return portage_util.PortageqBestVisible(
1079 atom,
1080 board=build_target.name if build_target else None,
1081 sysroot=build_target.root if build_target else None,
1082 )
Alex Kleinda39c6d2019-09-16 14:36:36 -06001083
1084
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001085def has_prebuilt(
1086 atom: str,
Alex Klein1699fab2022-09-08 08:46:06 -06001087 build_target: "build_target_lib.BuildTarget" = None,
1088 useflags: Union[Iterable[str], str] = None,
1089) -> bool:
1090 """Check if a prebuilt exists.
Alex Kleinda39c6d2019-09-16 14:36:36 -06001091
Alex Klein1699fab2022-09-08 08:46:06 -06001092 Args:
1093 atom: The package whose prebuilt is being queried.
1094 build_target: The build target whose sysroot should be searched, or the
1095 SDK if not provided.
1096 useflags: Any additional USE flags that should be set. May be a string
1097 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001098
Alex Klein1699fab2022-09-08 08:46:06 -06001099 Returns:
1100 True if there is an available prebuilt, False otherwise.
1101 """
1102 assert atom
Alex Kleinda39c6d2019-09-16 14:36:36 -06001103
Alex Klein1699fab2022-09-08 08:46:06 -06001104 board = build_target.name if build_target else None
1105 extra_env = None
1106 if useflags:
1107 new_flags = useflags
1108 if not isinstance(useflags, str):
1109 new_flags = " ".join(useflags)
Alex Klein149fd3b2019-12-16 16:01:05 -07001110
Alex Klein1699fab2022-09-08 08:46:06 -06001111 existing = os.environ.get("USE", "")
1112 final_flags = "%s %s" % (existing, new_flags)
1113 extra_env = {"USE": final_flags.strip()}
1114 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -06001115
1116
David Burger0f9dd4e2019-10-08 12:33:42 -06001117def builds(atom, build_target, packages=None):
Alex Klein1699fab2022-09-08 08:46:06 -06001118 """Check if |build_target| builds |atom| (has it in its depgraph)."""
1119 cros_build_lib.AssertInsideChroot()
Alex Klein36b117f2019-09-30 15:13:46 -06001120
Alex Klein1699fab2022-09-08 08:46:06 -06001121 pkgs = tuple(packages) if packages else None
1122 # TODO(crbug/1081828): Receive and use sysroot.
1123 graph, _sdk_graph = dependency.GetBuildDependency(
1124 build_target.root, build_target.name, pkgs
1125 )
1126 return any(atom in package for package in graph["package_deps"])
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001127
1128
Alex Klein6becabc2020-09-11 14:03:05 -06001129def needs_chrome_source(
Alex Klein1699fab2022-09-08 08:46:06 -06001130 build_target: "build_target_lib.BuildTarget",
Alex Klein6becabc2020-09-11 14:03:05 -06001131 compile_source=False,
1132 packages: Optional[List[package_info.PackageInfo]] = None,
Alex Klein1699fab2022-09-08 08:46:06 -06001133 useflags=None,
1134):
1135 """Check if the chrome source is needed.
Alex Klein6becabc2020-09-11 14:03:05 -06001136
Alex Klein1699fab2022-09-08 08:46:06 -06001137 The chrome source is needed if the build target builds chrome or any of its
1138 follower packages, and can't use a prebuilt for them either because it's not
1139 available, or because we can't use prebuilts because it must build from
1140 source.
1141 """
1142 cros_build_lib.AssertInsideChroot()
Alex Klein6becabc2020-09-11 14:03:05 -06001143
Alex Klein1699fab2022-09-08 08:46:06 -06001144 # Check if it builds chrome and/or a follower package.
1145 graph = depgraph.get_sysroot_dependency_graph(build_target.root, packages)
1146 builds_chrome = constants.CHROME_CP in graph
1147 builds_follower = {
1148 pkg: pkg in graph for pkg in constants.OTHER_CHROME_PACKAGES
1149 }
Alex Klein6becabc2020-09-11 14:03:05 -06001150
Alex Klein1699fab2022-09-08 08:46:06 -06001151 local_uprev = builds_chrome and revbump_chrome([build_target])
Alex Klein9ce3f682021-06-23 15:06:44 -06001152
Alex Klein1699fab2022-09-08 08:46:06 -06001153 # When we are compiling source set False since we do not use prebuilts.
1154 # When not compiling from source, start with True, i.e. we have every prebuilt
1155 # we've checked for up to this point.
1156 has_chrome_prebuilt = not compile_source
1157 has_follower_prebuilts = not compile_source
1158 # Save packages that need prebuilts for reporting.
1159 pkgs_needing_prebuilts = []
1160 if compile_source:
1161 # Need everything.
Alex Klein6becabc2020-09-11 14:03:05 -06001162 pkgs_needing_prebuilts.append(constants.CHROME_CP)
Alex Klein1699fab2022-09-08 08:46:06 -06001163 pkgs_needing_prebuilts.extend(
1164 [pkg for pkg, builds_pkg in builds_follower.items() if builds_pkg]
1165 )
1166 else:
1167 # Check chrome itself.
1168 if builds_chrome:
1169 has_chrome_prebuilt = has_prebuilt(
1170 constants.CHROME_CP,
1171 build_target=build_target,
1172 useflags=useflags,
1173 )
1174 if not has_chrome_prebuilt:
1175 pkgs_needing_prebuilts.append(constants.CHROME_CP)
1176 # Check follower packages.
1177 for pkg, builds_pkg in builds_follower.items():
1178 if not builds_pkg:
1179 continue
1180 prebuilt = has_prebuilt(
1181 pkg, build_target=build_target, useflags=useflags
1182 )
1183 has_follower_prebuilts &= prebuilt
1184 if not prebuilt:
1185 pkgs_needing_prebuilts.append(pkg)
1186 # Postcondition: has_chrome_prebuilt and has_follower_prebuilts now correctly
1187 # reflect whether we actually have the corresponding prebuilts for the build.
Alex Klein6becabc2020-09-11 14:03:05 -06001188
Alex Klein1699fab2022-09-08 08:46:06 -06001189 needs_chrome = builds_chrome and not has_chrome_prebuilt
1190 needs_follower = (
1191 any(builds_follower.values()) and not has_follower_prebuilts
1192 )
Alex Klein6becabc2020-09-11 14:03:05 -06001193
Alex Klein1699fab2022-09-08 08:46:06 -06001194 return NeedsChromeSourceResult(
1195 needs_chrome_source=needs_chrome or needs_follower,
1196 builds_chrome=builds_chrome,
1197 packages=[package_info.parse(p) for p in pkgs_needing_prebuilts],
1198 missing_chrome_prebuilt=not has_chrome_prebuilt,
1199 missing_follower_prebuilt=not has_follower_prebuilts,
1200 local_uprev=local_uprev,
1201 )
Alex Klein6becabc2020-09-11 14:03:05 -06001202
1203
Alex Klein68a28712021-11-08 11:08:30 -07001204class TargetVersions(NamedTuple):
Alex Klein1699fab2022-09-08 08:46:06 -06001205 """Data class for the info that makes up the "target versions"."""
1206
1207 android_version: str
1208 android_branch: str
1209 android_target: str
1210 chrome_version: str
1211 platform_version: str
1212 milestone_version: str
1213 full_version: str
Alex Klein68a28712021-11-08 11:08:30 -07001214
1215
1216def get_target_versions(
Alex Klein1699fab2022-09-08 08:46:06 -06001217 build_target: "build_target_lib.BuildTarget",
1218 packages: List[package_info.PackageInfo] = None,
Alex Klein68a28712021-11-08 11:08:30 -07001219) -> TargetVersions:
Alex Klein1699fab2022-09-08 08:46:06 -06001220 """Aggregate version info for a few key packages and the OS as a whole."""
1221 # Android version.
1222 android_version = determine_android_version(build_target.name)
1223 logging.info("Found android version: %s", android_version)
1224 # Android branch version.
1225 android_branch = determine_android_branch(build_target.name)
1226 logging.info("Found android branch version: %s", android_branch)
1227 # Android target version.
1228 android_target = determine_android_target(build_target.name)
1229 logging.info("Found android target version: %s", android_target)
Alex Klein68a28712021-11-08 11:08:30 -07001230
Alex Klein1699fab2022-09-08 08:46:06 -06001231 # TODO(crbug/1019770): Investigate cases where builds_chrome is true but
1232 # chrome_version is None.
Alex Klein68a28712021-11-08 11:08:30 -07001233
Alex Klein1699fab2022-09-08 08:46:06 -06001234 builds_chrome = builds(constants.CHROME_CP, build_target, packages=packages)
1235 chrome_version = None
1236 if builds_chrome:
1237 # Chrome version fetch.
1238 chrome_version = determine_chrome_version(build_target)
1239 logging.info("Found chrome version: %s", chrome_version)
Alex Klein68a28712021-11-08 11:08:30 -07001240
Alex Klein1699fab2022-09-08 08:46:06 -06001241 # The ChromeOS version info.
1242 platform_version = determine_platform_version()
1243 milestone_version = determine_milestone_version()
1244 full_version = determine_full_version()
Alex Klein68a28712021-11-08 11:08:30 -07001245
Alex Klein1699fab2022-09-08 08:46:06 -06001246 return TargetVersions(
1247 android_version,
1248 android_branch,
1249 android_target,
1250 chrome_version,
1251 platform_version,
1252 milestone_version,
1253 full_version,
1254 )
Alex Klein68a28712021-11-08 11:08:30 -07001255
1256
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001257def determine_chrome_version(
Alex Klein1699fab2022-09-08 08:46:06 -06001258 build_target: "build_target_lib.BuildTarget",
1259) -> Optional[str]:
1260 """Returns the current Chrome version for the board (or in buildroot).
Michael Mortensenc2615b72019-10-15 08:12:24 -06001261
Alex Klein1699fab2022-09-08 08:46:06 -06001262 Args:
1263 build_target: The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001264
Alex Klein1699fab2022-09-08 08:46:06 -06001265 Returns:
1266 The chrome version if available.
1267 """
1268 # TODO(crbug/1019770): Long term we should not need the try/catch here once
1269 # the builds function above only returns True for chrome when
1270 # determine_chrome_version will succeed.
1271 try:
1272 pkg_info = portage_util.PortageqBestVisible(
1273 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT
1274 )
1275 except cros_build_lib.RunCommandError as e:
1276 # Return None because portage failed when trying to determine the chrome
1277 # version.
1278 logging.warning("Caught exception in determine_chrome_package: %s", e)
1279 return None
1280 # Something like 78.0.3877.4_rc -> 78.0.3877.4
1281 return pkg_info.version.partition("_")[0]
Michael Mortensenc2615b72019-10-15 08:12:24 -06001282
1283
Alex Klein68a28712021-11-08 11:08:30 -07001284@functools.lru_cache()
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001285def determine_android_package(board: str) -> Optional[str]:
Alex Klein1699fab2022-09-08 08:46:06 -06001286 """Returns the active Android container package in use by the board.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001287
Alex Klein1699fab2022-09-08 08:46:06 -06001288 Args:
1289 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -07001290
Alex Klein1699fab2022-09-08 08:46:06 -06001291 Returns:
1292 The android package string if there is one.
1293 """
1294 try:
1295 packages = portage_util.GetPackageDependencies(
1296 "virtual/target-os", board=board
1297 )
1298 except cros_build_lib.RunCommandError as e:
1299 # Return None because a command (likely portage) failed when trying to
1300 # determine the package.
1301 logging.warning("Caught exception in determine_android_package: %s", e)
1302 return None
1303
1304 # We assume there is only one Android package in the depgraph.
1305 for package in packages:
1306 if package.startswith(
1307 "chromeos-base/android-container-"
1308 ) or package.startswith("chromeos-base/android-vm-"):
1309 return package
Michael Mortensene0f4b542019-10-24 15:30:23 -06001310 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001311
1312
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001313def determine_android_version(board: str, package: str = None):
Alex Klein1699fab2022-09-08 08:46:06 -06001314 """Determine the current Android version in buildroot now and return it.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001315
Alex Klein1699fab2022-09-08 08:46:06 -06001316 This uses the typical portage logic to determine which version of Android
1317 is active right now in the buildroot.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001318
Alex Klein1699fab2022-09-08 08:46:06 -06001319 Args:
1320 board: The board name this is specific to.
1321 package: The Android package, if already computed.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001322
Alex Klein1699fab2022-09-08 08:46:06 -06001323 Returns:
1324 The Android build ID of the container for the board.
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001325
Alex Klein1699fab2022-09-08 08:46:06 -06001326 Raises:
1327 NoAndroidVersionError: if no unique Android version can be determined.
1328 """
1329 if not package:
1330 package = determine_android_package(board)
1331 if not package:
1332 return None
1333 cpv = package_info.SplitCPV(package)
1334 if not cpv:
1335 raise NoAndroidVersionError(
1336 "Android version could not be determined for %s" % board
1337 )
1338 return cpv.version_no_rev
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001339
Alex Klein7a3a7dd2020-01-08 16:44:38 -07001340
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001341def determine_android_branch(board, package=None):
Alex Klein1699fab2022-09-08 08:46:06 -06001342 """Returns the Android branch in use by the active container ebuild."""
1343 if not package:
1344 package = determine_android_package(board)
1345 if not package:
1346 return None
1347 ebuild_path = portage_util.FindEbuildForBoardPackage(package, board)
1348 # We assume all targets pull from the same branch and that we always
1349 # have at least one of the following targets.
1350 targets = constants.ANDROID_ALL_BUILD_TARGETS
1351 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
1352 for target in targets:
1353 if target in ebuild_content:
1354 branch = re.search(r"(.*?)-linux-", ebuild_content[target])
1355 if branch is not None:
1356 return branch.group(1)
1357 raise NoAndroidBranchError(
1358 "Android branch could not be determined for %s (ebuild empty?)" % board
1359 )
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001360
1361
Mike Frysinger8e1c99a2021-03-05 00:58:11 -05001362def determine_android_target(board, package=None):
Alex Klein1699fab2022-09-08 08:46:06 -06001363 """Returns the Android target in use by the active container ebuild."""
1364 if not package:
1365 package = determine_android_package(board)
1366 if not package:
1367 return None
1368 if package.startswith("chromeos-base/android-vm-"):
1369 return "bertha"
1370 elif package.startswith("chromeos-base/android-container-"):
1371 return "cheets"
Michael Mortensenb70e8a82019-10-10 18:43:41 -06001372
Alex Klein1699fab2022-09-08 08:46:06 -06001373 raise NoAndroidTargetError(
1374 "Android Target cannot be determined for the package: %s" % package
1375 )
Michael Mortensen9fdb14b2019-10-17 11:17:30 -06001376
1377
1378def determine_platform_version():
Alex Klein1699fab2022-09-08 08:46:06 -06001379 """Returns the platform version from the source root."""
1380 # Platform version is something like '12575.0.0'.
1381 version = chromeos_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
1382 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -06001383
1384
1385def determine_milestone_version():
Alex Klein1699fab2022-09-08 08:46:06 -06001386 """Returns the platform version from the source root."""
1387 # Milestone version is something like '79'.
1388 version = chromeos_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
1389 return version.chrome_branch
Michael Mortensen009cb662019-10-21 11:38:43 -06001390
Alex Klein7a3a7dd2020-01-08 16:44:38 -07001391
Michael Mortensen009cb662019-10-21 11:38:43 -06001392def determine_full_version():
Alex Klein1699fab2022-09-08 08:46:06 -06001393 """Returns the full version from the source root."""
1394 # Full version is something like 'R79-12575.0.0'.
1395 milestone_version = determine_milestone_version()
1396 platform_version = determine_platform_version()
1397 full_version = "R%s-%s" % (milestone_version, platform_version)
1398 return full_version
Michael Mortensen71ef5682020-05-07 14:29:24 -06001399
1400
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001401def find_fingerprints(
Alex Klein1699fab2022-09-08 08:46:06 -06001402 build_target: "build_target_lib.BuildTarget",
1403) -> List[str]:
1404 """Returns a list of fingerprints for this build.
Michael Mortensende716a12020-05-15 11:27:00 -06001405
Alex Klein1699fab2022-09-08 08:46:06 -06001406 Args:
1407 build_target: The build target.
Michael Mortensende716a12020-05-15 11:27:00 -06001408
Alex Klein1699fab2022-09-08 08:46:06 -06001409 Returns:
1410 List of fingerprint strings.
1411 """
1412 cros_build_lib.AssertInsideChroot()
1413 fp_file = "cheets-fingerprint.txt"
1414 fp_path = os.path.join(
1415 image_lib.GetLatestImageLink(build_target.name), fp_file
1416 )
1417 if not os.path.isfile(fp_path):
1418 logging.info("Fingerprint file not found: %s", fp_path)
1419 return []
1420 logging.info("Reading fingerprint file: %s", fp_path)
1421 fingerprints = osutils.ReadFile(fp_path).splitlines()
1422 return fingerprints
Michael Mortensende716a12020-05-15 11:27:00 -06001423
1424
Alex Klein1699fab2022-09-08 08:46:06 -06001425def get_all_firmware_versions(build_target: "build_target_lib.BuildTarget"):
1426 """Extract firmware version for all models present.
Michael Mortensen59e30872020-05-18 14:12:49 -06001427
Alex Klein1699fab2022-09-08 08:46:06 -06001428 Args:
1429 build_target: The build target.
Michael Mortensen59e30872020-05-18 14:12:49 -06001430
Alex Klein1699fab2022-09-08 08:46:06 -06001431 Returns:
1432 A dict of FirmwareVersions namedtuple instances by model.
1433 Each element will be populated based on whether it was present in the
1434 command output.
1435 """
1436 cros_build_lib.AssertInsideChroot()
1437 result = {}
1438 # Note that example output for _get_firmware_version_cmd_result is available
1439 # in the packages_unittest.py for testing get_all_firmware_versions.
1440 cmd_result = _get_firmware_version_cmd_result(build_target)
Michael Mortensen59e30872020-05-18 14:12:49 -06001441
Alex Klein1699fab2022-09-08 08:46:06 -06001442 if cmd_result:
1443 # There is a blank line between the version info for each model.
1444 firmware_version_payloads = cmd_result.split("\n\n")
1445 for firmware_version_payload in firmware_version_payloads:
1446 if "BIOS" in firmware_version_payload:
1447 firmware_version = _find_firmware_versions(
1448 firmware_version_payload
1449 )
1450 result[firmware_version.model] = firmware_version
1451 return result
Michael Mortensen59e30872020-05-18 14:12:49 -06001452
1453
Benjamin Shai0858cd32022-01-10 20:23:49 +00001454class FirmwareVersions(NamedTuple):
Alex Klein1699fab2022-09-08 08:46:06 -06001455 """Tuple to hold firmware versions, with truthiness."""
Benjamin Shai0858cd32022-01-10 20:23:49 +00001456
Alex Klein1699fab2022-09-08 08:46:06 -06001457 model: Optional[str]
1458 main: Optional[str]
1459 main_rw: Optional[str]
1460 ec: Optional[str]
1461 ec_rw: Optional[str]
1462
1463 def __bool__(self):
1464 return bool(
1465 self.model or self.main or self.main_rw or self.ec or self.ec_rw
1466 )
Michael Mortensen71ef5682020-05-07 14:29:24 -06001467
1468
Alex Klein1699fab2022-09-08 08:46:06 -06001469def get_firmware_versions(build_target: "build_target_lib.BuildTarget"):
1470 """Extract version information from the firmware updater, if one exists.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001471
Alex Klein1699fab2022-09-08 08:46:06 -06001472 Args:
1473 build_target: The build target.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001474
Alex Klein1699fab2022-09-08 08:46:06 -06001475 Returns:
1476 A FirmwareVersions namedtuple instance.
1477 Each element will either be set to the string output by the firmware
1478 updater shellball, or None if there is no firmware updater.
1479 """
1480 cros_build_lib.AssertInsideChroot()
1481 cmd_result = _get_firmware_version_cmd_result(build_target)
1482 if cmd_result:
1483 return _find_firmware_versions(cmd_result)
1484 else:
1485 return FirmwareVersions(None, None, None, None, None)
Michael Mortensen71ef5682020-05-07 14:29:24 -06001486
1487
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001488def _get_firmware_version_cmd_result(
Alex Klein1699fab2022-09-08 08:46:06 -06001489 build_target: "build_target_lib.BuildTarget",
1490) -> Optional[str]:
1491 """Gets the raw result output of the firmware updater version command.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001492
Alex Klein1699fab2022-09-08 08:46:06 -06001493 Args:
1494 build_target: The build target.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001495
Alex Klein1699fab2022-09-08 08:46:06 -06001496 Returns:
1497 Command execution result.
1498 """
1499 updater = os.path.join(
1500 build_target.root, "usr/sbin/chromeos-firmwareupdate"
1501 )
1502 logging.info("Calling updater %s", updater)
1503 # Call the updater using the chroot-based path.
1504 try:
1505 return cros_build_lib.run(
1506 [updater, "-V"],
1507 capture_output=True,
1508 log_output=True,
1509 encoding="utf-8",
1510 ).stdout
1511 except cros_build_lib.RunCommandError:
1512 # Updater probably doesn't exist (e.g. betty).
1513 return None
Michael Mortensen71ef5682020-05-07 14:29:24 -06001514
1515
1516def _find_firmware_versions(cmd_output):
Alex Klein1699fab2022-09-08 08:46:06 -06001517 """Finds firmware version output via regex matches against the cmd_output.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001518
Alex Klein1699fab2022-09-08 08:46:06 -06001519 Args:
1520 cmd_output: The raw output to search against.
Michael Mortensen71ef5682020-05-07 14:29:24 -06001521
Alex Klein1699fab2022-09-08 08:46:06 -06001522 Returns:
1523 FirmwareVersions namedtuple with results.
1524 Each element will either be set to the string output by the firmware
1525 updater shellball, or None if there is no match.
1526 """
Michael Mortensen71ef5682020-05-07 14:29:24 -06001527
Alex Klein1699fab2022-09-08 08:46:06 -06001528 # Sometimes a firmware bundle includes a special combination of RO+RW
1529 # firmware. In this case, the RW firmware version is indicated with a "(RW)
1530 # version" field. In other cases, the "(RW) version" field is not present.
1531 # Therefore, search for the "(RW)" fields first and if they aren't present,
1532 # fallback to the other format. e.g. just "BIOS version:".
1533 # TODO(mmortensen): Use JSON once the firmware updater supports it.
1534 main = None
1535 main_rw = None
1536 ec = None
1537 ec_rw = None
1538 model = None
Michael Mortensen71ef5682020-05-07 14:29:24 -06001539
Alex Klein1699fab2022-09-08 08:46:06 -06001540 match = re.search(r"BIOS version:\s*(?P<version>.*)", cmd_output)
1541 if match:
1542 main = match.group("version")
Michael Mortensen71ef5682020-05-07 14:29:24 -06001543
Alex Klein1699fab2022-09-08 08:46:06 -06001544 match = re.search(r"BIOS \(RW\) version:\s*(?P<version>.*)", cmd_output)
1545 if match:
1546 main_rw = match.group("version")
Michael Mortensen71ef5682020-05-07 14:29:24 -06001547
Alex Klein1699fab2022-09-08 08:46:06 -06001548 match = re.search(r"EC version:\s*(?P<version>.*)", cmd_output)
1549 if match:
1550 ec = match.group("version")
Michael Mortensen71ef5682020-05-07 14:29:24 -06001551
Alex Klein1699fab2022-09-08 08:46:06 -06001552 match = re.search(r"EC \(RW\) version:\s*(?P<version>.*)", cmd_output)
1553 if match:
1554 ec_rw = match.group("version")
Michael Mortensen71ef5682020-05-07 14:29:24 -06001555
Alex Klein1699fab2022-09-08 08:46:06 -06001556 match = re.search(r"Model:\s*(?P<model>.*)", cmd_output)
1557 if match:
1558 model = match.group("model")
Michael Mortensen71ef5682020-05-07 14:29:24 -06001559
Alex Klein1699fab2022-09-08 08:46:06 -06001560 return FirmwareVersions(model, main, main_rw, ec, ec_rw)
Michael Mortensena4af79e2020-05-06 16:18:48 -06001561
1562
Benjamin Shai0858cd32022-01-10 20:23:49 +00001563class MainEcFirmwareVersions(NamedTuple):
Alex Klein1699fab2022-09-08 08:46:06 -06001564 """Tuple to hold main and ec firmware versions, with truthiness."""
Benjamin Shai0858cd32022-01-10 20:23:49 +00001565
Alex Klein1699fab2022-09-08 08:46:06 -06001566 main_fw_version: Optional[str]
1567 ec_fw_version: Optional[str]
1568
1569 def __bool__(self):
1570 return bool(self.main_fw_version or self.ec_fw_version)
Benjamin Shai0858cd32022-01-10 20:23:49 +00001571
Michael Mortensena4af79e2020-05-06 16:18:48 -06001572
Alex Klein1699fab2022-09-08 08:46:06 -06001573def determine_firmware_versions(build_target: "build_target_lib.BuildTarget"):
1574 """Returns a namedtuple with main and ec firmware versions.
Michael Mortensena4af79e2020-05-06 16:18:48 -06001575
Alex Klein1699fab2022-09-08 08:46:06 -06001576 Args:
1577 build_target: The build target.
Michael Mortensena4af79e2020-05-06 16:18:48 -06001578
Alex Klein1699fab2022-09-08 08:46:06 -06001579 Returns:
1580 MainEcFirmwareVersions namedtuple with results.
1581 """
1582 fw_versions = get_firmware_versions(build_target)
1583 main_fw_version = fw_versions.main_rw or fw_versions.main
1584 ec_fw_version = fw_versions.ec_rw or fw_versions.ec
Michael Mortensena4af79e2020-05-06 16:18:48 -06001585
Alex Klein1699fab2022-09-08 08:46:06 -06001586 return MainEcFirmwareVersions(main_fw_version, ec_fw_version)
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001587
Benjamin Shai0858cd32022-01-10 20:23:49 +00001588
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001589def determine_kernel_version(
Alex Klein1699fab2022-09-08 08:46:06 -06001590 build_target: "build_target_lib.BuildTarget",
1591) -> Optional[str]:
1592 """Returns a string containing the kernel version for this build target.
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001593
Alex Klein1699fab2022-09-08 08:46:06 -06001594 Args:
1595 build_target: The build target.
Michael Mortensenfbf2b2d2020-05-14 16:33:06 -06001596
Alex Klein1699fab2022-09-08 08:46:06 -06001597 Returns:
1598 The kernel versions, or None.
1599 """
1600 try:
1601 packages = portage_util.GetPackageDependencies(
1602 "virtual/linux-sources", board=build_target.name
1603 )
1604 except cros_build_lib.RunCommandError as e:
1605 logging.warning("Unable to get package list for metadata: %s", e)
1606 return None
1607 for package in packages:
1608 if package.startswith("sys-kernel/chromeos-kernel-"):
1609 kernel_version = package_info.SplitCPV(package).version
1610 logging.info("Found active kernel version: %s", kernel_version)
1611 return kernel_version
Benjamin Shai5ce11a22022-05-06 14:10:53 +00001612 return None
Michael Mortensen125bb012020-05-21 14:02:10 -06001613
1614
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001615def get_models(
Alex Klein1699fab2022-09-08 08:46:06 -06001616 build_target: "build_target_lib.BuildTarget", log_output: bool = True
1617) -> Optional[List[str]]:
1618 """Obtain a list of models supported by a unified board.
Michael Mortensen125bb012020-05-21 14:02:10 -06001619
Alex Klein1699fab2022-09-08 08:46:06 -06001620 This ignored whitelabel models since GoldenEye has no specific support for
1621 these at present.
Michael Mortensen125bb012020-05-21 14:02:10 -06001622
Alex Klein1699fab2022-09-08 08:46:06 -06001623 Args:
1624 build_target: The build target.
1625 log_output: Whether to log the output of the cros_config_host invocation.
Michael Mortensen125bb012020-05-21 14:02:10 -06001626
Alex Klein1699fab2022-09-08 08:46:06 -06001627 Returns:
1628 A list of models supported by this board, if it is a unified build; None,
1629 if it is not a unified build.
1630 """
1631 return _run_cros_config_host(
1632 build_target, ["list-models"], log_output=log_output
1633 )
Michael Mortensen125bb012020-05-21 14:02:10 -06001634
1635
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001636def get_key_id(
Alex Klein1699fab2022-09-08 08:46:06 -06001637 build_target: "build_target_lib.BuildTarget", model: str
1638) -> Optional[str]:
1639 """Obtain the key_id for a model within the build_target.
Michael Mortensen359c1f32020-05-28 19:35:42 -06001640
Alex Klein1699fab2022-09-08 08:46:06 -06001641 Args:
1642 build_target: The build target.
1643 model: The model name
Michael Mortensen359c1f32020-05-28 19:35:42 -06001644
Alex Klein1699fab2022-09-08 08:46:06 -06001645 Returns:
1646 A key_id or None.
1647 """
1648 model_arg = "--model=" + model
1649 key_id_list = _run_cros_config_host(
1650 build_target, [model_arg, "get", "/firmware-signing", "key-id"]
1651 )
1652 key_id = None
1653 if len(key_id_list) == 1:
1654 key_id = key_id_list[0]
1655 return key_id
Michael Mortensen359c1f32020-05-28 19:35:42 -06001656
1657
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001658def _run_cros_config_host(
Alex Klein1699fab2022-09-08 08:46:06 -06001659 build_target: "build_target_lib.BuildTarget",
Matthias Kaehlckebf7d1772021-11-04 16:01:36 -07001660 args: List[str],
Alex Klein1699fab2022-09-08 08:46:06 -06001661 log_output: bool = True,
1662) -> Optional[List[str]]:
1663 """Run the cros_config_host tool.
Michael Mortensen125bb012020-05-21 14:02:10 -06001664
Alex Klein1699fab2022-09-08 08:46:06 -06001665 Args:
1666 build_target: The build target.
1667 args: List of arguments to pass.
1668 log_output: Whether to log the output of the cros_config_host.
Michael Mortensen125bb012020-05-21 14:02:10 -06001669
Alex Klein1699fab2022-09-08 08:46:06 -06001670 Returns:
1671 Output of the tool
1672 """
1673 cros_build_lib.AssertInsideChroot()
1674 tool = "/usr/bin/cros_config_host"
1675 if not os.path.isfile(tool):
1676 return None
Michael Mortensen125bb012020-05-21 14:02:10 -06001677
Alex Klein1699fab2022-09-08 08:46:06 -06001678 config_fname = build_target.full_path(
1679 "usr/share/chromeos-config/yaml/config.yaml"
1680 )
Michael Mortensen125bb012020-05-21 14:02:10 -06001681
Alex Klein1699fab2022-09-08 08:46:06 -06001682 result = cros_build_lib.run(
1683 [tool, "-c", config_fname] + args,
1684 capture_output=True,
1685 encoding="utf-8",
1686 log_output=log_output,
1687 check=False,
1688 )
1689 if result.returncode:
1690 # Show the output for debugging purposes.
1691 if "No such file or directory" not in result.stderr:
1692 logging.error("cros_config_host failed: %s\n", result.stderr)
1693 return None
1694 return result.stdout.strip().splitlines()