blob: 8d612a8dbe4f90d8e0e6f037fcb2eef98d429b4a [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Mike Frysingerbafb3182020-02-21 03:15:43 -050036
Alex Klein87531182019-08-12 15:23:37 -060037# Registered handlers for uprevving versioned packages.
38_UPREV_FUNCS = {}
39
Alex Kleineb77ffa2019-05-28 14:47:44 -060040
41class Error(Exception):
42 """Module's base error class."""
43
44
Alex Klein4de25e82019-08-05 15:58:39 -060045class UnknownPackageError(Error):
46 """Uprev attempted for a package without a registered handler."""
47
48
Alex Kleineb77ffa2019-05-28 14:47:44 -060049class UprevError(Error):
50 """An error occurred while uprevving packages."""
51
52
Michael Mortensenb70e8a82019-10-10 18:43:41 -060053class NoAndroidVersionError(Error):
54 """An error occurred while trying to determine the android version."""
55
56
57class NoAndroidBranchError(Error):
58 """An error occurred while trying to determine the android branch."""
59
60
61class NoAndroidTargetError(Error):
62 """An error occurred while trying to determine the android target."""
63
64
Alex Klein4de25e82019-08-05 15:58:39 -060065class AndroidIsPinnedUprevError(UprevError):
66 """Raised when we try to uprev while Android is pinned."""
67
68 def __init__(self, new_android_atom):
69 """Initialize a AndroidIsPinnedUprevError.
70
71 Args:
72 new_android_atom: The Android atom that we failed to
73 uprev to, due to Android being pinned.
74 """
75 assert new_android_atom
76 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
77 new_android_atom)
78 super(AndroidIsPinnedUprevError, self).__init__(msg)
79 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060080
81
Yaakov Shaul1eafe832019-09-10 16:50:26 -060082class EbuildManifestError(Error):
83 """Error when running ebuild manifest."""
84
85
Andrew Lamb9563a152019-12-04 11:42:18 -070086class GeneratedCrosConfigFilesError(Error):
87 """Error when cros_config_schema does not produce expected files"""
88
89 def __init__(self, expected_files, found_files):
90 msg = ('Expected to find generated C files: %s. Actually found: %s' %
91 (expected_files, found_files))
92 super(GeneratedCrosConfigFilesError, self).__init__(msg)
93
Alex Klein7a3a7dd2020-01-08 16:44:38 -070094
Yaakov Shaul730814a2019-09-10 13:58:25 -060095UprevVersionedPackageModifications = collections.namedtuple(
96 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -060097
Yaakov Shaul730814a2019-09-10 13:58:25 -060098
99class UprevVersionedPackageResult(object):
100 """Data object for uprev_versioned_package."""
101
102 def __init__(self):
103 self.modified = []
104
105 def add_result(self, new_version, modified_files):
106 """Adds version/ebuilds tuple to result.
107
108 Args:
109 new_version: New version number of package.
110 modified_files: List of files modified for the given version.
111 """
112 result = UprevVersionedPackageModifications(new_version, modified_files)
113 self.modified.append(result)
114 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600115
116 @property
117 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600118 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600119
120
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600121def patch_ebuild_vars(ebuild_path, variables):
122 """Updates variables in ebuild.
123
124 Use this function rather than portage_util.EBuild.UpdateEBuild when you
125 want to preserve the variable position and quotes within the ebuild.
126
127 Args:
128 ebuild_path: The path of the ebuild.
129 variables: Dictionary of variables to update in ebuild.
130 """
131 try:
132 for line in fileinput.input(ebuild_path, inplace=1):
133 varname, eq, _ = line.partition('=')
134 if eq == '=' and varname.strip() in variables:
135 value = variables[varname]
136 sys.stdout.write('%s="%s"\n' % (varname, value))
137 else:
138 sys.stdout.write(line)
139 finally:
140 fileinput.close()
141
142
Alex Klein87531182019-08-12 15:23:37 -0600143def uprevs_versioned_package(package):
144 """Decorator to register package uprev handlers."""
145 assert package
146
147 def register(func):
148 """Registers |func| as a handler for |package|."""
149 _UPREV_FUNCS[package] = func
150
151 @functools.wraps(func)
152 def pass_through(*args, **kwargs):
153 return func(*args, **kwargs)
154
155 return pass_through
156
157 return register
158
159
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700160def uprev_android(tracking_branch,
161 android_package,
162 android_build_branch,
163 chroot,
164 build_targets=None,
165 android_version=None,
Alex Klein4de25e82019-08-05 15:58:39 -0600166 android_gts_build_branch=None):
167 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700168 command = [
169 'cros_mark_android_as_stable',
170 '--tracking_branch=%s' % tracking_branch,
171 '--android_package=%s' % android_package,
172 '--android_build_branch=%s' % android_build_branch,
173 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600174 if build_targets:
175 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
176 if android_version:
177 command.append('--force_version=%s' % android_version)
178 if android_gts_build_branch:
179 command.append('--android_gts_build_branch=%s' % android_gts_build_branch)
180
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700181 result = cros_build_lib.run(
182 command,
183 stdout=True,
184 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500185 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700186 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600187
Mike Frysinger88d96362020-02-14 19:05:45 -0500188 portage_atom_string = result.stdout.strip()
189 android_atom = None
190 if portage_atom_string:
191 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600192 if not android_atom:
193 logging.info('Found nothing to rev.')
194 return None
195
196 for target in build_targets or []:
197 # Sanity check: We should always be able to merge the version of
198 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700199 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600200 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700201 cros_build_lib.run(
202 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600203 except cros_build_lib.RunCommandError:
204 logging.error(
205 'Cannot emerge-%s =%s\nIs Android pinned to an older '
206 'version?', target, android_atom)
207 raise AndroidIsPinnedUprevError(android_atom)
208
209 return android_atom
210
211
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700212def uprev_build_targets(build_targets,
213 overlay_type,
214 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600215 output_dir=None):
216 """Uprev the set provided build targets, or all if not specified.
217
218 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600219 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600220 whose overlays should be uprevved, empty or None for all.
221 overlay_type (str): One of the valid overlay types except None (see
222 constants.VALID_OVERLAYS).
223 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
224 output_dir (str|None): The path to optionally dump result files.
225 """
226 # Need a valid overlay, but exclude None.
227 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
228
229 if build_targets:
230 overlays = portage_util.FindOverlaysForBoards(
231 overlay_type, boards=[t.name for t in build_targets])
232 else:
233 overlays = portage_util.FindOverlays(overlay_type)
234
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700235 return uprev_overlays(
236 overlays,
237 build_targets=build_targets,
238 chroot=chroot,
239 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600240
241
242def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
243 """Uprev the given overlays.
244
245 Args:
246 overlays (list[str]): The list of overlay paths.
Alex Klein2960c752020-03-09 13:43:38 -0600247 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600248 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
249 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
250 output_dir (str|None): The path to optionally dump result files.
251
252 Returns:
253 list[str] - The paths to all of the modified ebuild files. This includes the
254 new files that were added (i.e. the new versions) and all of the removed
255 files (i.e. the old versions).
256 """
257 assert overlays
258
259 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
260
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700261 uprev_manager = uprev_lib.UprevOverlayManager(
262 overlays,
263 manifest,
264 build_targets=build_targets,
265 chroot=chroot,
266 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600267 uprev_manager.uprev()
268
269 return uprev_manager.modified_ebuilds
270
271
Alex Klein87531182019-08-12 15:23:37 -0600272def uprev_versioned_package(package, build_targets, refs, chroot):
273 """Call registered uprev handler function for the package.
274
275 Args:
276 package (portage_util.CPV): The package being uprevved.
Alex Klein2960c752020-03-09 13:43:38 -0600277 build_targets (list[build_target_lib.BuildTarget]): The build targets to
Alex Klein87531182019-08-12 15:23:37 -0600278 clean on a successful uprev.
279 refs (list[uprev_lib.GitRef]):
280 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
281
282 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600283 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600284 """
285 assert package
286
287 if package.cp not in _UPREV_FUNCS:
288 raise UnknownPackageError(
289 'Package "%s" does not have a registered handler.' % package.cp)
290
Andrew Lambea9a8a22019-12-12 14:03:43 -0700291 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600292
293
Yaakov Shaul395ae832019-09-09 14:45:32 -0600294@uprevs_versioned_package('afdo/kernel-profiles')
295def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600296 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600297
298 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600299
300 Raises:
301 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600302 """
303 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
304 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
305
David Burger92485342019-09-10 17:52:45 -0600306 with open(path, 'r') as f:
307 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600308
Yaakov Shaul730814a2019-09-10 13:58:25 -0600309 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600310 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600311 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
312 'sys-kernel', version)
313 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
314 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600315 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
316 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600317 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600318 patch_ebuild_vars(ebuild_path,
319 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600320
321 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600322 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400323 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600324 except cros_build_lib.RunCommandError as e:
325 raise EbuildManifestError(
326 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600327 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600328
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600329 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600330
Yaakov Shaul730814a2019-09-10 13:58:25 -0600331 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
332
333 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600334
335
Trent Beginaf51f1b2020-03-09 17:35:31 -0600336@uprevs_versioned_package('chromeos-base/termina-image-amd64')
337def uprev_termina_amd64(_build_targets, _refs, chroot):
338 """Updates termina amd64 VM - chromeos-base/termina-image-amd64.
339
340 See: uprev_versioned_package.
341 """
342 return uprev_termina('termina-image-amd64', chroot)
343
344
345@uprevs_versioned_package('chromeos-base/termina-image-arm')
346def uprev_termina_arm(_build_targets, _refs, chroot):
347 """Updates termina arm VM - chromeos-base/termina-image-arm.
348
349 See: uprev_versioned_package.
350 """
351 return uprev_termina('termina-image-arm', chroot)
352
353
354def uprev_termina(package, chroot):
355 """Helper function to uprev termina VM.
356
357 Args:
358 package (string): name of the package
359 chroot (chroot_lib.Chroot): specify a chroot to enter.
360
361 Returns:
362 UprevVersionedPackageResult: The result.
363 """
364 package_path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos-base',
365 package)
366 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
367 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
368
369
Trent Begin315d9d92019-12-03 21:55:53 -0700370@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Beginaf51f1b2020-03-09 17:35:31 -0600371def uprev_sludge(_build_targets, _refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700372 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
373
374 See: uprev_versioned_package.
375 """
376 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700377 package_path = os.path.join('src', 'private-overlays',
378 'project-wilco-private', 'chromeos-base', package)
379 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700380
Trent Begin6daa8702020-01-29 14:58:12 -0700381 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700382
383
Trent Begin6daa8702020-01-29 14:58:12 -0700384def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700385 """Changes the package ebuild's version to match the version pin file.
386
387 Args:
388 package_path: The path of the package relative to the src root. This path
389 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700390 version_pin_path: The path of the version_pin file that contains only a
391 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700392 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700393 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700394
395 Returns:
396 UprevVersionedPackageResult: The result.
397 """
398 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700399
400 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
401 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700402 if not ebuild_paths:
403 raise UprevError('No ebuilds found for %s' % package)
404 elif len(ebuild_paths) > 1:
405 raise UprevError('Multiple ebuilds found for %s' % package)
406 else:
407 ebuild_path = ebuild_paths[0]
408
Trent Begind943df92020-02-25 10:30:10 -0700409 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
410 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700411 new_ebuild_path = os.path.join(package_path,
412 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700413 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
414 os.rename(ebuild_path, new_ebuild_src_path)
Trent Begin4a11a632020-02-28 12:59:58 -0700415 manifest_src_path = os.path.join(package_src_path, 'Manifest')
Trent Begind943df92020-02-25 10:30:10 -0700416 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
417 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700418
Trent Begin6daa8702020-01-29 14:58:12 -0700419 try:
Trent Begind943df92020-02-25 10:30:10 -0700420 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700421 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700422 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700423 'Unable to update manifest for %s: %s' % (package, e.stderr))
424
Trent Begin315d9d92019-12-03 21:55:53 -0700425 result = UprevVersionedPackageResult()
Trent Begin4a11a632020-02-28 12:59:58 -0700426 result.add_result(version,
427 [new_ebuild_src_path, ebuild_path, manifest_src_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700428 return result
429
430
Alex Klein87531182019-08-12 15:23:37 -0600431@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700432def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600433 """Uprev chrome and its related packages.
434
435 See: uprev_versioned_package.
436 """
437 # Determine the version from the refs (tags), i.e. the chrome versions are the
438 # tag names.
439 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
440
441 uprev_manager = uprev_lib.UprevChromeManager(
442 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600443 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600444 # Start with chrome itself, as we can't do anything else unless chrome
445 # uprevs successfully.
446 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600447 return result
Alex Klein87531182019-08-12 15:23:37 -0600448
449 # With a successful chrome rev, also uprev related packages.
450 for package in constants.OTHER_CHROME_PACKAGES:
451 uprev_manager.uprev(package)
452
David Burger37f48672019-09-18 17:07:56 -0600453 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600454
455
Andrew Lamb9563a152019-12-04 11:42:18 -0700456def _generate_platform_c_files(replication_config, chroot):
457 """Generates platform C files from a platform JSON payload.
458
459 Args:
460 replication_config (replication_config_pb2.ReplicationConfig): A
461 ReplicationConfig that has already been run. If it produced a
462 build_config.json file, that file will be used to generate platform C
463 files. Otherwise, nothing will be generated.
464 chroot (chroot_lib.Chroot): The chroot to use to generate.
465
466 Returns:
467 A list of generated files.
468 """
469 # Generate the platform C files from the build config. Note that it would be
470 # more intuitive to generate the platform C files from the platform config;
471 # however, cros_config_schema does not allow this, because the platform config
472 # payload is not always valid input. For example, if a property is both
473 # 'required' and 'build-only', it will fail schema validation. Thus, use the
474 # build config, and use '-f' to filter.
475 build_config_path = [
476 rule.destination_path
477 for rule in replication_config.file_replication_rules
478 if rule.destination_path.endswith('build_config.json')
479 ]
480
481 if not build_config_path:
482 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700483 'No build_config.json found, will not generate platform C files. '
484 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700485 return []
486
487 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700488 raise ValueError('Expected at most one build_config.json destination path. '
489 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700490
491 build_config_path = build_config_path[0]
492
493 # Paths to the build_config.json and dir to output C files to, in the
494 # chroot.
495 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
496 build_config_path)
497 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
498 os.path.dirname(build_config_path))
499
500 command = [
501 'cros_config_schema', '-m', build_config_chroot_path, '-g',
502 generated_output_chroot_dir, '-f', '"TRUE"'
503 ]
504
505 cros_build_lib.run(
506 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
507
508 # A relative (to the source root) path to the generated C files.
509 generated_output_dir = os.path.dirname(build_config_path)
510 generated_files = []
511 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
512 for f in expected_c_files:
513 if os.path.exists(
514 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
515 generated_files.append(os.path.join(generated_output_dir, f))
516
517 if len(expected_c_files) != len(generated_files):
518 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
519
520 return generated_files
521
522
Andrew Lambe836f222019-12-09 12:27:38 -0700523def _get_private_overlay_package_root(ref, package):
524 """Returns the absolute path to the root of a given private overlay.
525
526 Args:
527 ref (uprev_lib.GitRef): GitRef for the private overlay.
528 package (str): Path to the package in the overlay.
529 """
530 # There might be a cleaner way to map from package -> path within the source
531 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700532 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700533 match = re.match(private_overlay_ref_pattern, ref.path)
534 if not match:
535 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
536 (private_overlay_ref_pattern, ref))
537
538 overlay = match.group(1)
539
540 return os.path.join(constants.SOURCE_ROOT,
541 'src/private-overlays/overlay-%s-private' % overlay,
542 package)
543
544
Andrew Lambea9a8a22019-12-12 14:03:43 -0700545@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
546def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700547 """Replicate a private cros_config change to the corresponding public config.
548
Alex Kleinad6b48a2020-01-08 16:57:41 -0700549 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700550 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700551 package = 'chromeos-base/chromeos-config-bsp'
552
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700553 if len(refs) != 1:
554 raise ValueError('Expected exactly one ref, actual %s' % refs)
555
556 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700557 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700558 replication_config_path = os.path.join(package_root,
559 'replication_config.jsonpb')
560
561 try:
562 replication_config = json_format.Parse(
563 osutils.ReadFile(replication_config_path),
564 replication_config_pb2.ReplicationConfig())
565 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700566 raise ValueError(
567 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700568
569 replication_lib.Replicate(replication_config)
570
571 modified_files = [
572 rule.destination_path
573 for rule in replication_config.file_replication_rules
574 ]
575
Andrew Lamb9563a152019-12-04 11:42:18 -0700576 # The generated platform C files are not easily filtered by replication rules,
577 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
578 # files cannot. Therefore, replicate and filter the JSON payloads, and then
579 # generate filtered C files from the JSON payload.
580 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700581
582 # Use the private repo's commit hash as the new version.
583 new_private_version = refs[0].revision
584
Andrew Lamb988f4da2019-12-10 10:16:43 -0700585 # modified_files should contain only relative paths at this point, but the
586 # returned UprevVersionedPackageResult must contain only absolute paths.
587 for i, modified_file in enumerate(modified_files):
588 assert not os.path.isabs(modified_file)
589 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
590
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700591 return UprevVersionedPackageResult().add_result(new_private_version,
592 modified_files)
593
594
Alex Kleinbbef2b32019-08-27 10:38:50 -0600595def get_best_visible(atom, build_target=None):
596 """Returns the best visible CPV for the given atom.
597
598 Args:
599 atom (str): The atom to look up.
Alex Klein2960c752020-03-09 13:43:38 -0600600 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600601 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700602
603 Returns:
604 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600605 """
David Burger1e0fe232019-07-01 14:52:07 -0600606 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600607
608 board = build_target.name if build_target else None
609 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600610
611
Alex Klein149fd3b2019-12-16 16:01:05 -0700612def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600613 """Check if a prebuilt exists.
614
615 Args:
616 atom (str): The package whose prebuilt is being queried.
Alex Klein2960c752020-03-09 13:43:38 -0600617 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600618 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700619 useflags: Any additional USE flags that should be set. May be a string
620 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700621
622 Returns:
623 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600624 """
625 assert atom
626
627 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700628 extra_env = None
629 if useflags:
630 new_flags = useflags
631 if not isinstance(useflags, six.string_types):
632 new_flags = ' '.join(useflags)
633
634 existing = os.environ.get('USE', '')
635 final_flags = '%s %s' % (existing, new_flags)
636 extra_env = {'USE': final_flags.strip()}
637 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600638
639
David Burger0f9dd4e2019-10-08 12:33:42 -0600640def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600641 """Check if |build_target| builds |atom| (has it in its depgraph)."""
642 cros_build_lib.AssertInsideChroot()
643
Chris McDonalda22b74f2019-11-22 13:55:06 -0700644 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600645 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600646
647
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600648def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600649 """Returns the current Chrome version for the board (or in buildroot).
650
651 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600652 build_target (build_target_lib.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700653
654 Returns:
655 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600656 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600657 # TODO(crbug/1019770): Long term we should not need the try/catch here once
658 # the builds function above only returns True for chrome when
659 # determine_chrome_version will succeed.
660 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700661 cpv = portage_util.PortageqBestVisible(
662 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600663 except cros_build_lib.RunCommandError as e:
664 # Return None because portage failed when trying to determine the chrome
665 # version.
666 logging.warning('Caught exception in determine_chrome_package: %s', e)
667 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600668 # Something like 78.0.3877.4_rc -> 78.0.3877.4
669 return cpv.version_no_rev.partition('_')[0]
670
671
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600672def determine_android_package(board):
673 """Returns the active Android container package in use by the board.
674
675 Args:
676 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700677
678 Returns:
679 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600680 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600681 try:
682 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600683 except cros_build_lib.RunCommandError as e:
684 # Return None because a command (likely portage) failed when trying to
685 # determine the package.
686 logging.warning('Caught exception in determine_android_package: %s', e)
687 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600688
Alex Kleinad6b48a2020-01-08 16:57:41 -0700689 # We assume there is only one Android package in the depgraph.
690 for package in packages:
691 if package.startswith('chromeos-base/android-container-') or \
692 package.startswith('chromeos-base/android-vm-'):
693 return package
694 return None
695
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600696
697def determine_android_version(boards=None):
698 """Determine the current Android version in buildroot now and return it.
699
700 This uses the typical portage logic to determine which version of Android
701 is active right now in the buildroot.
702
703 Args:
704 boards: List of boards to check version of.
705
706 Returns:
707 The Android build ID of the container for the boards.
708
709 Raises:
710 NoAndroidVersionError: if no unique Android version can be determined.
711 """
712 if not boards:
713 return None
714 # Verify that all boards have the same version.
715 version = None
716 for board in boards:
717 package = determine_android_package(board)
718 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600719 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600720 cpv = portage_util.SplitCPV(package)
721 if not cpv:
722 raise NoAndroidVersionError(
723 'Android version could not be determined for %s' % board)
724 if not version:
725 version = cpv.version_no_rev
726 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700727 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
728 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600729 return version
730
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700731
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600732def determine_android_branch(board):
733 """Returns the Android branch in use by the active container ebuild."""
734 try:
735 android_package = determine_android_package(board)
736 except cros_build_lib.RunCommandError:
737 raise NoAndroidBranchError(
738 'Android branch could not be determined for %s' % board)
739 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600740 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600741 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
742 # We assume all targets pull from the same branch and that we always
743 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
744 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
745 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
746 for target in targets:
747 if target in ebuild_content:
748 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
749 if branch is not None:
750 return branch.group(1)
751 raise NoAndroidBranchError(
752 'Android branch could not be determined for %s (ebuild empty?)' % board)
753
754
755def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600756 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600757 try:
758 android_package = determine_android_package(board)
759 except cros_build_lib.RunCommandError:
760 raise NoAndroidTargetError(
761 'Android Target could not be determined for %s' % board)
762 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600763 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600764 if android_package.startswith('chromeos-base/android-vm-'):
765 return 'bertha'
766 elif android_package.startswith('chromeos-base/android-container-'):
767 return 'cheets'
768
769 raise NoAndroidTargetError(
770 'Android Target cannot be determined for the package: %s' %
771 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600772
773
774def determine_platform_version():
775 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600776 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600777 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
778 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600779
780
781def determine_milestone_version():
782 """Returns the platform version from the source root."""
783 # Milestone version is something like '79'.
784 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
785 return version.chrome_branch
786
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700787
Michael Mortensen009cb662019-10-21 11:38:43 -0600788def determine_full_version():
789 """Returns the full version from the source root."""
790 # Full version is something like 'R79-12575.0.0'.
791 milestone_version = determine_milestone_version()
792 platform_version = determine_platform_version()
793 full_version = ('R%s-%s' % (milestone_version, platform_version))
794 return full_version