blob: c24f889283c9645bcceb8fd372935ae03293baed [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Mike Frysingerbafb3182020-02-21 03:15:43 -050036
Alex Klein87531182019-08-12 15:23:37 -060037# Registered handlers for uprevving versioned packages.
38_UPREV_FUNCS = {}
39
Alex Kleineb77ffa2019-05-28 14:47:44 -060040
41class Error(Exception):
42 """Module's base error class."""
43
44
Alex Klein4de25e82019-08-05 15:58:39 -060045class UnknownPackageError(Error):
46 """Uprev attempted for a package without a registered handler."""
47
48
Alex Kleineb77ffa2019-05-28 14:47:44 -060049class UprevError(Error):
50 """An error occurred while uprevving packages."""
51
52
Michael Mortensenb70e8a82019-10-10 18:43:41 -060053class NoAndroidVersionError(Error):
54 """An error occurred while trying to determine the android version."""
55
56
57class NoAndroidBranchError(Error):
58 """An error occurred while trying to determine the android branch."""
59
60
61class NoAndroidTargetError(Error):
62 """An error occurred while trying to determine the android target."""
63
64
Alex Klein4de25e82019-08-05 15:58:39 -060065class AndroidIsPinnedUprevError(UprevError):
66 """Raised when we try to uprev while Android is pinned."""
67
68 def __init__(self, new_android_atom):
69 """Initialize a AndroidIsPinnedUprevError.
70
71 Args:
72 new_android_atom: The Android atom that we failed to
73 uprev to, due to Android being pinned.
74 """
75 assert new_android_atom
76 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
77 new_android_atom)
78 super(AndroidIsPinnedUprevError, self).__init__(msg)
79 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060080
81
Yaakov Shaul1eafe832019-09-10 16:50:26 -060082class EbuildManifestError(Error):
83 """Error when running ebuild manifest."""
84
85
Andrew Lamb9563a152019-12-04 11:42:18 -070086class GeneratedCrosConfigFilesError(Error):
87 """Error when cros_config_schema does not produce expected files"""
88
89 def __init__(self, expected_files, found_files):
90 msg = ('Expected to find generated C files: %s. Actually found: %s' %
91 (expected_files, found_files))
92 super(GeneratedCrosConfigFilesError, self).__init__(msg)
93
Alex Klein7a3a7dd2020-01-08 16:44:38 -070094
Yaakov Shaul730814a2019-09-10 13:58:25 -060095UprevVersionedPackageModifications = collections.namedtuple(
96 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -060097
Yaakov Shaul730814a2019-09-10 13:58:25 -060098
99class UprevVersionedPackageResult(object):
100 """Data object for uprev_versioned_package."""
101
102 def __init__(self):
103 self.modified = []
104
105 def add_result(self, new_version, modified_files):
106 """Adds version/ebuilds tuple to result.
107
108 Args:
109 new_version: New version number of package.
110 modified_files: List of files modified for the given version.
111 """
112 result = UprevVersionedPackageModifications(new_version, modified_files)
113 self.modified.append(result)
114 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600115
116 @property
117 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600118 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600119
120
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600121def patch_ebuild_vars(ebuild_path, variables):
122 """Updates variables in ebuild.
123
124 Use this function rather than portage_util.EBuild.UpdateEBuild when you
125 want to preserve the variable position and quotes within the ebuild.
126
127 Args:
128 ebuild_path: The path of the ebuild.
129 variables: Dictionary of variables to update in ebuild.
130 """
131 try:
132 for line in fileinput.input(ebuild_path, inplace=1):
133 varname, eq, _ = line.partition('=')
134 if eq == '=' and varname.strip() in variables:
135 value = variables[varname]
136 sys.stdout.write('%s="%s"\n' % (varname, value))
137 else:
138 sys.stdout.write(line)
139 finally:
140 fileinput.close()
141
142
Alex Klein87531182019-08-12 15:23:37 -0600143def uprevs_versioned_package(package):
144 """Decorator to register package uprev handlers."""
145 assert package
146
147 def register(func):
148 """Registers |func| as a handler for |package|."""
149 _UPREV_FUNCS[package] = func
150
151 @functools.wraps(func)
152 def pass_through(*args, **kwargs):
153 return func(*args, **kwargs)
154
155 return pass_through
156
157 return register
158
159
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700160def uprev_android(tracking_branch,
161 android_package,
162 android_build_branch,
163 chroot,
164 build_targets=None,
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +0900165 android_version=None):
Alex Klein4de25e82019-08-05 15:58:39 -0600166 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700167 command = [
168 'cros_mark_android_as_stable',
169 '--tracking_branch=%s' % tracking_branch,
170 '--android_package=%s' % android_package,
171 '--android_build_branch=%s' % android_build_branch,
172 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600173 if build_targets:
174 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
175 if android_version:
176 command.append('--force_version=%s' % android_version)
Alex Klein4de25e82019-08-05 15:58:39 -0600177
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700178 result = cros_build_lib.run(
179 command,
180 stdout=True,
181 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500182 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700183 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600184
Mike Frysinger88d96362020-02-14 19:05:45 -0500185 portage_atom_string = result.stdout.strip()
186 android_atom = None
187 if portage_atom_string:
188 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600189 if not android_atom:
190 logging.info('Found nothing to rev.')
191 return None
192
193 for target in build_targets or []:
194 # Sanity check: We should always be able to merge the version of
195 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700196 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600197 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700198 cros_build_lib.run(
199 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600200 except cros_build_lib.RunCommandError:
201 logging.error(
202 'Cannot emerge-%s =%s\nIs Android pinned to an older '
203 'version?', target, android_atom)
204 raise AndroidIsPinnedUprevError(android_atom)
205
206 return android_atom
207
208
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700209def uprev_build_targets(build_targets,
210 overlay_type,
211 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600212 output_dir=None):
213 """Uprev the set provided build targets, or all if not specified.
214
215 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600216 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600217 whose overlays should be uprevved, empty or None for all.
218 overlay_type (str): One of the valid overlay types except None (see
219 constants.VALID_OVERLAYS).
220 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
221 output_dir (str|None): The path to optionally dump result files.
222 """
223 # Need a valid overlay, but exclude None.
224 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
225
226 if build_targets:
227 overlays = portage_util.FindOverlaysForBoards(
228 overlay_type, boards=[t.name for t in build_targets])
229 else:
230 overlays = portage_util.FindOverlays(overlay_type)
231
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700232 return uprev_overlays(
233 overlays,
234 build_targets=build_targets,
235 chroot=chroot,
236 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600237
238
239def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
240 """Uprev the given overlays.
241
242 Args:
243 overlays (list[str]): The list of overlay paths.
Alex Klein2960c752020-03-09 13:43:38 -0600244 build_targets (list[build_target_lib.BuildTarget]|None): The build targets
Alex Kleineb77ffa2019-05-28 14:47:44 -0600245 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
246 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
247 output_dir (str|None): The path to optionally dump result files.
248
249 Returns:
250 list[str] - The paths to all of the modified ebuild files. This includes the
251 new files that were added (i.e. the new versions) and all of the removed
252 files (i.e. the old versions).
253 """
254 assert overlays
255
256 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
257
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700258 uprev_manager = uprev_lib.UprevOverlayManager(
259 overlays,
260 manifest,
261 build_targets=build_targets,
262 chroot=chroot,
263 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600264 uprev_manager.uprev()
265
266 return uprev_manager.modified_ebuilds
267
268
Alex Klein87531182019-08-12 15:23:37 -0600269def uprev_versioned_package(package, build_targets, refs, chroot):
270 """Call registered uprev handler function for the package.
271
272 Args:
273 package (portage_util.CPV): The package being uprevved.
Alex Klein2960c752020-03-09 13:43:38 -0600274 build_targets (list[build_target_lib.BuildTarget]): The build targets to
Alex Klein87531182019-08-12 15:23:37 -0600275 clean on a successful uprev.
276 refs (list[uprev_lib.GitRef]):
277 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
278
279 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600280 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600281 """
282 assert package
283
284 if package.cp not in _UPREV_FUNCS:
285 raise UnknownPackageError(
286 'Package "%s" does not have a registered handler.' % package.cp)
287
Andrew Lambea9a8a22019-12-12 14:03:43 -0700288 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600289
290
Yaakov Shaul395ae832019-09-09 14:45:32 -0600291@uprevs_versioned_package('afdo/kernel-profiles')
292def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600293 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600294
295 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600296
297 Raises:
298 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600299 """
300 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
301 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
302
David Burger92485342019-09-10 17:52:45 -0600303 with open(path, 'r') as f:
304 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600305
Yaakov Shaul730814a2019-09-10 13:58:25 -0600306 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600307 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600308 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
309 'sys-kernel', version)
310 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
311 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600312 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
313 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600314 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600315 patch_ebuild_vars(ebuild_path,
316 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600317
318 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600319 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400320 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600321 except cros_build_lib.RunCommandError as e:
322 raise EbuildManifestError(
323 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600324 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600325
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600326 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600327
Yaakov Shaul730814a2019-09-10 13:58:25 -0600328 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
329
330 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600331
332
Trent Beginaf51f1b2020-03-09 17:35:31 -0600333@uprevs_versioned_package('chromeos-base/termina-image-amd64')
334def uprev_termina_amd64(_build_targets, _refs, chroot):
335 """Updates termina amd64 VM - chromeos-base/termina-image-amd64.
336
337 See: uprev_versioned_package.
338 """
339 return uprev_termina('termina-image-amd64', chroot)
340
341
342@uprevs_versioned_package('chromeos-base/termina-image-arm')
343def uprev_termina_arm(_build_targets, _refs, chroot):
344 """Updates termina arm VM - chromeos-base/termina-image-arm.
345
346 See: uprev_versioned_package.
347 """
348 return uprev_termina('termina-image-arm', chroot)
349
350
351def uprev_termina(package, chroot):
352 """Helper function to uprev termina VM.
353
354 Args:
355 package (string): name of the package
356 chroot (chroot_lib.Chroot): specify a chroot to enter.
357
358 Returns:
359 UprevVersionedPackageResult: The result.
360 """
361 package_path = os.path.join(constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos-base',
362 package)
363 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
364 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
365
366
Trent Begin315d9d92019-12-03 21:55:53 -0700367@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Beginaf51f1b2020-03-09 17:35:31 -0600368def uprev_sludge(_build_targets, _refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700369 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
370
371 See: uprev_versioned_package.
372 """
373 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700374 package_path = os.path.join('src', 'private-overlays',
375 'project-wilco-private', 'chromeos-base', package)
376 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700377
Trent Begin6daa8702020-01-29 14:58:12 -0700378 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700379
380
Trent Begin6daa8702020-01-29 14:58:12 -0700381def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700382 """Changes the package ebuild's version to match the version pin file.
383
384 Args:
385 package_path: The path of the package relative to the src root. This path
386 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700387 version_pin_path: The path of the version_pin file that contains only a
388 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700389 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700390 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700391
392 Returns:
393 UprevVersionedPackageResult: The result.
394 """
395 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700396
397 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
398 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700399 if not ebuild_paths:
400 raise UprevError('No ebuilds found for %s' % package)
401 elif len(ebuild_paths) > 1:
402 raise UprevError('Multiple ebuilds found for %s' % package)
403 else:
404 ebuild_path = ebuild_paths[0]
405
Trent Begind943df92020-02-25 10:30:10 -0700406 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
407 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700408 new_ebuild_path = os.path.join(package_path,
409 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700410 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
411 os.rename(ebuild_path, new_ebuild_src_path)
Trent Begin4a11a632020-02-28 12:59:58 -0700412 manifest_src_path = os.path.join(package_src_path, 'Manifest')
Trent Begind943df92020-02-25 10:30:10 -0700413 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
414 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700415
Trent Begin6daa8702020-01-29 14:58:12 -0700416 try:
Trent Begind943df92020-02-25 10:30:10 -0700417 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700418 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700419 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700420 'Unable to update manifest for %s: %s' % (package, e.stderr))
421
Trent Begin315d9d92019-12-03 21:55:53 -0700422 result = UprevVersionedPackageResult()
Trent Begin4a11a632020-02-28 12:59:58 -0700423 result.add_result(version,
424 [new_ebuild_src_path, ebuild_path, manifest_src_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700425 return result
426
427
Alex Klein87531182019-08-12 15:23:37 -0600428@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700429def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600430 """Uprev chrome and its related packages.
431
432 See: uprev_versioned_package.
433 """
434 # Determine the version from the refs (tags), i.e. the chrome versions are the
435 # tag names.
436 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
437
438 uprev_manager = uprev_lib.UprevChromeManager(
439 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600440 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600441 # Start with chrome itself, as we can't do anything else unless chrome
442 # uprevs successfully.
443 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600444 return result
Alex Klein87531182019-08-12 15:23:37 -0600445
446 # With a successful chrome rev, also uprev related packages.
447 for package in constants.OTHER_CHROME_PACKAGES:
448 uprev_manager.uprev(package)
449
David Burger37f48672019-09-18 17:07:56 -0600450 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600451
452
Andrew Lamb9563a152019-12-04 11:42:18 -0700453def _generate_platform_c_files(replication_config, chroot):
454 """Generates platform C files from a platform JSON payload.
455
456 Args:
457 replication_config (replication_config_pb2.ReplicationConfig): A
458 ReplicationConfig that has already been run. If it produced a
459 build_config.json file, that file will be used to generate platform C
460 files. Otherwise, nothing will be generated.
461 chroot (chroot_lib.Chroot): The chroot to use to generate.
462
463 Returns:
464 A list of generated files.
465 """
466 # Generate the platform C files from the build config. Note that it would be
467 # more intuitive to generate the platform C files from the platform config;
468 # however, cros_config_schema does not allow this, because the platform config
469 # payload is not always valid input. For example, if a property is both
470 # 'required' and 'build-only', it will fail schema validation. Thus, use the
471 # build config, and use '-f' to filter.
472 build_config_path = [
473 rule.destination_path
474 for rule in replication_config.file_replication_rules
475 if rule.destination_path.endswith('build_config.json')
476 ]
477
478 if not build_config_path:
479 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700480 'No build_config.json found, will not generate platform C files. '
481 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700482 return []
483
484 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700485 raise ValueError('Expected at most one build_config.json destination path. '
486 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700487
488 build_config_path = build_config_path[0]
489
490 # Paths to the build_config.json and dir to output C files to, in the
491 # chroot.
492 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
493 build_config_path)
494 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
495 os.path.dirname(build_config_path))
496
497 command = [
498 'cros_config_schema', '-m', build_config_chroot_path, '-g',
499 generated_output_chroot_dir, '-f', '"TRUE"'
500 ]
501
502 cros_build_lib.run(
503 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
504
505 # A relative (to the source root) path to the generated C files.
506 generated_output_dir = os.path.dirname(build_config_path)
507 generated_files = []
508 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
509 for f in expected_c_files:
510 if os.path.exists(
511 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
512 generated_files.append(os.path.join(generated_output_dir, f))
513
514 if len(expected_c_files) != len(generated_files):
515 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
516
517 return generated_files
518
519
Andrew Lambe836f222019-12-09 12:27:38 -0700520def _get_private_overlay_package_root(ref, package):
521 """Returns the absolute path to the root of a given private overlay.
522
523 Args:
524 ref (uprev_lib.GitRef): GitRef for the private overlay.
525 package (str): Path to the package in the overlay.
526 """
527 # There might be a cleaner way to map from package -> path within the source
528 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700529 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700530 match = re.match(private_overlay_ref_pattern, ref.path)
531 if not match:
532 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
533 (private_overlay_ref_pattern, ref))
534
535 overlay = match.group(1)
536
537 return os.path.join(constants.SOURCE_ROOT,
538 'src/private-overlays/overlay-%s-private' % overlay,
539 package)
540
541
Andrew Lambea9a8a22019-12-12 14:03:43 -0700542@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
543def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700544 """Replicate a private cros_config change to the corresponding public config.
545
Alex Kleinad6b48a2020-01-08 16:57:41 -0700546 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700547 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700548 package = 'chromeos-base/chromeos-config-bsp'
549
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700550 if len(refs) != 1:
551 raise ValueError('Expected exactly one ref, actual %s' % refs)
552
553 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700554 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700555 replication_config_path = os.path.join(package_root,
556 'replication_config.jsonpb')
557
558 try:
559 replication_config = json_format.Parse(
560 osutils.ReadFile(replication_config_path),
561 replication_config_pb2.ReplicationConfig())
562 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700563 raise ValueError(
564 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700565
566 replication_lib.Replicate(replication_config)
567
568 modified_files = [
569 rule.destination_path
570 for rule in replication_config.file_replication_rules
571 ]
572
Andrew Lamb9563a152019-12-04 11:42:18 -0700573 # The generated platform C files are not easily filtered by replication rules,
574 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
575 # files cannot. Therefore, replicate and filter the JSON payloads, and then
576 # generate filtered C files from the JSON payload.
577 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700578
579 # Use the private repo's commit hash as the new version.
580 new_private_version = refs[0].revision
581
Andrew Lamb988f4da2019-12-10 10:16:43 -0700582 # modified_files should contain only relative paths at this point, but the
583 # returned UprevVersionedPackageResult must contain only absolute paths.
584 for i, modified_file in enumerate(modified_files):
585 assert not os.path.isabs(modified_file)
586 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
587
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700588 return UprevVersionedPackageResult().add_result(new_private_version,
589 modified_files)
590
591
Alex Kleinbbef2b32019-08-27 10:38:50 -0600592def get_best_visible(atom, build_target=None):
593 """Returns the best visible CPV for the given atom.
594
595 Args:
596 atom (str): The atom to look up.
Alex Klein2960c752020-03-09 13:43:38 -0600597 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600598 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700599
600 Returns:
601 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600602 """
David Burger1e0fe232019-07-01 14:52:07 -0600603 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600604
605 board = build_target.name if build_target else None
606 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600607
608
Alex Klein149fd3b2019-12-16 16:01:05 -0700609def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600610 """Check if a prebuilt exists.
611
612 Args:
613 atom (str): The package whose prebuilt is being queried.
Alex Klein2960c752020-03-09 13:43:38 -0600614 build_target (build_target_lib.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600615 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700616 useflags: Any additional USE flags that should be set. May be a string
617 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700618
619 Returns:
620 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600621 """
622 assert atom
623
624 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700625 extra_env = None
626 if useflags:
627 new_flags = useflags
628 if not isinstance(useflags, six.string_types):
629 new_flags = ' '.join(useflags)
630
631 existing = os.environ.get('USE', '')
632 final_flags = '%s %s' % (existing, new_flags)
633 extra_env = {'USE': final_flags.strip()}
634 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600635
636
David Burger0f9dd4e2019-10-08 12:33:42 -0600637def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600638 """Check if |build_target| builds |atom| (has it in its depgraph)."""
639 cros_build_lib.AssertInsideChroot()
640
Chris McDonalda22b74f2019-11-22 13:55:06 -0700641 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600642 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600643
644
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600645def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600646 """Returns the current Chrome version for the board (or in buildroot).
647
648 Args:
Alex Klein2960c752020-03-09 13:43:38 -0600649 build_target (build_target_lib.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700650
651 Returns:
652 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600653 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600654 # TODO(crbug/1019770): Long term we should not need the try/catch here once
655 # the builds function above only returns True for chrome when
656 # determine_chrome_version will succeed.
657 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700658 cpv = portage_util.PortageqBestVisible(
659 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600660 except cros_build_lib.RunCommandError as e:
661 # Return None because portage failed when trying to determine the chrome
662 # version.
663 logging.warning('Caught exception in determine_chrome_package: %s', e)
664 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600665 # Something like 78.0.3877.4_rc -> 78.0.3877.4
666 return cpv.version_no_rev.partition('_')[0]
667
668
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600669def determine_android_package(board):
670 """Returns the active Android container package in use by the board.
671
672 Args:
673 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700674
675 Returns:
676 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600677 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600678 try:
679 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600680 except cros_build_lib.RunCommandError as e:
681 # Return None because a command (likely portage) failed when trying to
682 # determine the package.
683 logging.warning('Caught exception in determine_android_package: %s', e)
684 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600685
Alex Kleinad6b48a2020-01-08 16:57:41 -0700686 # We assume there is only one Android package in the depgraph.
687 for package in packages:
688 if package.startswith('chromeos-base/android-container-') or \
689 package.startswith('chromeos-base/android-vm-'):
690 return package
691 return None
692
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600693
694def determine_android_version(boards=None):
695 """Determine the current Android version in buildroot now and return it.
696
697 This uses the typical portage logic to determine which version of Android
698 is active right now in the buildroot.
699
700 Args:
701 boards: List of boards to check version of.
702
703 Returns:
704 The Android build ID of the container for the boards.
705
706 Raises:
707 NoAndroidVersionError: if no unique Android version can be determined.
708 """
709 if not boards:
710 return None
711 # Verify that all boards have the same version.
712 version = None
713 for board in boards:
714 package = determine_android_package(board)
715 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600716 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600717 cpv = portage_util.SplitCPV(package)
718 if not cpv:
719 raise NoAndroidVersionError(
720 'Android version could not be determined for %s' % board)
721 if not version:
722 version = cpv.version_no_rev
723 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700724 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
725 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600726 return version
727
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700728
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600729def determine_android_branch(board):
730 """Returns the Android branch in use by the active container ebuild."""
731 try:
732 android_package = determine_android_package(board)
733 except cros_build_lib.RunCommandError:
734 raise NoAndroidBranchError(
735 'Android branch could not be determined for %s' % board)
736 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600737 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600738 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
739 # We assume all targets pull from the same branch and that we always
740 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
741 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
742 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
743 for target in targets:
744 if target in ebuild_content:
745 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
746 if branch is not None:
747 return branch.group(1)
748 raise NoAndroidBranchError(
749 'Android branch could not be determined for %s (ebuild empty?)' % board)
750
751
752def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600753 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600754 try:
755 android_package = determine_android_package(board)
756 except cros_build_lib.RunCommandError:
757 raise NoAndroidTargetError(
758 'Android Target could not be determined for %s' % board)
759 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600760 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600761 if android_package.startswith('chromeos-base/android-vm-'):
762 return 'bertha'
763 elif android_package.startswith('chromeos-base/android-container-'):
764 return 'cheets'
765
766 raise NoAndroidTargetError(
767 'Android Target cannot be determined for the package: %s' %
768 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600769
770
771def determine_platform_version():
772 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600773 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600774 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
775 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600776
777
778def determine_milestone_version():
779 """Returns the platform version from the source root."""
780 # Milestone version is something like '79'.
781 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
782 return version.chrome_branch
783
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700784
Michael Mortensen009cb662019-10-21 11:38:43 -0600785def determine_full_version():
786 """Returns the full version from the source root."""
787 # Full version is something like 'R79-12575.0.0'.
788 milestone_version = determine_milestone_version()
789 platform_version = determine_platform_version()
790 full_version = ('R%s-%s' % (milestone_version, platform_version))
791 return full_version