blob: c8aba63f40a095d2282309a7b658f2b4d6ab8b5a [file] [log] [blame]
Alex Kleineb77ffa2019-05-28 14:47:44 -06001# -*- coding: utf-8 -*-
2# Copyright 2019 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Package utility functionality."""
7
8from __future__ import print_function
9
Yaakov Shaul730814a2019-09-10 13:58:25 -060010import collections
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060011import fileinput
Alex Klein87531182019-08-12 15:23:37 -060012import functools
Yaakov Shaul395ae832019-09-09 14:45:32 -060013import json
Evan Hernandezb51f1522019-08-15 11:29:40 -060014import os
Michael Mortensenb70e8a82019-10-10 18:43:41 -060015import re
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -060016import sys
Alex Klein87531182019-08-12 15:23:37 -060017
Alex Klein149fd3b2019-12-16 16:01:05 -070018import six
19
Andrew Lamb2bde9e42019-11-04 13:24:09 -070020from google.protobuf import json_format
Yaakov Shaul730814a2019-09-10 13:58:25 -060021
Andrew Lamb2bde9e42019-11-04 13:24:09 -070022from chromite.api.gen.config import replication_config_pb2
Michael Mortensen9fdb14b2019-10-17 11:17:30 -060023from chromite.cbuildbot import manifest_version
Alex Kleineb77ffa2019-05-28 14:47:44 -060024from chromite.lib import constants
Evan Hernandezb51f1522019-08-15 11:29:40 -060025from chromite.lib import cros_build_lib
Alex Klein4de25e82019-08-05 15:58:39 -060026from chromite.lib import cros_logging as logging
Alex Kleineb77ffa2019-05-28 14:47:44 -060027from chromite.lib import git
Michael Mortensenb70e8a82019-10-10 18:43:41 -060028from chromite.lib import osutils
Alex Kleineb77ffa2019-05-28 14:47:44 -060029from chromite.lib import portage_util
Andrew Lamb2bde9e42019-11-04 13:24:09 -070030from chromite.lib import replication_lib
Alex Kleind6195b62019-08-06 16:01:16 -060031from chromite.lib import uprev_lib
Alex Kleineb77ffa2019-05-28 14:47:44 -060032
Alex Klein36b117f2019-09-30 15:13:46 -060033if cros_build_lib.IsInsideChroot():
34 from chromite.service import dependency
35
Alex Klein87531182019-08-12 15:23:37 -060036# Registered handlers for uprevving versioned packages.
37_UPREV_FUNCS = {}
38
Alex Kleineb77ffa2019-05-28 14:47:44 -060039
40class Error(Exception):
41 """Module's base error class."""
42
43
Alex Klein4de25e82019-08-05 15:58:39 -060044class UnknownPackageError(Error):
45 """Uprev attempted for a package without a registered handler."""
46
47
Alex Kleineb77ffa2019-05-28 14:47:44 -060048class UprevError(Error):
49 """An error occurred while uprevving packages."""
50
51
Michael Mortensenb70e8a82019-10-10 18:43:41 -060052class NoAndroidVersionError(Error):
53 """An error occurred while trying to determine the android version."""
54
55
56class NoAndroidBranchError(Error):
57 """An error occurred while trying to determine the android branch."""
58
59
60class NoAndroidTargetError(Error):
61 """An error occurred while trying to determine the android target."""
62
63
Alex Klein4de25e82019-08-05 15:58:39 -060064class AndroidIsPinnedUprevError(UprevError):
65 """Raised when we try to uprev while Android is pinned."""
66
67 def __init__(self, new_android_atom):
68 """Initialize a AndroidIsPinnedUprevError.
69
70 Args:
71 new_android_atom: The Android atom that we failed to
72 uprev to, due to Android being pinned.
73 """
74 assert new_android_atom
75 msg = ('Failed up uprev to Android version %s as Android was pinned.' %
76 new_android_atom)
77 super(AndroidIsPinnedUprevError, self).__init__(msg)
78 self.new_android_atom = new_android_atom
Alex Klein87531182019-08-12 15:23:37 -060079
80
Yaakov Shaul1eafe832019-09-10 16:50:26 -060081class EbuildManifestError(Error):
82 """Error when running ebuild manifest."""
83
84
Andrew Lamb9563a152019-12-04 11:42:18 -070085class GeneratedCrosConfigFilesError(Error):
86 """Error when cros_config_schema does not produce expected files"""
87
88 def __init__(self, expected_files, found_files):
89 msg = ('Expected to find generated C files: %s. Actually found: %s' %
90 (expected_files, found_files))
91 super(GeneratedCrosConfigFilesError, self).__init__(msg)
92
Alex Klein7a3a7dd2020-01-08 16:44:38 -070093
Yaakov Shaul730814a2019-09-10 13:58:25 -060094UprevVersionedPackageModifications = collections.namedtuple(
95 'UprevVersionedPackageModifications', ('new_version', 'files'))
Alex Klein34afcbc2019-08-22 16:14:31 -060096
Yaakov Shaul730814a2019-09-10 13:58:25 -060097
98class UprevVersionedPackageResult(object):
99 """Data object for uprev_versioned_package."""
100
101 def __init__(self):
102 self.modified = []
103
104 def add_result(self, new_version, modified_files):
105 """Adds version/ebuilds tuple to result.
106
107 Args:
108 new_version: New version number of package.
109 modified_files: List of files modified for the given version.
110 """
111 result = UprevVersionedPackageModifications(new_version, modified_files)
112 self.modified.append(result)
113 return self
Alex Klein34afcbc2019-08-22 16:14:31 -0600114
115 @property
116 def uprevved(self):
Yaakov Shaul730814a2019-09-10 13:58:25 -0600117 return bool(self.modified)
Alex Klein34afcbc2019-08-22 16:14:31 -0600118
119
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600120def patch_ebuild_vars(ebuild_path, variables):
121 """Updates variables in ebuild.
122
123 Use this function rather than portage_util.EBuild.UpdateEBuild when you
124 want to preserve the variable position and quotes within the ebuild.
125
126 Args:
127 ebuild_path: The path of the ebuild.
128 variables: Dictionary of variables to update in ebuild.
129 """
130 try:
131 for line in fileinput.input(ebuild_path, inplace=1):
132 varname, eq, _ = line.partition('=')
133 if eq == '=' and varname.strip() in variables:
134 value = variables[varname]
135 sys.stdout.write('%s="%s"\n' % (varname, value))
136 else:
137 sys.stdout.write(line)
138 finally:
139 fileinput.close()
140
141
Alex Klein87531182019-08-12 15:23:37 -0600142def uprevs_versioned_package(package):
143 """Decorator to register package uprev handlers."""
144 assert package
145
146 def register(func):
147 """Registers |func| as a handler for |package|."""
148 _UPREV_FUNCS[package] = func
149
150 @functools.wraps(func)
151 def pass_through(*args, **kwargs):
152 return func(*args, **kwargs)
153
154 return pass_through
155
156 return register
157
158
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700159def uprev_android(tracking_branch,
160 android_package,
161 android_build_branch,
162 chroot,
163 build_targets=None,
164 android_version=None,
Alex Klein4de25e82019-08-05 15:58:39 -0600165 android_gts_build_branch=None):
166 """Returns the portage atom for the revved Android ebuild - see man emerge."""
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700167 command = [
168 'cros_mark_android_as_stable',
169 '--tracking_branch=%s' % tracking_branch,
170 '--android_package=%s' % android_package,
171 '--android_build_branch=%s' % android_build_branch,
172 ]
Alex Klein4de25e82019-08-05 15:58:39 -0600173 if build_targets:
174 command.append('--boards=%s' % ':'.join(bt.name for bt in build_targets))
175 if android_version:
176 command.append('--force_version=%s' % android_version)
177 if android_gts_build_branch:
178 command.append('--android_gts_build_branch=%s' % android_gts_build_branch)
179
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700180 result = cros_build_lib.run(
181 command,
182 stdout=True,
183 enter_chroot=True,
Mike Frysinger88d96362020-02-14 19:05:45 -0500184 encoding='utf-8',
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700185 chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600186
Mike Frysinger88d96362020-02-14 19:05:45 -0500187 portage_atom_string = result.stdout.strip()
188 android_atom = None
189 if portage_atom_string:
190 android_atom = portage_atom_string.splitlines()[-1].partition('=')[-1]
Alex Klein4de25e82019-08-05 15:58:39 -0600191 if not android_atom:
192 logging.info('Found nothing to rev.')
193 return None
194
195 for target in build_targets or []:
196 # Sanity check: We should always be able to merge the version of
197 # Android we just unmasked.
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700198 command = ['emerge-%s' % target.name, '-p', '--quiet', '=%s' % android_atom]
Alex Klein4de25e82019-08-05 15:58:39 -0600199 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700200 cros_build_lib.run(
201 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
Alex Klein4de25e82019-08-05 15:58:39 -0600202 except cros_build_lib.RunCommandError:
203 logging.error(
204 'Cannot emerge-%s =%s\nIs Android pinned to an older '
205 'version?', target, android_atom)
206 raise AndroidIsPinnedUprevError(android_atom)
207
208 return android_atom
209
210
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700211def uprev_build_targets(build_targets,
212 overlay_type,
213 chroot=None,
Alex Kleineb77ffa2019-05-28 14:47:44 -0600214 output_dir=None):
215 """Uprev the set provided build targets, or all if not specified.
216
217 Args:
218 build_targets (list[build_target_util.BuildTarget]|None): The build targets
219 whose overlays should be uprevved, empty or None for all.
220 overlay_type (str): One of the valid overlay types except None (see
221 constants.VALID_OVERLAYS).
222 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
223 output_dir (str|None): The path to optionally dump result files.
224 """
225 # Need a valid overlay, but exclude None.
226 assert overlay_type and overlay_type in constants.VALID_OVERLAYS
227
228 if build_targets:
229 overlays = portage_util.FindOverlaysForBoards(
230 overlay_type, boards=[t.name for t in build_targets])
231 else:
232 overlays = portage_util.FindOverlays(overlay_type)
233
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700234 return uprev_overlays(
235 overlays,
236 build_targets=build_targets,
237 chroot=chroot,
238 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600239
240
241def uprev_overlays(overlays, build_targets=None, chroot=None, output_dir=None):
242 """Uprev the given overlays.
243
244 Args:
245 overlays (list[str]): The list of overlay paths.
246 build_targets (list[build_target_util.BuildTarget]|None): The build targets
247 to clean in |chroot|, if desired. No effect unless |chroot| is provided.
248 chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
249 output_dir (str|None): The path to optionally dump result files.
250
251 Returns:
252 list[str] - The paths to all of the modified ebuild files. This includes the
253 new files that were added (i.e. the new versions) and all of the removed
254 files (i.e. the old versions).
255 """
256 assert overlays
257
258 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
259
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700260 uprev_manager = uprev_lib.UprevOverlayManager(
261 overlays,
262 manifest,
263 build_targets=build_targets,
264 chroot=chroot,
265 output_dir=output_dir)
Alex Kleineb77ffa2019-05-28 14:47:44 -0600266 uprev_manager.uprev()
267
268 return uprev_manager.modified_ebuilds
269
270
Alex Klein87531182019-08-12 15:23:37 -0600271def uprev_versioned_package(package, build_targets, refs, chroot):
272 """Call registered uprev handler function for the package.
273
274 Args:
275 package (portage_util.CPV): The package being uprevved.
276 build_targets (list[build_target_util.BuildTarget]): The build targets to
277 clean on a successful uprev.
278 refs (list[uprev_lib.GitRef]):
279 chroot (chroot_lib.Chroot): The chroot to enter for cleaning.
280
281 Returns:
Alex Klein34afcbc2019-08-22 16:14:31 -0600282 UprevVersionedPackageResult: The result.
Alex Klein87531182019-08-12 15:23:37 -0600283 """
284 assert package
285
286 if package.cp not in _UPREV_FUNCS:
287 raise UnknownPackageError(
288 'Package "%s" does not have a registered handler.' % package.cp)
289
Andrew Lambea9a8a22019-12-12 14:03:43 -0700290 return _UPREV_FUNCS[package.cp](build_targets, refs, chroot)
Alex Klein87531182019-08-12 15:23:37 -0600291
292
Evan Hernandezb51f1522019-08-15 11:29:40 -0600293# TODO(evanhernandez): Remove this. Only a quick hack for testing.
294@uprevs_versioned_package('sample/sample')
295def uprev_sample(*_args, **_kwargs):
296 """Mimics an uprev by changing files in sandbox repos.
297
298 See: uprev_versioned_package.
299 """
300 paths = [
301 os.path.join(constants.SOURCE_ROOT, 'infra/dummies', repo, 'sample.txt')
302 for repo in ('general-sandbox', 'merge-sandbox')
303 ]
304
Yaakov Shaul730814a2019-09-10 13:58:25 -0600305 return UprevVersionedPackageResult().add_result('1.2.3', paths)
Evan Hernandezb51f1522019-08-15 11:29:40 -0600306
307
Yaakov Shaul395ae832019-09-09 14:45:32 -0600308@uprevs_versioned_package('afdo/kernel-profiles')
309def uprev_kernel_afdo(*_args, **_kwargs):
David Burger92485342019-09-10 17:52:45 -0600310 """Updates kernel ebuilds with versions from kernel_afdo.json.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600311
312 See: uprev_versioned_package.
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600313
314 Raises:
315 EbuildManifestError: When ebuild manifest does not complete successfuly.
Yaakov Shaul395ae832019-09-09 14:45:32 -0600316 """
317 path = os.path.join(constants.SOURCE_ROOT, 'src', 'third_party',
318 'toolchain-utils', 'afdo_metadata', 'kernel_afdo.json')
319
David Burger92485342019-09-10 17:52:45 -0600320 with open(path, 'r') as f:
321 versions = json.load(f)
Yaakov Shaul395ae832019-09-09 14:45:32 -0600322
Yaakov Shaul730814a2019-09-10 13:58:25 -0600323 result = UprevVersionedPackageResult()
Yaakov Shaul395ae832019-09-09 14:45:32 -0600324 for version, version_info in versions.items():
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600325 path = os.path.join('src', 'third_party', 'chromiumos-overlay',
326 'sys-kernel', version)
327 ebuild_path = os.path.join(constants.SOURCE_ROOT, path,
328 '%s-9999.ebuild' % version)
Yaakov Shaula187b152019-09-11 12:41:32 -0600329 chroot_ebuild_path = os.path.join(constants.CHROOT_SOURCE_ROOT, path,
330 '%s-9999.ebuild' % version)
Yaakov Shaul730814a2019-09-10 13:58:25 -0600331 afdo_profile_version = version_info['name']
Yaakov Shaulcb1cfc32019-09-16 13:51:19 -0600332 patch_ebuild_vars(ebuild_path,
333 dict(AFDO_PROFILE_VERSION=afdo_profile_version))
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600334
335 try:
Yaakov Shaul730814a2019-09-10 13:58:25 -0600336 cmd = ['ebuild', chroot_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400337 cros_build_lib.run(cmd, enter_chroot=True)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600338 except cros_build_lib.RunCommandError as e:
339 raise EbuildManifestError(
340 'Error encountered when regenerating the manifest for ebuild: %s\n%s'
Yaakov Shaula187b152019-09-11 12:41:32 -0600341 % (chroot_ebuild_path, e), e)
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600342
Yaakov Shauldd8b4112019-09-11 11:44:03 -0600343 manifest_path = os.path.join(constants.SOURCE_ROOT, path, 'Manifest')
Yaakov Shaul1eafe832019-09-10 16:50:26 -0600344
Yaakov Shaul730814a2019-09-10 13:58:25 -0600345 result.add_result(afdo_profile_version, [ebuild_path, manifest_path])
346
347 return result
Yaakov Shaul395ae832019-09-09 14:45:32 -0600348
349
Trent Begin315d9d92019-12-03 21:55:53 -0700350@uprevs_versioned_package('chromeos-base/chromeos-dtc-vm')
Trent Begin6daa8702020-01-29 14:58:12 -0700351def uprev_sludge(build_targets, refs, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700352 """Updates sludge VM - chromeos-base/chromeos-dtc-vm.
353
354 See: uprev_versioned_package.
355 """
Trent Begin6daa8702020-01-29 14:58:12 -0700356 # Unused by uprev_sludge
357 del build_targets, refs
358
Trent Begin315d9d92019-12-03 21:55:53 -0700359 package = 'chromeos-dtc-vm'
Trent Begind943df92020-02-25 10:30:10 -0700360 package_path = os.path.join('src', 'private-overlays',
361 'project-wilco-private', 'chromeos-base', package)
362 version_pin_path = os.path.join(package_path, 'VERSION-PIN')
Trent Begin315d9d92019-12-03 21:55:53 -0700363
Trent Begin6daa8702020-01-29 14:58:12 -0700364 return uprev_ebuild_from_pin(package_path, version_pin_path, chroot)
Trent Begin315d9d92019-12-03 21:55:53 -0700365
366
Trent Begin6daa8702020-01-29 14:58:12 -0700367def uprev_ebuild_from_pin(package_path, version_pin_path, chroot):
Trent Begin315d9d92019-12-03 21:55:53 -0700368 """Changes the package ebuild's version to match the version pin file.
369
370 Args:
371 package_path: The path of the package relative to the src root. This path
372 should contain a single ebuild with the same name as the package.
Trent Begind943df92020-02-25 10:30:10 -0700373 version_pin_path: The path of the version_pin file that contains only a
374 version string. The ebuild's version will be directly set to this
Trent Begin315d9d92019-12-03 21:55:53 -0700375 number.
Trent Begin6daa8702020-01-29 14:58:12 -0700376 chroot (chroot_lib.Chroot): specify a chroot to enter.
Trent Begin315d9d92019-12-03 21:55:53 -0700377
378 Returns:
379 UprevVersionedPackageResult: The result.
380 """
381 package = os.path.basename(package_path)
Trent Begind943df92020-02-25 10:30:10 -0700382
383 package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
384 ebuild_paths = list(portage_util.EBuild.List(package_src_path))
Trent Begin315d9d92019-12-03 21:55:53 -0700385 if not ebuild_paths:
386 raise UprevError('No ebuilds found for %s' % package)
387 elif len(ebuild_paths) > 1:
388 raise UprevError('Multiple ebuilds found for %s' % package)
389 else:
390 ebuild_path = ebuild_paths[0]
391
Trent Begind943df92020-02-25 10:30:10 -0700392 version_pin_src_path = os.path.join(constants.SOURCE_ROOT, version_pin_path)
393 version = osutils.ReadFile(version_pin_src_path).strip()
Trent Begin315d9d92019-12-03 21:55:53 -0700394 new_ebuild_path = os.path.join(package_path,
395 '%s-%s-r1.ebuild' % (package, version))
Trent Begind943df92020-02-25 10:30:10 -0700396 new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
397 os.rename(ebuild_path, new_ebuild_src_path)
398 new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
399 new_ebuild_path)
Trent Begin315d9d92019-12-03 21:55:53 -0700400
Trent Begin6daa8702020-01-29 14:58:12 -0700401 try:
Trent Begind943df92020-02-25 10:30:10 -0700402 portage_util.UpdateEbuildManifest(new_ebuild_chroot_path, chroot=chroot)
Trent Begin6daa8702020-01-29 14:58:12 -0700403 except cros_build_lib.RunCommandError as e:
Trent Begind943df92020-02-25 10:30:10 -0700404 raise EbuildManifestError(
Trent Begin6daa8702020-01-29 14:58:12 -0700405 'Unable to update manifest for %s: %s' % (package, e.stderr))
406
Trent Begin315d9d92019-12-03 21:55:53 -0700407 result = UprevVersionedPackageResult()
Trent Begind943df92020-02-25 10:30:10 -0700408 result.add_result(version, [new_ebuild_src_path, ebuild_path])
Trent Begin315d9d92019-12-03 21:55:53 -0700409 return result
410
411
Alex Klein87531182019-08-12 15:23:37 -0600412@uprevs_versioned_package(constants.CHROME_CP)
Andrew Lambea9a8a22019-12-12 14:03:43 -0700413def uprev_chrome(build_targets, refs, chroot):
Alex Klein87531182019-08-12 15:23:37 -0600414 """Uprev chrome and its related packages.
415
416 See: uprev_versioned_package.
417 """
418 # Determine the version from the refs (tags), i.e. the chrome versions are the
419 # tag names.
420 chrome_version = uprev_lib.get_chrome_version_from_refs(refs)
421
422 uprev_manager = uprev_lib.UprevChromeManager(
423 chrome_version, build_targets=build_targets, chroot=chroot)
David Burger37f48672019-09-18 17:07:56 -0600424 result = UprevVersionedPackageResult()
Alex Klein87531182019-08-12 15:23:37 -0600425 # Start with chrome itself, as we can't do anything else unless chrome
426 # uprevs successfully.
427 if not uprev_manager.uprev(constants.CHROME_CP):
David Burger37f48672019-09-18 17:07:56 -0600428 return result
Alex Klein87531182019-08-12 15:23:37 -0600429
430 # With a successful chrome rev, also uprev related packages.
431 for package in constants.OTHER_CHROME_PACKAGES:
432 uprev_manager.uprev(package)
433
David Burger37f48672019-09-18 17:07:56 -0600434 return result.add_result(chrome_version, uprev_manager.modified_ebuilds)
Alex Klein87531182019-08-12 15:23:37 -0600435
436
Andrew Lamb9563a152019-12-04 11:42:18 -0700437def _generate_platform_c_files(replication_config, chroot):
438 """Generates platform C files from a platform JSON payload.
439
440 Args:
441 replication_config (replication_config_pb2.ReplicationConfig): A
442 ReplicationConfig that has already been run. If it produced a
443 build_config.json file, that file will be used to generate platform C
444 files. Otherwise, nothing will be generated.
445 chroot (chroot_lib.Chroot): The chroot to use to generate.
446
447 Returns:
448 A list of generated files.
449 """
450 # Generate the platform C files from the build config. Note that it would be
451 # more intuitive to generate the platform C files from the platform config;
452 # however, cros_config_schema does not allow this, because the platform config
453 # payload is not always valid input. For example, if a property is both
454 # 'required' and 'build-only', it will fail schema validation. Thus, use the
455 # build config, and use '-f' to filter.
456 build_config_path = [
457 rule.destination_path
458 for rule in replication_config.file_replication_rules
459 if rule.destination_path.endswith('build_config.json')
460 ]
461
462 if not build_config_path:
463 logging.info(
Alex Kleinad6b48a2020-01-08 16:57:41 -0700464 'No build_config.json found, will not generate platform C files. '
465 'Replication config: %s', replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700466 return []
467
468 if len(build_config_path) > 1:
Alex Kleinad6b48a2020-01-08 16:57:41 -0700469 raise ValueError('Expected at most one build_config.json destination path. '
470 'Replication config: %s' % replication_config)
Andrew Lamb9563a152019-12-04 11:42:18 -0700471
472 build_config_path = build_config_path[0]
473
474 # Paths to the build_config.json and dir to output C files to, in the
475 # chroot.
476 build_config_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
477 build_config_path)
478 generated_output_chroot_dir = os.path.join(constants.CHROOT_SOURCE_ROOT,
479 os.path.dirname(build_config_path))
480
481 command = [
482 'cros_config_schema', '-m', build_config_chroot_path, '-g',
483 generated_output_chroot_dir, '-f', '"TRUE"'
484 ]
485
486 cros_build_lib.run(
487 command, enter_chroot=True, chroot_args=chroot.get_enter_args())
488
489 # A relative (to the source root) path to the generated C files.
490 generated_output_dir = os.path.dirname(build_config_path)
491 generated_files = []
492 expected_c_files = ['config.c', 'ec_config.c', 'ec_config.h']
493 for f in expected_c_files:
494 if os.path.exists(
495 os.path.join(constants.SOURCE_ROOT, generated_output_dir, f)):
496 generated_files.append(os.path.join(generated_output_dir, f))
497
498 if len(expected_c_files) != len(generated_files):
499 raise GeneratedCrosConfigFilesError(expected_c_files, generated_files)
500
501 return generated_files
502
503
Andrew Lambe836f222019-12-09 12:27:38 -0700504def _get_private_overlay_package_root(ref, package):
505 """Returns the absolute path to the root of a given private overlay.
506
507 Args:
508 ref (uprev_lib.GitRef): GitRef for the private overlay.
509 package (str): Path to the package in the overlay.
510 """
511 # There might be a cleaner way to map from package -> path within the source
512 # tree. For now, just use string patterns.
Andrew Lamb4aa09912020-01-08 13:55:56 -0700513 private_overlay_ref_pattern = r'/chromeos\/overlays\/overlay-([\w-]+)-private'
Andrew Lambe836f222019-12-09 12:27:38 -0700514 match = re.match(private_overlay_ref_pattern, ref.path)
515 if not match:
516 raise ValueError('ref.path must match the pattern: %s. Actual ref: %s' %
517 (private_overlay_ref_pattern, ref))
518
519 overlay = match.group(1)
520
521 return os.path.join(constants.SOURCE_ROOT,
522 'src/private-overlays/overlay-%s-private' % overlay,
523 package)
524
525
Andrew Lambea9a8a22019-12-12 14:03:43 -0700526@uprevs_versioned_package('chromeos-base/chromeos-config-bsp')
527def replicate_private_config(_build_targets, refs, chroot):
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700528 """Replicate a private cros_config change to the corresponding public config.
529
Alex Kleinad6b48a2020-01-08 16:57:41 -0700530 See uprev_versioned_package for args
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700531 """
Andrew Lambea9a8a22019-12-12 14:03:43 -0700532 package = 'chromeos-base/chromeos-config-bsp'
533
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700534 if len(refs) != 1:
535 raise ValueError('Expected exactly one ref, actual %s' % refs)
536
537 # Expect a replication_config.jsonpb in the package root.
Andrew Lambe836f222019-12-09 12:27:38 -0700538 package_root = _get_private_overlay_package_root(refs[0], package)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700539 replication_config_path = os.path.join(package_root,
540 'replication_config.jsonpb')
541
542 try:
543 replication_config = json_format.Parse(
544 osutils.ReadFile(replication_config_path),
545 replication_config_pb2.ReplicationConfig())
546 except IOError:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700547 raise ValueError(
548 'Expected ReplicationConfig missing at %s' % replication_config_path)
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700549
550 replication_lib.Replicate(replication_config)
551
552 modified_files = [
553 rule.destination_path
554 for rule in replication_config.file_replication_rules
555 ]
556
Andrew Lamb9563a152019-12-04 11:42:18 -0700557 # The generated platform C files are not easily filtered by replication rules,
558 # i.e. JSON / proto filtering can be described by a FieldMask, arbitrary C
559 # files cannot. Therefore, replicate and filter the JSON payloads, and then
560 # generate filtered C files from the JSON payload.
561 modified_files.extend(_generate_platform_c_files(replication_config, chroot))
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700562
563 # Use the private repo's commit hash as the new version.
564 new_private_version = refs[0].revision
565
Andrew Lamb988f4da2019-12-10 10:16:43 -0700566 # modified_files should contain only relative paths at this point, but the
567 # returned UprevVersionedPackageResult must contain only absolute paths.
568 for i, modified_file in enumerate(modified_files):
569 assert not os.path.isabs(modified_file)
570 modified_files[i] = os.path.join(constants.SOURCE_ROOT, modified_file)
571
Andrew Lamb2bde9e42019-11-04 13:24:09 -0700572 return UprevVersionedPackageResult().add_result(new_private_version,
573 modified_files)
574
575
Alex Kleinbbef2b32019-08-27 10:38:50 -0600576def get_best_visible(atom, build_target=None):
577 """Returns the best visible CPV for the given atom.
578
579 Args:
580 atom (str): The atom to look up.
581 build_target (build_target_util.BuildTarget): The build target whose
Alex Kleinda39c6d2019-09-16 14:36:36 -0600582 sysroot should be searched, or the SDK if not provided.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700583
584 Returns:
585 portage_util.CPV|None: The best visible package.
Alex Kleinbbef2b32019-08-27 10:38:50 -0600586 """
David Burger1e0fe232019-07-01 14:52:07 -0600587 assert atom
Alex Kleinbbef2b32019-08-27 10:38:50 -0600588
589 board = build_target.name if build_target else None
590 return portage_util.PortageqBestVisible(atom, board=board)
Alex Kleinda39c6d2019-09-16 14:36:36 -0600591
592
Alex Klein149fd3b2019-12-16 16:01:05 -0700593def has_prebuilt(atom, build_target=None, useflags=None):
Alex Kleinda39c6d2019-09-16 14:36:36 -0600594 """Check if a prebuilt exists.
595
596 Args:
597 atom (str): The package whose prebuilt is being queried.
598 build_target (build_target_util.BuildTarget): The build target whose
599 sysroot should be searched, or the SDK if not provided.
Alex Klein149fd3b2019-12-16 16:01:05 -0700600 useflags: Any additional USE flags that should be set. May be a string
601 of properly formatted USE flags, or an iterable of individual flags.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700602
603 Returns:
604 bool: True iff there is an available prebuilt, False otherwise.
Alex Kleinda39c6d2019-09-16 14:36:36 -0600605 """
606 assert atom
607
608 board = build_target.name if build_target else None
Alex Klein149fd3b2019-12-16 16:01:05 -0700609 extra_env = None
610 if useflags:
611 new_flags = useflags
612 if not isinstance(useflags, six.string_types):
613 new_flags = ' '.join(useflags)
614
615 existing = os.environ.get('USE', '')
616 final_flags = '%s %s' % (existing, new_flags)
617 extra_env = {'USE': final_flags.strip()}
618 return portage_util.HasPrebuilt(atom, board=board, extra_env=extra_env)
Alex Klein36b117f2019-09-30 15:13:46 -0600619
620
David Burger0f9dd4e2019-10-08 12:33:42 -0600621def builds(atom, build_target, packages=None):
Alex Klein36b117f2019-09-30 15:13:46 -0600622 """Check if |build_target| builds |atom| (has it in its depgraph)."""
623 cros_build_lib.AssertInsideChroot()
624
Chris McDonalda22b74f2019-11-22 13:55:06 -0700625 graph, _sdk_graph = dependency.GetBuildDependency(build_target.name, packages)
Alex Klein36b117f2019-09-30 15:13:46 -0600626 return any(atom in package for package in graph['package_deps'])
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600627
628
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600629def determine_chrome_version(build_target):
Michael Mortensenc2615b72019-10-15 08:12:24 -0600630 """Returns the current Chrome version for the board (or in buildroot).
631
632 Args:
Michael Mortensenb51a1f02019-10-16 13:28:20 -0600633 build_target (build_target_util.BuildTarget): The board build target.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700634
635 Returns:
636 str|None: The chrome version if available.
Michael Mortensenc2615b72019-10-15 08:12:24 -0600637 """
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600638 # TODO(crbug/1019770): Long term we should not need the try/catch here once
639 # the builds function above only returns True for chrome when
640 # determine_chrome_version will succeed.
641 try:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700642 cpv = portage_util.PortageqBestVisible(
643 constants.CHROME_CP, build_target.name, cwd=constants.SOURCE_ROOT)
Michael Mortensen9fe740c2019-10-29 14:42:48 -0600644 except cros_build_lib.RunCommandError as e:
645 # Return None because portage failed when trying to determine the chrome
646 # version.
647 logging.warning('Caught exception in determine_chrome_package: %s', e)
648 return None
Michael Mortensenc2615b72019-10-15 08:12:24 -0600649 # Something like 78.0.3877.4_rc -> 78.0.3877.4
650 return cpv.version_no_rev.partition('_')[0]
651
652
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600653def determine_android_package(board):
654 """Returns the active Android container package in use by the board.
655
656 Args:
657 board: The board name this is specific to.
Alex Kleinad6b48a2020-01-08 16:57:41 -0700658
659 Returns:
660 str|None: The android package string if there is one.
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600661 """
Michael Mortensene0f4b542019-10-24 15:30:23 -0600662 try:
663 packages = portage_util.GetPackageDependencies(board, 'virtual/target-os')
Michael Mortensene0f4b542019-10-24 15:30:23 -0600664 except cros_build_lib.RunCommandError as e:
665 # Return None because a command (likely portage) failed when trying to
666 # determine the package.
667 logging.warning('Caught exception in determine_android_package: %s', e)
668 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600669
Alex Kleinad6b48a2020-01-08 16:57:41 -0700670 # We assume there is only one Android package in the depgraph.
671 for package in packages:
672 if package.startswith('chromeos-base/android-container-') or \
673 package.startswith('chromeos-base/android-vm-'):
674 return package
675 return None
676
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600677
678def determine_android_version(boards=None):
679 """Determine the current Android version in buildroot now and return it.
680
681 This uses the typical portage logic to determine which version of Android
682 is active right now in the buildroot.
683
684 Args:
685 boards: List of boards to check version of.
686
687 Returns:
688 The Android build ID of the container for the boards.
689
690 Raises:
691 NoAndroidVersionError: if no unique Android version can be determined.
692 """
693 if not boards:
694 return None
695 # Verify that all boards have the same version.
696 version = None
697 for board in boards:
698 package = determine_android_package(board)
699 if not package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600700 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600701 cpv = portage_util.SplitCPV(package)
702 if not cpv:
703 raise NoAndroidVersionError(
704 'Android version could not be determined for %s' % board)
705 if not version:
706 version = cpv.version_no_rev
707 elif version != cpv.version_no_rev:
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700708 raise NoAndroidVersionError('Different Android versions (%s vs %s) for %s'
709 % (version, cpv.version_no_rev, boards))
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600710 return version
711
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700712
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600713def determine_android_branch(board):
714 """Returns the Android branch in use by the active container ebuild."""
715 try:
716 android_package = determine_android_package(board)
717 except cros_build_lib.RunCommandError:
718 raise NoAndroidBranchError(
719 'Android branch could not be determined for %s' % board)
720 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600721 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600722 ebuild_path = portage_util.FindEbuildForBoardPackage(android_package, board)
723 # We assume all targets pull from the same branch and that we always
724 # have an ARM_TARGET, ARM_USERDEBUG_TARGET, or an X86_USERDEBUG_TARGET.
725 targets = ['ARM_TARGET', 'ARM_USERDEBUG_TARGET', 'X86_USERDEBUG_TARGET']
726 ebuild_content = osutils.SourceEnvironment(ebuild_path, targets)
727 for target in targets:
728 if target in ebuild_content:
729 branch = re.search(r'(.*?)-linux-', ebuild_content[target])
730 if branch is not None:
731 return branch.group(1)
732 raise NoAndroidBranchError(
733 'Android branch could not be determined for %s (ebuild empty?)' % board)
734
735
736def determine_android_target(board):
Michael Mortensen14960d02019-10-18 07:53:59 -0600737 """Returns the Android target in use by the active container ebuild."""
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600738 try:
739 android_package = determine_android_package(board)
740 except cros_build_lib.RunCommandError:
741 raise NoAndroidTargetError(
742 'Android Target could not be determined for %s' % board)
743 if not android_package:
Michael Mortensenedf76532019-10-16 14:22:37 -0600744 return None
Michael Mortensenb70e8a82019-10-10 18:43:41 -0600745 if android_package.startswith('chromeos-base/android-vm-'):
746 return 'bertha'
747 elif android_package.startswith('chromeos-base/android-container-'):
748 return 'cheets'
749
750 raise NoAndroidTargetError(
751 'Android Target cannot be determined for the package: %s' %
752 android_package)
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600753
754
755def determine_platform_version():
756 """Returns the platform version from the source root."""
Michael Mortensen009cb662019-10-21 11:38:43 -0600757 # Platform version is something like '12575.0.0'.
Michael Mortensen9fdb14b2019-10-17 11:17:30 -0600758 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
759 return version.VersionString()
Michael Mortensen009cb662019-10-21 11:38:43 -0600760
761
762def determine_milestone_version():
763 """Returns the platform version from the source root."""
764 # Milestone version is something like '79'.
765 version = manifest_version.VersionInfo.from_repo(constants.SOURCE_ROOT)
766 return version.chrome_branch
767
Alex Klein7a3a7dd2020-01-08 16:44:38 -0700768
Michael Mortensen009cb662019-10-21 11:38:43 -0600769def determine_full_version():
770 """Returns the full version from the source root."""
771 # Full version is something like 'R79-12575.0.0'.
772 milestone_version = determine_milestone_version()
773 platform_version = determine_platform_version()
774 full_version = ('R%s-%s' % (milestone_version, platform_version))
775 return full_version