blob: f75fabe19e9def0427b5a844a38e427821bfce86 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
David Rileyc0da9d92016-02-01 12:11:01 -08002# Copyright 2016 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This module uprevs Android for cbuildbot.
7
8After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A
9caller could then use this atom with emerge to build the newly uprevved version
10of Android e.g.
11
Shuhei Takahashi6d02c192017-04-05 14:01:24 +090012./cros_mark_android_as_stable \
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090013 --android_build_branch=git_pi-arc \
14 --android_package=android-container-pi
Shuhei Takahashi6d02c192017-04-05 14:01:24 +090015
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090016Returns chromeos-base/android-container-pi-6417892-r1
David Rileyc0da9d92016-02-01 12:11:01 -080017
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090018emerge-eve =chromeos-base/android-container-pi-6417892-r1
David Rileyc0da9d92016-02-01 12:11:01 -080019"""
20
21from __future__ import print_function
22
23import filecmp
24import glob
Junichi Uekawad21f94d2020-07-27 15:50:05 +090025import json
David Rileyc0da9d92016-02-01 12:11:01 -080026import os
Hidehiko Abe12727dd2016-05-27 23:23:45 +090027import re
khmel@google.com96c193e2018-05-10 14:00:38 -070028import time
Mike Frysinger00a02292020-04-19 06:28:03 -040029import sys
David Rileyc0da9d92016-02-01 12:11:01 -080030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
David Rileyc0da9d92016-02-01 12:11:01 -080032from chromite.lib import commandline
33from chromite.lib import cros_build_lib
34from chromite.lib import cros_logging as logging
35from chromite.lib import git
36from chromite.lib import gs
Shao-Chuan Lee301a4192021-02-08 11:53:49 +090037from chromite.lib import osutils
David Rileyc0da9d92016-02-01 12:11:01 -080038from chromite.lib import portage_util
39from chromite.scripts import cros_mark_as_stable
40
41
Mike Frysinger00a02292020-04-19 06:28:03 -040042assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
43
44
David Rileyc0da9d92016-02-01 12:11:01 -080045# Dir where all the action happens.
46_OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/'
47
Junichi Uekawa6d61ab02020-04-15 14:52:28 +090048_GIT_COMMIT_MESSAGE = """Marking latest for %(android_package)s ebuild with \
49version %(android_version)s as stable.
50
51BUG=None
52TEST=CQ
53"""
David Rileyc0da9d92016-02-01 12:11:01 -080054
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -070055_RUNTIME_ARTIFACTS_BUCKET_URL = 'gs://chromeos-arc-images/runtime_artifacts'
David Rileyc0da9d92016-02-01 12:11:01 -080056
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090057def IsBuildIdValid(bucket_url, build_branch, build_id, targets):
David Rileyc0da9d92016-02-01 12:11:01 -080058 """Checks that a specific build_id is valid.
59
60 Looks for that build_id for all builds. Confirms that the subpath can
61 be found and that the zip file is present in that subdirectory.
62
63 Args:
64 bucket_url: URL of Android build gs bucket
65 build_branch: branch of Android builds
66 build_id: A string. The Android build id number to check.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090067 targets: Dict from build key to (targe build suffix, artifact file pattern)
68 pair.
David Rileyc0da9d92016-02-01 12:11:01 -080069
70 Returns:
71 Returns subpaths dictionary if build_id is valid.
72 None if the build_id is not valid.
73 """
74 gs_context = gs.GSContext()
75 subpaths_dict = {}
Mike Frysinger0bdbc102019-06-13 15:27:29 -040076 for build, (target, _) in targets.items():
David Rileyc0da9d92016-02-01 12:11:01 -080077 build_dir = '%s-%s' % (build_branch, target)
78 build_id_path = os.path.join(bucket_url, build_dir, build_id)
79
80 # Find name of subpath.
81 try:
82 subpaths = gs_context.List(build_id_path)
83 except gs.GSNoSuchKey:
Mike Frysinger968c1142020-05-09 00:37:56 -040084 logging.warning(
David Rileyc0da9d92016-02-01 12:11:01 -080085 'Directory [%s] does not contain any subpath, ignoring it.',
86 build_id_path)
87 return None
88 if len(subpaths) > 1:
Mike Frysinger968c1142020-05-09 00:37:56 -040089 logging.warning(
David Rileyc0da9d92016-02-01 12:11:01 -080090 'Directory [%s] contains more than one subpath, ignoring it.',
91 build_id_path)
92 return None
93
94 subpath_dir = subpaths[0].url.rstrip('/')
95 subpath_name = os.path.basename(subpath_dir)
96
97 # Look for a zipfile ending in the build_id number.
98 try:
Hidehiko Abe12727dd2016-05-27 23:23:45 +090099 gs_context.List(subpath_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800100 except gs.GSNoSuchKey:
Mike Frysinger968c1142020-05-09 00:37:56 -0400101 logging.warning(
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900102 'Did not find a file for build id [%s] in directory [%s].',
David Rileyc0da9d92016-02-01 12:11:01 -0800103 build_id, subpath_dir)
104 return None
105
106 # Record subpath for the build.
107 subpaths_dict[build] = subpath_name
108
109 # If we got here, it means we found an appropriate build for all platforms.
110 return subpaths_dict
111
112
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900113def GetLatestBuild(bucket_url, build_branch, targets):
David Rileyc0da9d92016-02-01 12:11:01 -0800114 """Searches the gs bucket for the latest green build.
115
116 Args:
117 bucket_url: URL of Android build gs bucket
118 build_branch: branch of Android builds
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900119 targets: Dict from build key to (targe build suffix, artifact file pattern)
120 pair.
David Rileyc0da9d92016-02-01 12:11:01 -0800121
122 Returns:
123 Tuple of (latest version string, subpaths dictionary)
124 If no latest build can be found, returns None, None
125 """
126 gs_context = gs.GSContext()
127 common_build_ids = None
128 # Find builds for each target.
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400129 for target, _ in targets.values():
David Rileyc0da9d92016-02-01 12:11:01 -0800130 build_dir = '-'.join((build_branch, target))
131 base_path = os.path.join(bucket_url, build_dir)
132 build_ids = []
133 for gs_result in gs_context.List(base_path):
134 # Remove trailing slashes and get the base name, which is the build_id.
135 build_id = os.path.basename(gs_result.url.rstrip('/'))
136 if not build_id.isdigit():
Mike Frysinger968c1142020-05-09 00:37:56 -0400137 logging.warning('Directory [%s] does not look like a valid build_id.',
138 gs_result.url)
David Rileyc0da9d92016-02-01 12:11:01 -0800139 continue
140 build_ids.append(build_id)
141
142 # Update current list of builds.
143 if common_build_ids is None:
144 # First run, populate it with the first platform.
145 common_build_ids = set(build_ids)
146 else:
147 # Already populated, find the ones that are common.
148 common_build_ids.intersection_update(build_ids)
149
150 if common_build_ids is None:
Mike Frysinger968c1142020-05-09 00:37:56 -0400151 logging.warning('Did not find a build_id common to all platforms.')
David Rileyc0da9d92016-02-01 12:11:01 -0800152 return None, None
153
154 # Otherwise, find the most recent one that is valid.
155 for build_id in sorted(common_build_ids, key=int, reverse=True):
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900156 subpaths = IsBuildIdValid(bucket_url, build_branch, build_id, targets)
David Rileyc0da9d92016-02-01 12:11:01 -0800157 if subpaths:
158 return build_id, subpaths
159
160 # If not found, no build_id is valid.
Mike Frysinger968c1142020-05-09 00:37:56 -0400161 logging.warning('Did not find a build_id valid on all platforms.')
David Rileyc0da9d92016-02-01 12:11:01 -0800162 return None, None
163
164
165def FindAndroidCandidates(package_dir):
166 """Return a tuple of Android's unstable ebuild and stable ebuilds.
167
168 Args:
169 package_dir: The path to where the package ebuild is stored.
170
171 Returns:
172 Tuple [unstable_ebuild, stable_ebuilds].
173
174 Raises:
175 Exception: if no unstable ebuild exists for Android.
176 """
177 stable_ebuilds = []
178 unstable_ebuilds = []
179 for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
180 ebuild = portage_util.EBuild(path)
181 if ebuild.version == '9999':
182 unstable_ebuilds.append(ebuild)
183 else:
184 stable_ebuilds.append(ebuild)
185
186 # Apply some sanity checks.
187 if not unstable_ebuilds:
188 raise Exception('Missing 9999 ebuild for %s' % package_dir)
189 if not stable_ebuilds:
Lann Martinffb95162018-08-28 12:02:54 -0600190 logging.warning('Missing stable ebuild for %s', package_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800191
192 return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
193
194
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800195def _GetArcBasename(build, basename):
196 """Tweaks filenames between Android bucket and ARC++ bucket.
197
198 Android builders create build artifacts with the same name for -user and
199 -userdebug builds, which breaks the android-container ebuild (b/33072485).
200 When copying the artifacts from the Android bucket to the ARC++ bucket some
201 artifacts will be renamed from the usual pattern
202 *cheets_${ARCH}-target_files-S{VERSION}.zip to
203 cheets_${BUILD_NAME}-target_files-S{VERSION}.zip which will typically look
204 like cheets_(${LABEL})*${ARCH}_userdebug-target_files-S{VERSION}.zip.
205
206 Args:
207 build: the build being mirrored, e.g. 'X86', 'ARM', 'X86_USERDEBUG'.
208 basename: the basename of the artifact to copy.
209
210 Returns:
211 The basename of the destination.
212 """
213 if build not in constants.ARC_BUILDS_NEED_ARTIFACTS_RENAMED:
214 return basename
Yūki Ishiief1ada92018-03-27 15:46:15 +0900215 if basename in constants.ARC_ARTIFACTS_RENAME_NOT_NEEDED:
216 return basename
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800217 to_discard, sep, to_keep = basename.partition('-')
218 if not sep:
219 logging.error(('Build %s: Could not find separator "-" in artifact'
220 ' basename %s'), build, basename)
221 return basename
Bernie Thompson63ed5612017-08-16 12:27:34 -0700222 if 'cheets_' in to_discard:
223 return 'cheets_%s-%s' % (build.lower(), to_keep)
224 elif 'bertha_' in to_discard:
225 return 'bertha_%s-%s' % (build.lower(), to_keep)
226 logging.error('Build %s: Unexpected artifact basename %s',
227 build, basename)
228 return basename
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800229
230
David Riley73f00d92016-02-16 18:54:20 -0800231def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths,
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900232 targets, arc_bucket_url, acls):
David Riley73f00d92016-02-16 18:54:20 -0800233 """Copies from source Android bucket to ARC++ specific bucket.
234
235 Copies each build to the ARC bucket eliminating the subpath.
236 Applies build specific ACLs for each file.
237
238 Args:
239 android_bucket_url: URL of Android build gs bucket
240 build_branch: branch of Android builds
241 build_id: A string. The Android build id number to check.
242 subpaths: Subpath dictionary for each build to copy.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900243 targets: Dict from build key to (targe build suffix, artifact file pattern)
244 pair.
David Riley73f00d92016-02-16 18:54:20 -0800245 arc_bucket_url: URL of the target ARC build gs bucket
246 acls: ACLs dictionary for each build to copy.
247 """
248 gs_context = gs.GSContext()
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400249 for build, subpath in subpaths.items():
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900250 target, pattern = targets[build]
David Riley73f00d92016-02-16 18:54:20 -0800251 build_dir = '%s-%s' % (build_branch, target)
252 android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath)
253 arc_dir = os.path.join(arc_bucket_url, build_dir, build_id)
254
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900255 # Copy all target files from android_dir to arc_dir, setting ACLs.
256 for targetfile in gs_context.List(android_dir):
257 if re.search(pattern, targetfile.url):
258 basename = os.path.basename(targetfile.url)
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800259 arc_path = os.path.join(arc_dir, _GetArcBasename(build, basename))
David Riley73f00d92016-02-16 18:54:20 -0800260 acl = acls[build]
261 needs_copy = True
khmel@google.com96c193e2018-05-10 14:00:38 -0700262 retry_count = 2
David Riley73f00d92016-02-16 18:54:20 -0800263
khmel@google.com96c193e2018-05-10 14:00:38 -0700264 # Retry in case race condition when several boards trying to copy the
265 # same resource
266 while True:
267 # Check a pre-existing file with the original source.
268 if gs_context.Exists(arc_path):
269 if (gs_context.Stat(targetfile.url).hash_crc32c !=
270 gs_context.Stat(arc_path).hash_crc32c):
Mike Frysinger968c1142020-05-09 00:37:56 -0400271 logging.warning('Removing incorrect file %s', arc_path)
khmel@google.com96c193e2018-05-10 14:00:38 -0700272 gs_context.Remove(arc_path)
273 else:
274 logging.info('Skipping already copied file %s', arc_path)
275 needs_copy = False
David Riley73f00d92016-02-16 18:54:20 -0800276
khmel@google.com96c193e2018-05-10 14:00:38 -0700277 # Copy if necessary, and set the ACL unconditionally.
278 # The Stat() call above doesn't verify the ACL is correct and
279 # the ChangeACL should be relatively cheap compared to the copy.
280 # This covers the following caes:
281 # - handling an interrupted copy from a previous run.
282 # - rerunning the copy in case one of the googlestorage_acl_X.txt
283 # files changes (e.g. we add a new variant which reuses a build).
284 if needs_copy:
285 logging.info('Copying %s -> %s (acl %s)',
286 targetfile.url, arc_path, acl)
287 try:
288 gs_context.Copy(targetfile.url, arc_path, version=0)
289 except gs.GSContextPreconditionFailed as error:
290 if not retry_count:
291 raise error
292 # Retry one more time after a short delay
293 logging.warning('Will retry copying %s -> %s',
294 targetfile.url, arc_path)
295 time.sleep(5)
296 retry_count = retry_count - 1
297 continue
298 gs_context.ChangeACL(arc_path, acl_args_file=acl)
299 break
David Riley73f00d92016-02-16 18:54:20 -0800300
301
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900302def MirrorArtifacts(android_bucket_url, android_build_branch, arc_bucket_url,
303 acls, targets, version=None):
304 """Mirrors artifacts from Android bucket to ARC bucket.
305
306 First, this function identifies which build version should be copied,
307 if not given. Please see GetLatestBuild() and IsBuildIdValid() for details.
308
309 On build version identified, then copies target artifacts to the ARC bucket,
310 with setting ACLs.
311
312 Args:
313 android_bucket_url: URL of Android build gs bucket
314 android_build_branch: branch of Android builds
315 arc_bucket_url: URL of the target ARC build gs bucket
316 acls: ACLs dictionary for each build to copy.
317 targets: Dict from build key to (targe build suffix, artifact file pattern)
318 pair.
319 version: (optional) A string. The Android build id number to check.
320 If not passed, detect latest good build version.
321
322 Returns:
323 Mirrored version.
324 """
325 if version:
326 subpaths = IsBuildIdValid(
327 android_bucket_url, android_build_branch, version, targets)
328 if not subpaths:
Lann Martinffb95162018-08-28 12:02:54 -0600329 logging.error('Requested build %s is not valid', version)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900330 else:
331 version, subpaths = GetLatestBuild(
332 android_bucket_url, android_build_branch, targets)
333
334 CopyToArcBucket(android_bucket_url, android_build_branch, version, subpaths,
335 targets, arc_bucket_url, acls)
khmel@google.com778a1cd2018-04-13 11:11:58 -0700336
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900337 return version
338
339
David Riley73f00d92016-02-16 18:54:20 -0800340def MakeAclDict(package_dir):
341 """Creates a dictionary of acl files for each build type.
342
343 Args:
344 package_dir: The path to where the package acl files are stored.
345
346 Returns:
347 Returns acls dictionary.
348 """
349 return dict(
350 (k, os.path.join(package_dir, v))
351 for k, v in constants.ARC_BUCKET_ACLS.items()
352 )
353
354
Qijiang Fan6588cc92019-11-20 13:26:04 +0900355def MakeBuildTargetDict(package_name, build_branch):
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700356 """Creates a dictionary of build targets.
357
Bernie Thompson63ed5612017-08-16 12:27:34 -0700358 Not all targets are common between branches, for example
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700359 sdk_google_cheets_x86 only exists on N.
360 This generates a dictionary listing the available build targets for a
361 specific branch.
362
363 Args:
Qijiang Fan6588cc92019-11-20 13:26:04 +0900364 package_name: package name of chromeos arc package.
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700365 build_branch: branch of Android builds.
366
367 Returns:
368 Returns build target dictionary.
369
370 Raises:
371 ValueError: if the Android build branch is invalid.
372 """
Qijiang Fan6588cc92019-11-20 13:26:04 +0900373 if constants.ANDROID_CONTAINER_PACKAGE_KEYWORD in package_name:
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900374 target_list = {
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900375 constants.ANDROID_PI_BUILD_BRANCH:
376 constants.ANDROID_PI_BUILD_TARGETS,
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900377 }
Qijiang Fan6588cc92019-11-20 13:26:04 +0900378 elif constants.ANDROID_VM_PACKAGE_KEYWORD in package_name:
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900379 target_list = {
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900380 constants.ANDROID_VMMST_BUILD_BRANCH:
381 constants.ANDROID_VMMST_BUILD_TARGETS,
382 constants.ANDROID_VMRVC_BUILD_BRANCH:
383 constants.ANDROID_VMRVC_BUILD_TARGETS,
384 }
385 else:
386 raise ValueError('Unknown package: %s' % package_name)
387 target = target_list.get(build_branch)
388 if not target:
Qijiang Fan6588cc92019-11-20 13:26:04 +0900389 raise ValueError('Unknown branch: %s' % build_branch)
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900390 return target
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700391
392
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900393def PrintUprevMetadata(build_branch, stable_candidate, new_ebuild):
394 """Shows metadata on buildbot page at UprevAndroid step.
David Rileyc0da9d92016-02-01 12:11:01 -0800395
396 Args:
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900397 build_branch: The branch of Android builds.
398 stable_candidate: The existing stable ebuild.
399 new_ebuild: The newly written ebuild.
David Rileyc0da9d92016-02-01 12:11:01 -0800400 """
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900401 # Examples:
402 # "android-container-pi revved 6461825-r1 -> 6468247-r1"
403 # "android-container-pi revved 6461825-r1 -> 6461825-r2 (ebuild update only)"
404 msg = '%s revved %s -> %s' % (stable_candidate.pkgname,
405 stable_candidate.version,
406 new_ebuild.version)
407
408 old_android = stable_candidate.version_no_rev
409 new_android = new_ebuild.version_no_rev
410
411 if old_android == new_android:
412 msg += ' (ebuild update only)'
413 else:
414 ab_link = ('https://android-build.googleplex.com'
415 '/builds/%s/branches/%s/cls?end=%s'
416 % (new_android, build_branch, old_android))
417 logging.PrintBuildbotLink('Android changelog', ab_link)
418
419 logging.PrintBuildbotStepText(msg)
Junichi Uekawad21f94d2020-07-27 15:50:05 +0900420 logging.PrintKitchenSetBuildProperty('android_uprev', json.dumps({
421 'branch': build_branch,
422 'new': new_ebuild.version,
423 'old': stable_candidate.version,
424 'pkgname': stable_candidate.pkgname,
425 }))
David Rileyc0da9d92016-02-01 12:11:01 -0800426
427
Yury Khmelb009aeb2020-08-19 19:40:00 -0700428def FindDataCollectorArtifacts(gs_context,
429 android_version,
430 runtime_artifacts_bucket_url,
431 version_reference):
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700432 r"""Finds and includes into variables artifacts from arc.DataCollector.
433
Yury Khmelb009aeb2020-08-19 19:40:00 -0700434 This is used from UpdateDataCollectorArtifacts in order to check the
435 particular version.
436
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700437 Args:
Yury Khmelb009aeb2020-08-19 19:40:00 -0700438 gs_context: context to execute gsutil
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700439 android_version: The \d+ build id of Android.
440 runtime_artifacts_bucket_url: root of runtime artifacts
Yury Khmelb009aeb2020-08-19 19:40:00 -0700441 build_branch: build branch. Used to determine the pinned version if exists.
442 version_reference: which version to use as a reference. Could be '${PV}' in
443 case version of data collector artifacts matches the
444 Android version or direct version in case of override.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700445
446 Returns:
Yury Khmelb009aeb2020-08-19 19:40:00 -0700447 dictionary with filled ebuild variables. This dictionary is empty in case
448 no artificats are found.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700449 """
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700450 variables = {}
Yury Khmelb009aeb2020-08-19 19:40:00 -0700451
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700452 buckets = ['ureadahead_pack', 'gms_core_cache']
453 archs = ['arm', 'arm64', 'x86', 'x86_64']
454 build_types = ['user', 'userdebug']
455
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700456 for bucket in buckets:
457 for arch in archs:
458 for build_type in build_types:
459 path = (f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_'
Yury Khmele1b74402020-05-18 08:41:35 -0700460 f'{android_version}.tar')
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700461 if gs_context.Exists(path):
462 variables[(f'{arch}_{build_type}_{bucket}').upper()] = (
463 f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_'
464 f'{version_reference}.tar')
465
466 return variables
467
468
Yury Khmelb009aeb2020-08-19 19:40:00 -0700469def UpdateDataCollectorArtifacts(android_version,
470 runtime_artifacts_bucket_url,
471 build_branch):
472 r"""Finds and includes into variables artifacts from arc.DataCollector.
473
474 This verifies default android version. In case artificts are not found for
475 default Android version it tries to find artifacts for pinned version. If
476 pinned version is provided, it is required artifacts exist for the pinned
477 version.
478
479 Args:
480 android_version: The \d+ build id of Android.
481 runtime_artifacts_bucket_url: root of runtime artifacts
482 build_branch: build branch. Used to determine the pinned version if exists.
483
484 Returns:
485 dictionary with filled ebuild variables.
486 """
487
488 gs_context = gs.GSContext()
489 # Check the existing version. If we find any artifacts, use them.
490 variables = FindDataCollectorArtifacts(gs_context,
491 android_version,
492 runtime_artifacts_bucket_url,
493 '${PV}')
494 if variables:
495 # Data artificts were found.
496 return variables
497
498 # Check pinned version for the current branch.
499 pin_path = (f'{runtime_artifacts_bucket_url}/{build_branch}_pin_version')
500 if not gs_context.Exists(pin_path):
501 # No pinned version.
502 logging.warning(
503 'No data collector artifacts were found for %s',
504 android_version)
505 return variables
506
507 pin_version = gs_context.Cat(pin_path, encoding='utf-8').rstrip()
508 logging.info('Pinned version %s overrides %s',
509 pin_version, android_version)
510 variables = FindDataCollectorArtifacts(gs_context,
511 pin_version,
512 runtime_artifacts_bucket_url,
513 pin_version)
514 if not variables:
515 # If pin version set it must contain data.
516 raise Exception('Pinned version %s:%s does not contain artificats' % (
517 build_branch, pin_version))
518
519 return variables
520
521
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900522def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild,
523 android_package, android_version, package_dir,
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700524 build_branch, arc_bucket_url,
525 runtime_artifacts_bucket_url,
526 build_targets):
David Rileyc0da9d92016-02-01 12:11:01 -0800527 r"""Uprevs the Android ebuild.
528
529 This is the main function that uprevs from a stable candidate
530 to its new version.
531
532 Args:
533 stable_candidate: ebuild that corresponds to the stable ebuild we are
534 revving from. If None, builds the a new ebuild given the version
535 with revision set to 1.
536 unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900537 android_package: android package name.
David Rileyc0da9d92016-02-01 12:11:01 -0800538 android_version: The \d+ build id of Android.
David Rileyc0da9d92016-02-01 12:11:01 -0800539 package_dir: Path to the android-container package dir.
David Riley73f00d92016-02-16 18:54:20 -0800540 build_branch: branch of Android builds.
541 arc_bucket_url: URL of the target ARC build gs bucket.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700542 runtime_artifacts_bucket_url: root of runtime artifacts
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700543 build_targets: build targets for this particular Android branch.
David Rileyc0da9d92016-02-01 12:11:01 -0800544
545 Returns:
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900546 Tuple[str, List[str], List[str]] if revved, or None
547 1. Full portage version atom (including rc's, etc) that was revved.
548 2. List of files to be `git add`ed.
549 3. List of files to be `git rm`ed.
David Rileyc0da9d92016-02-01 12:11:01 -0800550 """
551 def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
552 """Returns True if the new ebuild is redundant.
553
554 This is True if there if the current stable ebuild is the exact same copy
555 of the new one.
556 """
557 if not stable_ebuild:
558 return False
559
David Riley676f5402016-02-12 17:24:23 -0800560 if stable_candidate.version_no_rev == new_ebuild.version_no_rev:
David Rileyc0da9d92016-02-01 12:11:01 -0800561 return filecmp.cmp(
562 new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700563 return False
David Rileyc0da9d92016-02-01 12:11:01 -0800564
565 # Case where we have the last stable candidate with same version just rev.
David Riley676f5402016-02-12 17:24:23 -0800566 if stable_candidate and stable_candidate.version_no_rev == android_version:
David Rileyc0da9d92016-02-01 12:11:01 -0800567 new_ebuild_path = '%s-r%d.ebuild' % (
568 stable_candidate.ebuild_path_no_revision,
569 stable_candidate.current_revision + 1)
570 else:
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900571 pf = '%s-%s-r1' % (android_package, android_version)
David Rileyc0da9d92016-02-01 12:11:01 -0800572 new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
573
David Riley73f00d92016-02-16 18:54:20 -0800574 variables = {'BASE_URL': arc_bucket_url}
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400575 for build, (target, _) in build_targets.items():
David Riley73f00d92016-02-16 18:54:20 -0800576 variables[build + '_TARGET'] = '%s-%s' % (build_branch, target)
David Rileyc0da9d92016-02-01 12:11:01 -0800577
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700578 variables.update(UpdateDataCollectorArtifacts(
Yury Khmelb009aeb2020-08-19 19:40:00 -0700579 android_version, runtime_artifacts_bucket_url, build_branch))
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700580
David Rileyc0da9d92016-02-01 12:11:01 -0800581 portage_util.EBuild.MarkAsStable(
582 unstable_ebuild.ebuild_path, new_ebuild_path,
583 variables, make_stable=True)
584 new_ebuild = portage_util.EBuild(new_ebuild_path)
585
586 # Determine whether this is ebuild is redundant.
587 if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
588 msg = 'Previous ebuild with same version found and ebuild is redundant.'
589 logging.info(msg)
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900590 logging.PrintBuildbotStepText('%s %s not revved'
591 % (stable_candidate.pkgname,
592 stable_candidate.version))
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900593 osutils.SafeUnlink(new_ebuild_path)
David Rileyc0da9d92016-02-01 12:11:01 -0800594 return None
595
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900596 # PFQ runs should always be able to find a stable candidate.
David Rileyc0da9d92016-02-01 12:11:01 -0800597 if stable_candidate:
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900598 PrintUprevMetadata(build_branch, stable_candidate, new_ebuild)
David Rileyc0da9d92016-02-01 12:11:01 -0800599
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900600 files_to_add = [new_ebuild_path]
601 files_to_remove = []
David Rileyc0da9d92016-02-01 12:11:01 -0800602 if stable_candidate and not stable_candidate.IsSticky():
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900603 osutils.SafeUnlink(stable_candidate.ebuild_path)
604 files_to_remove.append(stable_candidate.ebuild_path)
David Rileyc0da9d92016-02-01 12:11:01 -0800605
606 # Update ebuild manifest and git add it.
607 gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400608 cros_build_lib.run(gen_manifest_cmd, extra_env=None, print_cmd=True)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900609 files_to_add.append('Manifest')
David Rileyc0da9d92016-02-01 12:11:01 -0800610
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900611 return (
612 f'{new_ebuild.package}-{new_ebuild.version}',
613 files_to_add,
614 files_to_remove,
615 )
David Rileyc0da9d92016-02-01 12:11:01 -0800616
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900617
618def _PrepareGitBranch(tracking_branch, overlay_dir):
619 """Prepares a git branch for the uprev commit."""
620 tracking_branch = f'remotes/m/{os.path.basename(tracking_branch)}'
621 existing_branch = git.GetCurrentBranchOrId(overlay_dir)
622 work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
623 tracking_branch,
624 overlay_dir)
625 work_branch.CreateBranch()
626
627 # In the case of uprevving overlays that have patches applied to them,
628 # include the patched changes in the stabilizing branch.
629 if existing_branch:
630 git.RunGit(overlay_dir, ['rebase', existing_branch])
631
632
633def _CommitChange(message, android_package_dir, files_to_add, files_to_remove):
634 """Commit changes to git with list of files to add/remove."""
635 git.RunGit(android_package_dir, ['add', '--'] + files_to_add)
636 git.RunGit(android_package_dir, ['rm', '--'] + files_to_remove)
637
638 portage_util.EBuild.CommitChange(message, android_package_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800639
640
641def GetParser():
642 """Creates the argument parser."""
643 parser = commandline.ArgumentParser()
644 parser.add_argument('-b', '--boards')
645 parser.add_argument('--android_bucket_url',
David Riley73f00d92016-02-16 18:54:20 -0800646 default=constants.ANDROID_BUCKET_URL,
647 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800648 parser.add_argument('--android_build_branch',
Shuhei Takahashi6d02c192017-04-05 14:01:24 +0900649 required=True,
650 help='Android branch to import from. '
651 'Ex: git_mnc-dr-arc-dev')
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900652 parser.add_argument('--android_package',
653 default=constants.ANDROID_PACKAGE_NAME)
David Riley73f00d92016-02-16 18:54:20 -0800654 parser.add_argument('--arc_bucket_url',
655 default=constants.ARC_BUCKET_URL,
656 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800657 parser.add_argument('-f', '--force_version',
658 help='Android build id to use')
659 parser.add_argument('-s', '--srcroot',
660 default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
661 help='Path to the src directory')
662 parser.add_argument('-t', '--tracking_branch', default='cros/master',
663 help='Branch we are tracking changes against')
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700664 parser.add_argument('--runtime_artifacts_bucket_url',
665 default=_RUNTIME_ARTIFACTS_BUCKET_URL,
666 type='gs_path')
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900667 parser.add_argument('--skip_commit',
668 action='store_true',
669 help='Skip commiting uprev changes to git')
David Rileyc0da9d92016-02-01 12:11:01 -0800670 return parser
671
672
673def main(argv):
Hidehiko Abec9ecf262017-07-05 15:17:41 +0900674 logging.EnableBuildbotMarkers()
David Rileyc0da9d92016-02-01 12:11:01 -0800675 parser = GetParser()
676 options = parser.parse_args(argv)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900677 # HACK(b/179456416): Skip git operations if tracking_branch is empty, which
678 # happens when called from CQ builders.
679 # TODO(b/179456416): Make CQ builders specify the --skip_commit option.
680 if not options.tracking_branch:
681 options.skip_commit = True
David Rileyc0da9d92016-02-01 12:11:01 -0800682 options.Freeze()
683
684 overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900685 android_package_dir = os.path.join(
686 overlay_dir,
687 portage_util.GetFullAndroidPortagePackageName(options.android_package))
David Rileyc0da9d92016-02-01 12:11:01 -0800688 version_to_uprev = None
David Rileyc0da9d92016-02-01 12:11:01 -0800689
690 (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
David Riley73f00d92016-02-16 18:54:20 -0800691 acls = MakeAclDict(android_package_dir)
Qijiang Fan6588cc92019-11-20 13:26:04 +0900692 build_targets = MakeBuildTargetDict(options.android_package,
693 options.android_build_branch)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900694 # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt).
695 version_to_uprev = MirrorArtifacts(options.android_bucket_url,
696 options.android_build_branch,
697 options.arc_bucket_url, acls,
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700698 build_targets,
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900699 options.force_version)
700
David Rileyc0da9d92016-02-01 12:11:01 -0800701 stable_candidate = portage_util.BestEBuild(stable_ebuilds)
702
703 if stable_candidate:
Lann Martinffb95162018-08-28 12:02:54 -0600704 logging.info('Stable candidate found %s', stable_candidate.version)
David Rileyc0da9d92016-02-01 12:11:01 -0800705 else:
706 logging.info('No stable candidate found.')
707
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900708 if not options.skip_commit:
709 _PrepareGitBranch(options.tracking_branch, overlay_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800710
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900711 revved = MarkAndroidEBuildAsStable(
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900712 stable_candidate, unstable_ebuild, options.android_package,
David Riley73f00d92016-02-16 18:54:20 -0800713 version_to_uprev, android_package_dir,
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700714 options.android_build_branch, options.arc_bucket_url,
715 options.runtime_artifacts_bucket_url, build_targets)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900716
717 if revved:
718 android_version_atom, files_to_add, files_to_remove = revved
719 if not options.skip_commit:
720 _CommitChange(
721 _GIT_COMMIT_MESSAGE % {'android_package': options.android_package,
722 'android_version': version_to_uprev},
723 android_package_dir,
724 files_to_add,
725 files_to_remove,
726 )
David Rileyc0da9d92016-02-01 12:11:01 -0800727 if options.boards:
728 cros_mark_as_stable.CleanStalePackages(options.srcroot,
729 options.boards.split(':'),
730 [android_version_atom])
731
732 # Explicit print to communicate to caller.
733 print('ANDROID_VERSION_ATOM=%s' % android_version_atom)