blob: a2db3160b92c9b6ac61eab5f4004e0ea3e14e959 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
David Rileyc0da9d92016-02-01 12:11:01 -08002# Copyright 2016 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""This module uprevs Android for cbuildbot.
7
8After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A
9caller could then use this atom with emerge to build the newly uprevved version
10of Android e.g.
11
Shuhei Takahashi6d02c192017-04-05 14:01:24 +090012./cros_mark_android_as_stable \
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090013 --android_build_branch=git_pi-arc \
14 --android_package=android-container-pi
Shuhei Takahashi6d02c192017-04-05 14:01:24 +090015
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090016Returns chromeos-base/android-container-pi-6417892-r1
David Rileyc0da9d92016-02-01 12:11:01 -080017
Shao-Chuan Lee9c39e0c2020-04-24 11:40:34 +090018emerge-eve =chromeos-base/android-container-pi-6417892-r1
David Rileyc0da9d92016-02-01 12:11:01 -080019"""
20
21from __future__ import print_function
22
23import filecmp
24import glob
Junichi Uekawad21f94d2020-07-27 15:50:05 +090025import json
David Rileyc0da9d92016-02-01 12:11:01 -080026import os
Hidehiko Abe12727dd2016-05-27 23:23:45 +090027import re
khmel@google.com96c193e2018-05-10 14:00:38 -070028import time
Mike Frysinger00a02292020-04-19 06:28:03 -040029import sys
David Rileyc0da9d92016-02-01 12:11:01 -080030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
David Rileyc0da9d92016-02-01 12:11:01 -080032from chromite.lib import commandline
33from chromite.lib import cros_build_lib
34from chromite.lib import cros_logging as logging
35from chromite.lib import git
36from chromite.lib import gs
Shao-Chuan Lee301a4192021-02-08 11:53:49 +090037from chromite.lib import osutils
David Rileyc0da9d92016-02-01 12:11:01 -080038from chromite.lib import portage_util
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +090039from chromite.lib import repo_util
David Rileyc0da9d92016-02-01 12:11:01 -080040from chromite.scripts import cros_mark_as_stable
41
42
Mike Frysinger00a02292020-04-19 06:28:03 -040043assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
44
45
David Rileyc0da9d92016-02-01 12:11:01 -080046# Dir where all the action happens.
47_OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/'
48
Junichi Uekawa6d61ab02020-04-15 14:52:28 +090049_GIT_COMMIT_MESSAGE = """Marking latest for %(android_package)s ebuild with \
50version %(android_version)s as stable.
51
52BUG=None
53TEST=CQ
54"""
David Rileyc0da9d92016-02-01 12:11:01 -080055
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -070056_RUNTIME_ARTIFACTS_BUCKET_URL = 'gs://chromeos-arc-images/runtime_artifacts'
David Rileyc0da9d92016-02-01 12:11:01 -080057
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090058def IsBuildIdValid(bucket_url, build_branch, build_id, targets):
David Rileyc0da9d92016-02-01 12:11:01 -080059 """Checks that a specific build_id is valid.
60
61 Looks for that build_id for all builds. Confirms that the subpath can
62 be found and that the zip file is present in that subdirectory.
63
64 Args:
65 bucket_url: URL of Android build gs bucket
66 build_branch: branch of Android builds
67 build_id: A string. The Android build id number to check.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +090068 targets: Dict from build key to (targe build suffix, artifact file pattern)
69 pair.
David Rileyc0da9d92016-02-01 12:11:01 -080070
71 Returns:
72 Returns subpaths dictionary if build_id is valid.
73 None if the build_id is not valid.
74 """
75 gs_context = gs.GSContext()
76 subpaths_dict = {}
Mike Frysinger0bdbc102019-06-13 15:27:29 -040077 for build, (target, _) in targets.items():
David Rileyc0da9d92016-02-01 12:11:01 -080078 build_dir = '%s-%s' % (build_branch, target)
79 build_id_path = os.path.join(bucket_url, build_dir, build_id)
80
81 # Find name of subpath.
82 try:
83 subpaths = gs_context.List(build_id_path)
84 except gs.GSNoSuchKey:
Mike Frysinger968c1142020-05-09 00:37:56 -040085 logging.warning(
David Rileyc0da9d92016-02-01 12:11:01 -080086 'Directory [%s] does not contain any subpath, ignoring it.',
87 build_id_path)
88 return None
89 if len(subpaths) > 1:
Mike Frysinger968c1142020-05-09 00:37:56 -040090 logging.warning(
David Rileyc0da9d92016-02-01 12:11:01 -080091 'Directory [%s] contains more than one subpath, ignoring it.',
92 build_id_path)
93 return None
94
95 subpath_dir = subpaths[0].url.rstrip('/')
96 subpath_name = os.path.basename(subpath_dir)
97
98 # Look for a zipfile ending in the build_id number.
99 try:
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900100 gs_context.List(subpath_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800101 except gs.GSNoSuchKey:
Mike Frysinger968c1142020-05-09 00:37:56 -0400102 logging.warning(
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900103 'Did not find a file for build id [%s] in directory [%s].',
David Rileyc0da9d92016-02-01 12:11:01 -0800104 build_id, subpath_dir)
105 return None
106
107 # Record subpath for the build.
108 subpaths_dict[build] = subpath_name
109
110 # If we got here, it means we found an appropriate build for all platforms.
111 return subpaths_dict
112
113
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900114def GetLatestBuild(bucket_url, build_branch, targets):
David Rileyc0da9d92016-02-01 12:11:01 -0800115 """Searches the gs bucket for the latest green build.
116
117 Args:
118 bucket_url: URL of Android build gs bucket
119 build_branch: branch of Android builds
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900120 targets: Dict from build key to (targe build suffix, artifact file pattern)
121 pair.
David Rileyc0da9d92016-02-01 12:11:01 -0800122
123 Returns:
124 Tuple of (latest version string, subpaths dictionary)
125 If no latest build can be found, returns None, None
126 """
127 gs_context = gs.GSContext()
128 common_build_ids = None
129 # Find builds for each target.
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400130 for target, _ in targets.values():
David Rileyc0da9d92016-02-01 12:11:01 -0800131 build_dir = '-'.join((build_branch, target))
132 base_path = os.path.join(bucket_url, build_dir)
133 build_ids = []
134 for gs_result in gs_context.List(base_path):
135 # Remove trailing slashes and get the base name, which is the build_id.
136 build_id = os.path.basename(gs_result.url.rstrip('/'))
137 if not build_id.isdigit():
Mike Frysinger968c1142020-05-09 00:37:56 -0400138 logging.warning('Directory [%s] does not look like a valid build_id.',
139 gs_result.url)
David Rileyc0da9d92016-02-01 12:11:01 -0800140 continue
141 build_ids.append(build_id)
142
143 # Update current list of builds.
144 if common_build_ids is None:
145 # First run, populate it with the first platform.
146 common_build_ids = set(build_ids)
147 else:
148 # Already populated, find the ones that are common.
149 common_build_ids.intersection_update(build_ids)
150
151 if common_build_ids is None:
Mike Frysinger968c1142020-05-09 00:37:56 -0400152 logging.warning('Did not find a build_id common to all platforms.')
David Rileyc0da9d92016-02-01 12:11:01 -0800153 return None, None
154
155 # Otherwise, find the most recent one that is valid.
156 for build_id in sorted(common_build_ids, key=int, reverse=True):
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900157 subpaths = IsBuildIdValid(bucket_url, build_branch, build_id, targets)
David Rileyc0da9d92016-02-01 12:11:01 -0800158 if subpaths:
159 return build_id, subpaths
160
161 # If not found, no build_id is valid.
Mike Frysinger968c1142020-05-09 00:37:56 -0400162 logging.warning('Did not find a build_id valid on all platforms.')
David Rileyc0da9d92016-02-01 12:11:01 -0800163 return None, None
164
165
166def FindAndroidCandidates(package_dir):
167 """Return a tuple of Android's unstable ebuild and stable ebuilds.
168
169 Args:
170 package_dir: The path to where the package ebuild is stored.
171
172 Returns:
173 Tuple [unstable_ebuild, stable_ebuilds].
174
175 Raises:
176 Exception: if no unstable ebuild exists for Android.
177 """
178 stable_ebuilds = []
179 unstable_ebuilds = []
180 for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
181 ebuild = portage_util.EBuild(path)
182 if ebuild.version == '9999':
183 unstable_ebuilds.append(ebuild)
184 else:
185 stable_ebuilds.append(ebuild)
186
187 # Apply some sanity checks.
188 if not unstable_ebuilds:
189 raise Exception('Missing 9999 ebuild for %s' % package_dir)
190 if not stable_ebuilds:
Lann Martinffb95162018-08-28 12:02:54 -0600191 logging.warning('Missing stable ebuild for %s', package_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800192
193 return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
194
195
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800196def _GetArcBasename(build, basename):
197 """Tweaks filenames between Android bucket and ARC++ bucket.
198
199 Android builders create build artifacts with the same name for -user and
200 -userdebug builds, which breaks the android-container ebuild (b/33072485).
201 When copying the artifacts from the Android bucket to the ARC++ bucket some
202 artifacts will be renamed from the usual pattern
203 *cheets_${ARCH}-target_files-S{VERSION}.zip to
204 cheets_${BUILD_NAME}-target_files-S{VERSION}.zip which will typically look
205 like cheets_(${LABEL})*${ARCH}_userdebug-target_files-S{VERSION}.zip.
206
207 Args:
208 build: the build being mirrored, e.g. 'X86', 'ARM', 'X86_USERDEBUG'.
209 basename: the basename of the artifact to copy.
210
211 Returns:
212 The basename of the destination.
213 """
214 if build not in constants.ARC_BUILDS_NEED_ARTIFACTS_RENAMED:
215 return basename
Yūki Ishiief1ada92018-03-27 15:46:15 +0900216 if basename in constants.ARC_ARTIFACTS_RENAME_NOT_NEEDED:
217 return basename
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800218 to_discard, sep, to_keep = basename.partition('-')
219 if not sep:
220 logging.error(('Build %s: Could not find separator "-" in artifact'
221 ' basename %s'), build, basename)
222 return basename
Bernie Thompson63ed5612017-08-16 12:27:34 -0700223 if 'cheets_' in to_discard:
224 return 'cheets_%s-%s' % (build.lower(), to_keep)
225 elif 'bertha_' in to_discard:
226 return 'bertha_%s-%s' % (build.lower(), to_keep)
227 logging.error('Build %s: Unexpected artifact basename %s',
228 build, basename)
229 return basename
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800230
231
David Riley73f00d92016-02-16 18:54:20 -0800232def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths,
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900233 targets, arc_bucket_url, acls):
David Riley73f00d92016-02-16 18:54:20 -0800234 """Copies from source Android bucket to ARC++ specific bucket.
235
236 Copies each build to the ARC bucket eliminating the subpath.
237 Applies build specific ACLs for each file.
238
239 Args:
240 android_bucket_url: URL of Android build gs bucket
241 build_branch: branch of Android builds
242 build_id: A string. The Android build id number to check.
243 subpaths: Subpath dictionary for each build to copy.
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900244 targets: Dict from build key to (targe build suffix, artifact file pattern)
245 pair.
David Riley73f00d92016-02-16 18:54:20 -0800246 arc_bucket_url: URL of the target ARC build gs bucket
247 acls: ACLs dictionary for each build to copy.
248 """
249 gs_context = gs.GSContext()
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400250 for build, subpath in subpaths.items():
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900251 target, pattern = targets[build]
David Riley73f00d92016-02-16 18:54:20 -0800252 build_dir = '%s-%s' % (build_branch, target)
253 android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath)
254 arc_dir = os.path.join(arc_bucket_url, build_dir, build_id)
255
Hidehiko Abe12727dd2016-05-27 23:23:45 +0900256 # Copy all target files from android_dir to arc_dir, setting ACLs.
257 for targetfile in gs_context.List(android_dir):
258 if re.search(pattern, targetfile.url):
259 basename = os.path.basename(targetfile.url)
Nicolas Norvezb08f54d2016-12-05 17:58:54 -0800260 arc_path = os.path.join(arc_dir, _GetArcBasename(build, basename))
David Riley73f00d92016-02-16 18:54:20 -0800261 acl = acls[build]
262 needs_copy = True
khmel@google.com96c193e2018-05-10 14:00:38 -0700263 retry_count = 2
David Riley73f00d92016-02-16 18:54:20 -0800264
khmel@google.com96c193e2018-05-10 14:00:38 -0700265 # Retry in case race condition when several boards trying to copy the
266 # same resource
267 while True:
268 # Check a pre-existing file with the original source.
269 if gs_context.Exists(arc_path):
270 if (gs_context.Stat(targetfile.url).hash_crc32c !=
271 gs_context.Stat(arc_path).hash_crc32c):
Mike Frysinger968c1142020-05-09 00:37:56 -0400272 logging.warning('Removing incorrect file %s', arc_path)
khmel@google.com96c193e2018-05-10 14:00:38 -0700273 gs_context.Remove(arc_path)
274 else:
275 logging.info('Skipping already copied file %s', arc_path)
276 needs_copy = False
David Riley73f00d92016-02-16 18:54:20 -0800277
khmel@google.com96c193e2018-05-10 14:00:38 -0700278 # Copy if necessary, and set the ACL unconditionally.
279 # The Stat() call above doesn't verify the ACL is correct and
280 # the ChangeACL should be relatively cheap compared to the copy.
281 # This covers the following caes:
282 # - handling an interrupted copy from a previous run.
283 # - rerunning the copy in case one of the googlestorage_acl_X.txt
284 # files changes (e.g. we add a new variant which reuses a build).
285 if needs_copy:
286 logging.info('Copying %s -> %s (acl %s)',
287 targetfile.url, arc_path, acl)
288 try:
289 gs_context.Copy(targetfile.url, arc_path, version=0)
290 except gs.GSContextPreconditionFailed as error:
291 if not retry_count:
292 raise error
293 # Retry one more time after a short delay
294 logging.warning('Will retry copying %s -> %s',
295 targetfile.url, arc_path)
296 time.sleep(5)
297 retry_count = retry_count - 1
298 continue
299 gs_context.ChangeACL(arc_path, acl_args_file=acl)
300 break
David Riley73f00d92016-02-16 18:54:20 -0800301
302
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900303def MirrorArtifacts(android_bucket_url, android_build_branch, arc_bucket_url,
304 acls, targets, version=None):
305 """Mirrors artifacts from Android bucket to ARC bucket.
306
307 First, this function identifies which build version should be copied,
308 if not given. Please see GetLatestBuild() and IsBuildIdValid() for details.
309
310 On build version identified, then copies target artifacts to the ARC bucket,
311 with setting ACLs.
312
313 Args:
314 android_bucket_url: URL of Android build gs bucket
315 android_build_branch: branch of Android builds
316 arc_bucket_url: URL of the target ARC build gs bucket
317 acls: ACLs dictionary for each build to copy.
318 targets: Dict from build key to (targe build suffix, artifact file pattern)
319 pair.
320 version: (optional) A string. The Android build id number to check.
321 If not passed, detect latest good build version.
322
323 Returns:
324 Mirrored version.
325 """
326 if version:
327 subpaths = IsBuildIdValid(
328 android_bucket_url, android_build_branch, version, targets)
329 if not subpaths:
Lann Martinffb95162018-08-28 12:02:54 -0600330 logging.error('Requested build %s is not valid', version)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900331 else:
332 version, subpaths = GetLatestBuild(
333 android_bucket_url, android_build_branch, targets)
334
335 CopyToArcBucket(android_bucket_url, android_build_branch, version, subpaths,
336 targets, arc_bucket_url, acls)
khmel@google.com778a1cd2018-04-13 11:11:58 -0700337
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900338 return version
339
340
David Riley73f00d92016-02-16 18:54:20 -0800341def MakeAclDict(package_dir):
342 """Creates a dictionary of acl files for each build type.
343
344 Args:
345 package_dir: The path to where the package acl files are stored.
346
347 Returns:
348 Returns acls dictionary.
349 """
350 return dict(
351 (k, os.path.join(package_dir, v))
352 for k, v in constants.ARC_BUCKET_ACLS.items()
353 )
354
355
Qijiang Fan6588cc92019-11-20 13:26:04 +0900356def MakeBuildTargetDict(package_name, build_branch):
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700357 """Creates a dictionary of build targets.
358
Bernie Thompson63ed5612017-08-16 12:27:34 -0700359 Not all targets are common between branches, for example
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700360 sdk_google_cheets_x86 only exists on N.
361 This generates a dictionary listing the available build targets for a
362 specific branch.
363
364 Args:
Qijiang Fan6588cc92019-11-20 13:26:04 +0900365 package_name: package name of chromeos arc package.
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700366 build_branch: branch of Android builds.
367
368 Returns:
369 Returns build target dictionary.
370
371 Raises:
372 ValueError: if the Android build branch is invalid.
373 """
Qijiang Fan6588cc92019-11-20 13:26:04 +0900374 if constants.ANDROID_CONTAINER_PACKAGE_KEYWORD in package_name:
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900375 target_list = {
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900376 constants.ANDROID_PI_BUILD_BRANCH:
377 constants.ANDROID_PI_BUILD_TARGETS,
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900378 }
Qijiang Fan6588cc92019-11-20 13:26:04 +0900379 elif constants.ANDROID_VM_PACKAGE_KEYWORD in package_name:
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900380 target_list = {
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900381 constants.ANDROID_VMMST_BUILD_BRANCH:
382 constants.ANDROID_VMMST_BUILD_TARGETS,
383 constants.ANDROID_VMRVC_BUILD_BRANCH:
384 constants.ANDROID_VMRVC_BUILD_TARGETS,
385 }
386 else:
387 raise ValueError('Unknown package: %s' % package_name)
388 target = target_list.get(build_branch)
389 if not target:
Qijiang Fan6588cc92019-11-20 13:26:04 +0900390 raise ValueError('Unknown branch: %s' % build_branch)
Federico 'Morg' Pareschi041ee652020-03-10 15:09:42 +0900391 return target
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700392
393
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900394def PrintUprevMetadata(build_branch, stable_candidate, new_ebuild):
395 """Shows metadata on buildbot page at UprevAndroid step.
David Rileyc0da9d92016-02-01 12:11:01 -0800396
397 Args:
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900398 build_branch: The branch of Android builds.
399 stable_candidate: The existing stable ebuild.
400 new_ebuild: The newly written ebuild.
David Rileyc0da9d92016-02-01 12:11:01 -0800401 """
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900402 # Examples:
403 # "android-container-pi revved 6461825-r1 -> 6468247-r1"
404 # "android-container-pi revved 6461825-r1 -> 6461825-r2 (ebuild update only)"
405 msg = '%s revved %s -> %s' % (stable_candidate.pkgname,
406 stable_candidate.version,
407 new_ebuild.version)
408
409 old_android = stable_candidate.version_no_rev
410 new_android = new_ebuild.version_no_rev
411
412 if old_android == new_android:
413 msg += ' (ebuild update only)'
414 else:
415 ab_link = ('https://android-build.googleplex.com'
416 '/builds/%s/branches/%s/cls?end=%s'
417 % (new_android, build_branch, old_android))
418 logging.PrintBuildbotLink('Android changelog', ab_link)
419
420 logging.PrintBuildbotStepText(msg)
Junichi Uekawad21f94d2020-07-27 15:50:05 +0900421 logging.PrintKitchenSetBuildProperty('android_uprev', json.dumps({
422 'branch': build_branch,
423 'new': new_ebuild.version,
424 'old': stable_candidate.version,
425 'pkgname': stable_candidate.pkgname,
426 }))
David Rileyc0da9d92016-02-01 12:11:01 -0800427
428
Yury Khmelb009aeb2020-08-19 19:40:00 -0700429def FindDataCollectorArtifacts(gs_context,
430 android_version,
431 runtime_artifacts_bucket_url,
432 version_reference):
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700433 r"""Finds and includes into variables artifacts from arc.DataCollector.
434
Yury Khmelb009aeb2020-08-19 19:40:00 -0700435 This is used from UpdateDataCollectorArtifacts in order to check the
436 particular version.
437
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700438 Args:
Yury Khmelb009aeb2020-08-19 19:40:00 -0700439 gs_context: context to execute gsutil
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700440 android_version: The \d+ build id of Android.
441 runtime_artifacts_bucket_url: root of runtime artifacts
Yury Khmelb009aeb2020-08-19 19:40:00 -0700442 build_branch: build branch. Used to determine the pinned version if exists.
443 version_reference: which version to use as a reference. Could be '${PV}' in
444 case version of data collector artifacts matches the
445 Android version or direct version in case of override.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700446
447 Returns:
Yury Khmelb009aeb2020-08-19 19:40:00 -0700448 dictionary with filled ebuild variables. This dictionary is empty in case
449 no artificats are found.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700450 """
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700451 variables = {}
Yury Khmelb009aeb2020-08-19 19:40:00 -0700452
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700453 buckets = ['ureadahead_pack', 'gms_core_cache']
454 archs = ['arm', 'arm64', 'x86', 'x86_64']
455 build_types = ['user', 'userdebug']
456
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700457 for bucket in buckets:
458 for arch in archs:
459 for build_type in build_types:
460 path = (f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_'
Yury Khmele1b74402020-05-18 08:41:35 -0700461 f'{android_version}.tar')
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700462 if gs_context.Exists(path):
463 variables[(f'{arch}_{build_type}_{bucket}').upper()] = (
464 f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_'
465 f'{version_reference}.tar')
466
467 return variables
468
469
Yury Khmelb009aeb2020-08-19 19:40:00 -0700470def UpdateDataCollectorArtifacts(android_version,
471 runtime_artifacts_bucket_url,
472 build_branch):
473 r"""Finds and includes into variables artifacts from arc.DataCollector.
474
475 This verifies default android version. In case artificts are not found for
476 default Android version it tries to find artifacts for pinned version. If
477 pinned version is provided, it is required artifacts exist for the pinned
478 version.
479
480 Args:
481 android_version: The \d+ build id of Android.
482 runtime_artifacts_bucket_url: root of runtime artifacts
483 build_branch: build branch. Used to determine the pinned version if exists.
484
485 Returns:
486 dictionary with filled ebuild variables.
487 """
488
489 gs_context = gs.GSContext()
490 # Check the existing version. If we find any artifacts, use them.
491 variables = FindDataCollectorArtifacts(gs_context,
492 android_version,
493 runtime_artifacts_bucket_url,
494 '${PV}')
495 if variables:
496 # Data artificts were found.
497 return variables
498
499 # Check pinned version for the current branch.
500 pin_path = (f'{runtime_artifacts_bucket_url}/{build_branch}_pin_version')
501 if not gs_context.Exists(pin_path):
502 # No pinned version.
503 logging.warning(
504 'No data collector artifacts were found for %s',
505 android_version)
506 return variables
507
508 pin_version = gs_context.Cat(pin_path, encoding='utf-8').rstrip()
509 logging.info('Pinned version %s overrides %s',
510 pin_version, android_version)
511 variables = FindDataCollectorArtifacts(gs_context,
512 pin_version,
513 runtime_artifacts_bucket_url,
514 pin_version)
515 if not variables:
516 # If pin version set it must contain data.
517 raise Exception('Pinned version %s:%s does not contain artificats' % (
518 build_branch, pin_version))
519
520 return variables
521
522
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900523def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild,
524 android_package, android_version, package_dir,
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700525 build_branch, arc_bucket_url,
526 runtime_artifacts_bucket_url,
527 build_targets):
David Rileyc0da9d92016-02-01 12:11:01 -0800528 r"""Uprevs the Android ebuild.
529
530 This is the main function that uprevs from a stable candidate
531 to its new version.
532
533 Args:
534 stable_candidate: ebuild that corresponds to the stable ebuild we are
535 revving from. If None, builds the a new ebuild given the version
536 with revision set to 1.
537 unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900538 android_package: android package name.
David Rileyc0da9d92016-02-01 12:11:01 -0800539 android_version: The \d+ build id of Android.
David Rileyc0da9d92016-02-01 12:11:01 -0800540 package_dir: Path to the android-container package dir.
David Riley73f00d92016-02-16 18:54:20 -0800541 build_branch: branch of Android builds.
542 arc_bucket_url: URL of the target ARC build gs bucket.
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700543 runtime_artifacts_bucket_url: root of runtime artifacts
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700544 build_targets: build targets for this particular Android branch.
David Rileyc0da9d92016-02-01 12:11:01 -0800545
546 Returns:
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900547 Tuple[str, List[str], List[str]] if revved, or None
548 1. Full portage version atom (including rc's, etc) that was revved.
549 2. List of files to be `git add`ed.
550 3. List of files to be `git rm`ed.
David Rileyc0da9d92016-02-01 12:11:01 -0800551 """
552 def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
553 """Returns True if the new ebuild is redundant.
554
555 This is True if there if the current stable ebuild is the exact same copy
556 of the new one.
557 """
558 if not stable_ebuild:
559 return False
560
David Riley676f5402016-02-12 17:24:23 -0800561 if stable_candidate.version_no_rev == new_ebuild.version_no_rev:
David Rileyc0da9d92016-02-01 12:11:01 -0800562 return filecmp.cmp(
563 new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False)
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700564 return False
David Rileyc0da9d92016-02-01 12:11:01 -0800565
566 # Case where we have the last stable candidate with same version just rev.
David Riley676f5402016-02-12 17:24:23 -0800567 if stable_candidate and stable_candidate.version_no_rev == android_version:
David Rileyc0da9d92016-02-01 12:11:01 -0800568 new_ebuild_path = '%s-r%d.ebuild' % (
569 stable_candidate.ebuild_path_no_revision,
570 stable_candidate.current_revision + 1)
571 else:
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900572 pf = '%s-%s-r1' % (android_package, android_version)
David Rileyc0da9d92016-02-01 12:11:01 -0800573 new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
574
David Riley73f00d92016-02-16 18:54:20 -0800575 variables = {'BASE_URL': arc_bucket_url}
Mike Frysinger0bdbc102019-06-13 15:27:29 -0400576 for build, (target, _) in build_targets.items():
David Riley73f00d92016-02-16 18:54:20 -0800577 variables[build + '_TARGET'] = '%s-%s' % (build_branch, target)
David Rileyc0da9d92016-02-01 12:11:01 -0800578
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700579 variables.update(UpdateDataCollectorArtifacts(
Yury Khmelb009aeb2020-08-19 19:40:00 -0700580 android_version, runtime_artifacts_bucket_url, build_branch))
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700581
David Rileyc0da9d92016-02-01 12:11:01 -0800582 portage_util.EBuild.MarkAsStable(
583 unstable_ebuild.ebuild_path, new_ebuild_path,
584 variables, make_stable=True)
585 new_ebuild = portage_util.EBuild(new_ebuild_path)
586
587 # Determine whether this is ebuild is redundant.
588 if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
589 msg = 'Previous ebuild with same version found and ebuild is redundant.'
590 logging.info(msg)
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900591 logging.PrintBuildbotStepText('%s %s not revved'
592 % (stable_candidate.pkgname,
593 stable_candidate.version))
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900594 osutils.SafeUnlink(new_ebuild_path)
David Rileyc0da9d92016-02-01 12:11:01 -0800595 return None
596
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900597 # PFQ runs should always be able to find a stable candidate.
David Rileyc0da9d92016-02-01 12:11:01 -0800598 if stable_candidate:
Shao-Chuan Lee085e3d72020-05-11 16:00:42 +0900599 PrintUprevMetadata(build_branch, stable_candidate, new_ebuild)
David Rileyc0da9d92016-02-01 12:11:01 -0800600
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900601 files_to_add = [new_ebuild_path]
602 files_to_remove = []
David Rileyc0da9d92016-02-01 12:11:01 -0800603 if stable_candidate and not stable_candidate.IsSticky():
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900604 osutils.SafeUnlink(stable_candidate.ebuild_path)
605 files_to_remove.append(stable_candidate.ebuild_path)
David Rileyc0da9d92016-02-01 12:11:01 -0800606
607 # Update ebuild manifest and git add it.
608 gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
Mike Frysinger45602c72019-09-22 02:15:11 -0400609 cros_build_lib.run(gen_manifest_cmd, extra_env=None, print_cmd=True)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900610 files_to_add.append('Manifest')
David Rileyc0da9d92016-02-01 12:11:01 -0800611
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900612 return (
613 f'{new_ebuild.package}-{new_ebuild.version}',
614 files_to_add,
615 files_to_remove,
616 )
David Rileyc0da9d92016-02-01 12:11:01 -0800617
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900618
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +0900619def _PrepareGitBranch(overlay_dir):
620 """Prepares a git branch for the uprev commit.
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900621
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +0900622 If the overlay project is currently on a branch (e.g. patches are being
623 applied), rebase the new branch on top of it.
624
625 Args:
626 overlay_dir: The overlay directory.
627 """
628 existing_branch = git.GetCurrentBranch(overlay_dir)
629 repo_util.Repository.MustFind(overlay_dir).StartBranch(
630 constants.STABLE_EBUILD_BRANCH, projects=['.'], cwd=overlay_dir)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900631 if existing_branch:
632 git.RunGit(overlay_dir, ['rebase', existing_branch])
633
634
635def _CommitChange(message, android_package_dir, files_to_add, files_to_remove):
636 """Commit changes to git with list of files to add/remove."""
637 git.RunGit(android_package_dir, ['add', '--'] + files_to_add)
638 git.RunGit(android_package_dir, ['rm', '--'] + files_to_remove)
639
640 portage_util.EBuild.CommitChange(message, android_package_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800641
642
643def GetParser():
644 """Creates the argument parser."""
645 parser = commandline.ArgumentParser()
646 parser.add_argument('-b', '--boards')
647 parser.add_argument('--android_bucket_url',
David Riley73f00d92016-02-16 18:54:20 -0800648 default=constants.ANDROID_BUCKET_URL,
649 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800650 parser.add_argument('--android_build_branch',
Shuhei Takahashi6d02c192017-04-05 14:01:24 +0900651 required=True,
652 help='Android branch to import from. '
653 'Ex: git_mnc-dr-arc-dev')
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900654 parser.add_argument('--android_package',
655 default=constants.ANDROID_PACKAGE_NAME)
David Riley73f00d92016-02-16 18:54:20 -0800656 parser.add_argument('--arc_bucket_url',
657 default=constants.ARC_BUCKET_URL,
658 type='gs_path')
David Rileyc0da9d92016-02-01 12:11:01 -0800659 parser.add_argument('-f', '--force_version',
660 help='Android build id to use')
661 parser.add_argument('-s', '--srcroot',
662 default=os.path.join(os.environ['HOME'], 'trunk', 'src'),
663 help='Path to the src directory')
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +0900664 # TODO(crbug/1074145): Remove once CQ specifies --skip_commit.
David Rileyc0da9d92016-02-01 12:11:01 -0800665 parser.add_argument('-t', '--tracking_branch', default='cros/master',
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +0900666 help='DEPRECATED')
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700667 parser.add_argument('--runtime_artifacts_bucket_url',
668 default=_RUNTIME_ARTIFACTS_BUCKET_URL,
669 type='gs_path')
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900670 parser.add_argument('--skip_commit',
671 action='store_true',
672 help='Skip commiting uprev changes to git')
David Rileyc0da9d92016-02-01 12:11:01 -0800673 return parser
674
675
676def main(argv):
Hidehiko Abec9ecf262017-07-05 15:17:41 +0900677 logging.EnableBuildbotMarkers()
David Rileyc0da9d92016-02-01 12:11:01 -0800678 parser = GetParser()
679 options = parser.parse_args(argv)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900680 # HACK(b/179456416): Skip git operations if tracking_branch is empty, which
681 # happens when called from CQ builders.
682 # TODO(b/179456416): Make CQ builders specify the --skip_commit option.
683 if not options.tracking_branch:
684 options.skip_commit = True
David Rileyc0da9d92016-02-01 12:11:01 -0800685 options.Freeze()
686
687 overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900688 android_package_dir = os.path.join(
689 overlay_dir,
690 portage_util.GetFullAndroidPortagePackageName(options.android_package))
David Rileyc0da9d92016-02-01 12:11:01 -0800691 version_to_uprev = None
David Rileyc0da9d92016-02-01 12:11:01 -0800692
693 (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir)
David Riley73f00d92016-02-16 18:54:20 -0800694 acls = MakeAclDict(android_package_dir)
Qijiang Fan6588cc92019-11-20 13:26:04 +0900695 build_targets = MakeBuildTargetDict(options.android_package,
696 options.android_build_branch)
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900697 # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt).
698 version_to_uprev = MirrorArtifacts(options.android_bucket_url,
699 options.android_build_branch,
700 options.arc_bucket_url, acls,
Nicolas Norvez4bd854f2017-05-23 10:04:45 -0700701 build_targets,
Hidehiko Abe1ebc25d2016-07-28 02:24:37 +0900702 options.force_version)
703
David Rileyc0da9d92016-02-01 12:11:01 -0800704 stable_candidate = portage_util.BestEBuild(stable_ebuilds)
705
706 if stable_candidate:
Lann Martinffb95162018-08-28 12:02:54 -0600707 logging.info('Stable candidate found %s', stable_candidate.version)
David Rileyc0da9d92016-02-01 12:11:01 -0800708 else:
709 logging.info('No stable candidate found.')
710
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900711 if not options.skip_commit:
Shao-Chuan Lee007dbe82021-02-09 14:05:39 +0900712 _PrepareGitBranch(overlay_dir)
David Rileyc0da9d92016-02-01 12:11:01 -0800713
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900714 revved = MarkAndroidEBuildAsStable(
Hidehiko Abe4fd94ae2017-01-24 18:59:55 +0900715 stable_candidate, unstable_ebuild, options.android_package,
David Riley73f00d92016-02-16 18:54:20 -0800716 version_to_uprev, android_package_dir,
khmel@chromium.orgd3ec3d72020-04-29 15:57:35 -0700717 options.android_build_branch, options.arc_bucket_url,
718 options.runtime_artifacts_bucket_url, build_targets)
Shao-Chuan Lee301a4192021-02-08 11:53:49 +0900719
720 if revved:
721 android_version_atom, files_to_add, files_to_remove = revved
722 if not options.skip_commit:
723 _CommitChange(
724 _GIT_COMMIT_MESSAGE % {'android_package': options.android_package,
725 'android_version': version_to_uprev},
726 android_package_dir,
727 files_to_add,
728 files_to_remove,
729 )
David Rileyc0da9d92016-02-01 12:11:01 -0800730 if options.boards:
731 cros_mark_as_stable.CleanStalePackages(options.srcroot,
732 options.boards.split(':'),
733 [android_version_atom])
734
735 # Explicit print to communicate to caller.
736 print('ANDROID_VERSION_ATOM=%s' % android_version_atom)