Mike Frysinger | e58c0e2 | 2017-10-04 15:43:30 -0400 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 2 | # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """This module uprevs Android for cbuildbot. |
| 7 | |
| 8 | After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A |
| 9 | caller could then use this atom with emerge to build the newly uprevved version |
| 10 | of Android e.g. |
| 11 | |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 12 | ./cros_mark_android_as_stable \ |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 13 | --android_build_branch=git_pi-arc \ |
| 14 | --android_package=android-container-pi |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 15 | |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 16 | Returns chromeos-base/android-container-pi-6417892-r1 |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 17 | |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 18 | emerge-eve =chromeos-base/android-container-pi-6417892-r1 |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 19 | """ |
| 20 | |
| 21 | from __future__ import print_function |
| 22 | |
| 23 | import filecmp |
| 24 | import glob |
Junichi Uekawa | d21f94d | 2020-07-27 15:50:05 +0900 | [diff] [blame] | 25 | import json |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 26 | import os |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 27 | import re |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 28 | import time |
Mike Frysinger | 00a0229 | 2020-04-19 06:28:03 -0400 | [diff] [blame] | 29 | import sys |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 30 | |
Aviv Keshet | b7519e1 | 2016-10-04 00:50:00 -0700 | [diff] [blame] | 31 | from chromite.lib import constants |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 32 | from chromite.lib import commandline |
| 33 | from chromite.lib import cros_build_lib |
| 34 | from chromite.lib import cros_logging as logging |
| 35 | from chromite.lib import git |
| 36 | from chromite.lib import gs |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 37 | from chromite.lib import osutils |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 38 | from chromite.lib import portage_util |
| 39 | from chromite.scripts import cros_mark_as_stable |
| 40 | |
| 41 | |
Mike Frysinger | 00a0229 | 2020-04-19 06:28:03 -0400 | [diff] [blame] | 42 | assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' |
| 43 | |
| 44 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 45 | # Dir where all the action happens. |
| 46 | _OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/' |
| 47 | |
Junichi Uekawa | 6d61ab0 | 2020-04-15 14:52:28 +0900 | [diff] [blame] | 48 | _GIT_COMMIT_MESSAGE = """Marking latest for %(android_package)s ebuild with \ |
| 49 | version %(android_version)s as stable. |
| 50 | |
| 51 | BUG=None |
| 52 | TEST=CQ |
| 53 | """ |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 54 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 55 | _RUNTIME_ARTIFACTS_BUCKET_URL = 'gs://chromeos-arc-images/runtime_artifacts' |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 56 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 57 | def IsBuildIdValid(bucket_url, build_branch, build_id, targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 58 | """Checks that a specific build_id is valid. |
| 59 | |
| 60 | Looks for that build_id for all builds. Confirms that the subpath can |
| 61 | be found and that the zip file is present in that subdirectory. |
| 62 | |
| 63 | Args: |
| 64 | bucket_url: URL of Android build gs bucket |
| 65 | build_branch: branch of Android builds |
| 66 | build_id: A string. The Android build id number to check. |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 67 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 68 | pair. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 69 | |
| 70 | Returns: |
| 71 | Returns subpaths dictionary if build_id is valid. |
| 72 | None if the build_id is not valid. |
| 73 | """ |
| 74 | gs_context = gs.GSContext() |
| 75 | subpaths_dict = {} |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 76 | for build, (target, _) in targets.items(): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 77 | build_dir = '%s-%s' % (build_branch, target) |
| 78 | build_id_path = os.path.join(bucket_url, build_dir, build_id) |
| 79 | |
| 80 | # Find name of subpath. |
| 81 | try: |
| 82 | subpaths = gs_context.List(build_id_path) |
| 83 | except gs.GSNoSuchKey: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 84 | logging.warning( |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 85 | 'Directory [%s] does not contain any subpath, ignoring it.', |
| 86 | build_id_path) |
| 87 | return None |
| 88 | if len(subpaths) > 1: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 89 | logging.warning( |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 90 | 'Directory [%s] contains more than one subpath, ignoring it.', |
| 91 | build_id_path) |
| 92 | return None |
| 93 | |
| 94 | subpath_dir = subpaths[0].url.rstrip('/') |
| 95 | subpath_name = os.path.basename(subpath_dir) |
| 96 | |
| 97 | # Look for a zipfile ending in the build_id number. |
| 98 | try: |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 99 | gs_context.List(subpath_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 100 | except gs.GSNoSuchKey: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 101 | logging.warning( |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 102 | 'Did not find a file for build id [%s] in directory [%s].', |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 103 | build_id, subpath_dir) |
| 104 | return None |
| 105 | |
| 106 | # Record subpath for the build. |
| 107 | subpaths_dict[build] = subpath_name |
| 108 | |
| 109 | # If we got here, it means we found an appropriate build for all platforms. |
| 110 | return subpaths_dict |
| 111 | |
| 112 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 113 | def GetLatestBuild(bucket_url, build_branch, targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 114 | """Searches the gs bucket for the latest green build. |
| 115 | |
| 116 | Args: |
| 117 | bucket_url: URL of Android build gs bucket |
| 118 | build_branch: branch of Android builds |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 119 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 120 | pair. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 121 | |
| 122 | Returns: |
| 123 | Tuple of (latest version string, subpaths dictionary) |
| 124 | If no latest build can be found, returns None, None |
| 125 | """ |
| 126 | gs_context = gs.GSContext() |
| 127 | common_build_ids = None |
| 128 | # Find builds for each target. |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 129 | for target, _ in targets.values(): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 130 | build_dir = '-'.join((build_branch, target)) |
| 131 | base_path = os.path.join(bucket_url, build_dir) |
| 132 | build_ids = [] |
| 133 | for gs_result in gs_context.List(base_path): |
| 134 | # Remove trailing slashes and get the base name, which is the build_id. |
| 135 | build_id = os.path.basename(gs_result.url.rstrip('/')) |
| 136 | if not build_id.isdigit(): |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 137 | logging.warning('Directory [%s] does not look like a valid build_id.', |
| 138 | gs_result.url) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 139 | continue |
| 140 | build_ids.append(build_id) |
| 141 | |
| 142 | # Update current list of builds. |
| 143 | if common_build_ids is None: |
| 144 | # First run, populate it with the first platform. |
| 145 | common_build_ids = set(build_ids) |
| 146 | else: |
| 147 | # Already populated, find the ones that are common. |
| 148 | common_build_ids.intersection_update(build_ids) |
| 149 | |
| 150 | if common_build_ids is None: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 151 | logging.warning('Did not find a build_id common to all platforms.') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 152 | return None, None |
| 153 | |
| 154 | # Otherwise, find the most recent one that is valid. |
| 155 | for build_id in sorted(common_build_ids, key=int, reverse=True): |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 156 | subpaths = IsBuildIdValid(bucket_url, build_branch, build_id, targets) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 157 | if subpaths: |
| 158 | return build_id, subpaths |
| 159 | |
| 160 | # If not found, no build_id is valid. |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 161 | logging.warning('Did not find a build_id valid on all platforms.') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 162 | return None, None |
| 163 | |
| 164 | |
| 165 | def FindAndroidCandidates(package_dir): |
| 166 | """Return a tuple of Android's unstable ebuild and stable ebuilds. |
| 167 | |
| 168 | Args: |
| 169 | package_dir: The path to where the package ebuild is stored. |
| 170 | |
| 171 | Returns: |
| 172 | Tuple [unstable_ebuild, stable_ebuilds]. |
| 173 | |
| 174 | Raises: |
| 175 | Exception: if no unstable ebuild exists for Android. |
| 176 | """ |
| 177 | stable_ebuilds = [] |
| 178 | unstable_ebuilds = [] |
| 179 | for path in glob.glob(os.path.join(package_dir, '*.ebuild')): |
| 180 | ebuild = portage_util.EBuild(path) |
| 181 | if ebuild.version == '9999': |
| 182 | unstable_ebuilds.append(ebuild) |
| 183 | else: |
| 184 | stable_ebuilds.append(ebuild) |
| 185 | |
| 186 | # Apply some sanity checks. |
| 187 | if not unstable_ebuilds: |
| 188 | raise Exception('Missing 9999 ebuild for %s' % package_dir) |
| 189 | if not stable_ebuilds: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 190 | logging.warning('Missing stable ebuild for %s', package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 191 | |
| 192 | return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds |
| 193 | |
| 194 | |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 195 | def _GetArcBasename(build, basename): |
| 196 | """Tweaks filenames between Android bucket and ARC++ bucket. |
| 197 | |
| 198 | Android builders create build artifacts with the same name for -user and |
| 199 | -userdebug builds, which breaks the android-container ebuild (b/33072485). |
| 200 | When copying the artifacts from the Android bucket to the ARC++ bucket some |
| 201 | artifacts will be renamed from the usual pattern |
| 202 | *cheets_${ARCH}-target_files-S{VERSION}.zip to |
| 203 | cheets_${BUILD_NAME}-target_files-S{VERSION}.zip which will typically look |
| 204 | like cheets_(${LABEL})*${ARCH}_userdebug-target_files-S{VERSION}.zip. |
| 205 | |
| 206 | Args: |
| 207 | build: the build being mirrored, e.g. 'X86', 'ARM', 'X86_USERDEBUG'. |
| 208 | basename: the basename of the artifact to copy. |
| 209 | |
| 210 | Returns: |
| 211 | The basename of the destination. |
| 212 | """ |
| 213 | if build not in constants.ARC_BUILDS_NEED_ARTIFACTS_RENAMED: |
| 214 | return basename |
Yūki Ishii | ef1ada9 | 2018-03-27 15:46:15 +0900 | [diff] [blame] | 215 | if basename in constants.ARC_ARTIFACTS_RENAME_NOT_NEEDED: |
| 216 | return basename |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 217 | to_discard, sep, to_keep = basename.partition('-') |
| 218 | if not sep: |
| 219 | logging.error(('Build %s: Could not find separator "-" in artifact' |
| 220 | ' basename %s'), build, basename) |
| 221 | return basename |
Bernie Thompson | 63ed561 | 2017-08-16 12:27:34 -0700 | [diff] [blame] | 222 | if 'cheets_' in to_discard: |
| 223 | return 'cheets_%s-%s' % (build.lower(), to_keep) |
| 224 | elif 'bertha_' in to_discard: |
| 225 | return 'bertha_%s-%s' % (build.lower(), to_keep) |
| 226 | logging.error('Build %s: Unexpected artifact basename %s', |
| 227 | build, basename) |
| 228 | return basename |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 229 | |
| 230 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 231 | def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths, |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 232 | targets, arc_bucket_url, acls): |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 233 | """Copies from source Android bucket to ARC++ specific bucket. |
| 234 | |
| 235 | Copies each build to the ARC bucket eliminating the subpath. |
| 236 | Applies build specific ACLs for each file. |
| 237 | |
| 238 | Args: |
| 239 | android_bucket_url: URL of Android build gs bucket |
| 240 | build_branch: branch of Android builds |
| 241 | build_id: A string. The Android build id number to check. |
| 242 | subpaths: Subpath dictionary for each build to copy. |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 243 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 244 | pair. |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 245 | arc_bucket_url: URL of the target ARC build gs bucket |
| 246 | acls: ACLs dictionary for each build to copy. |
| 247 | """ |
| 248 | gs_context = gs.GSContext() |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 249 | for build, subpath in subpaths.items(): |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 250 | target, pattern = targets[build] |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 251 | build_dir = '%s-%s' % (build_branch, target) |
| 252 | android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath) |
| 253 | arc_dir = os.path.join(arc_bucket_url, build_dir, build_id) |
| 254 | |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 255 | # Copy all target files from android_dir to arc_dir, setting ACLs. |
| 256 | for targetfile in gs_context.List(android_dir): |
| 257 | if re.search(pattern, targetfile.url): |
| 258 | basename = os.path.basename(targetfile.url) |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 259 | arc_path = os.path.join(arc_dir, _GetArcBasename(build, basename)) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 260 | acl = acls[build] |
| 261 | needs_copy = True |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 262 | retry_count = 2 |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 263 | |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 264 | # Retry in case race condition when several boards trying to copy the |
| 265 | # same resource |
| 266 | while True: |
| 267 | # Check a pre-existing file with the original source. |
| 268 | if gs_context.Exists(arc_path): |
| 269 | if (gs_context.Stat(targetfile.url).hash_crc32c != |
| 270 | gs_context.Stat(arc_path).hash_crc32c): |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 271 | logging.warning('Removing incorrect file %s', arc_path) |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 272 | gs_context.Remove(arc_path) |
| 273 | else: |
| 274 | logging.info('Skipping already copied file %s', arc_path) |
| 275 | needs_copy = False |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 276 | |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 277 | # Copy if necessary, and set the ACL unconditionally. |
| 278 | # The Stat() call above doesn't verify the ACL is correct and |
| 279 | # the ChangeACL should be relatively cheap compared to the copy. |
| 280 | # This covers the following caes: |
| 281 | # - handling an interrupted copy from a previous run. |
| 282 | # - rerunning the copy in case one of the googlestorage_acl_X.txt |
| 283 | # files changes (e.g. we add a new variant which reuses a build). |
| 284 | if needs_copy: |
| 285 | logging.info('Copying %s -> %s (acl %s)', |
| 286 | targetfile.url, arc_path, acl) |
| 287 | try: |
| 288 | gs_context.Copy(targetfile.url, arc_path, version=0) |
| 289 | except gs.GSContextPreconditionFailed as error: |
| 290 | if not retry_count: |
| 291 | raise error |
| 292 | # Retry one more time after a short delay |
| 293 | logging.warning('Will retry copying %s -> %s', |
| 294 | targetfile.url, arc_path) |
| 295 | time.sleep(5) |
| 296 | retry_count = retry_count - 1 |
| 297 | continue |
| 298 | gs_context.ChangeACL(arc_path, acl_args_file=acl) |
| 299 | break |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 300 | |
| 301 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 302 | def MirrorArtifacts(android_bucket_url, android_build_branch, arc_bucket_url, |
| 303 | acls, targets, version=None): |
| 304 | """Mirrors artifacts from Android bucket to ARC bucket. |
| 305 | |
| 306 | First, this function identifies which build version should be copied, |
| 307 | if not given. Please see GetLatestBuild() and IsBuildIdValid() for details. |
| 308 | |
| 309 | On build version identified, then copies target artifacts to the ARC bucket, |
| 310 | with setting ACLs. |
| 311 | |
| 312 | Args: |
| 313 | android_bucket_url: URL of Android build gs bucket |
| 314 | android_build_branch: branch of Android builds |
| 315 | arc_bucket_url: URL of the target ARC build gs bucket |
| 316 | acls: ACLs dictionary for each build to copy. |
| 317 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 318 | pair. |
| 319 | version: (optional) A string. The Android build id number to check. |
| 320 | If not passed, detect latest good build version. |
| 321 | |
| 322 | Returns: |
| 323 | Mirrored version. |
| 324 | """ |
| 325 | if version: |
| 326 | subpaths = IsBuildIdValid( |
| 327 | android_bucket_url, android_build_branch, version, targets) |
| 328 | if not subpaths: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 329 | logging.error('Requested build %s is not valid', version) |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 330 | else: |
| 331 | version, subpaths = GetLatestBuild( |
| 332 | android_bucket_url, android_build_branch, targets) |
| 333 | |
| 334 | CopyToArcBucket(android_bucket_url, android_build_branch, version, subpaths, |
| 335 | targets, arc_bucket_url, acls) |
khmel@google.com | 778a1cd | 2018-04-13 11:11:58 -0700 | [diff] [blame] | 336 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 337 | return version |
| 338 | |
| 339 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 340 | def MakeAclDict(package_dir): |
| 341 | """Creates a dictionary of acl files for each build type. |
| 342 | |
| 343 | Args: |
| 344 | package_dir: The path to where the package acl files are stored. |
| 345 | |
| 346 | Returns: |
| 347 | Returns acls dictionary. |
| 348 | """ |
| 349 | return dict( |
| 350 | (k, os.path.join(package_dir, v)) |
| 351 | for k, v in constants.ARC_BUCKET_ACLS.items() |
| 352 | ) |
| 353 | |
| 354 | |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 355 | def MakeBuildTargetDict(package_name, build_branch): |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 356 | """Creates a dictionary of build targets. |
| 357 | |
Bernie Thompson | 63ed561 | 2017-08-16 12:27:34 -0700 | [diff] [blame] | 358 | Not all targets are common between branches, for example |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 359 | sdk_google_cheets_x86 only exists on N. |
| 360 | This generates a dictionary listing the available build targets for a |
| 361 | specific branch. |
| 362 | |
| 363 | Args: |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 364 | package_name: package name of chromeos arc package. |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 365 | build_branch: branch of Android builds. |
| 366 | |
| 367 | Returns: |
| 368 | Returns build target dictionary. |
| 369 | |
| 370 | Raises: |
| 371 | ValueError: if the Android build branch is invalid. |
| 372 | """ |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 373 | if constants.ANDROID_CONTAINER_PACKAGE_KEYWORD in package_name: |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 374 | target_list = { |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 375 | constants.ANDROID_PI_BUILD_BRANCH: |
| 376 | constants.ANDROID_PI_BUILD_TARGETS, |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 377 | } |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 378 | elif constants.ANDROID_VM_PACKAGE_KEYWORD in package_name: |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 379 | target_list = { |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 380 | constants.ANDROID_VMMST_BUILD_BRANCH: |
| 381 | constants.ANDROID_VMMST_BUILD_TARGETS, |
| 382 | constants.ANDROID_VMRVC_BUILD_BRANCH: |
| 383 | constants.ANDROID_VMRVC_BUILD_TARGETS, |
| 384 | } |
| 385 | else: |
| 386 | raise ValueError('Unknown package: %s' % package_name) |
| 387 | target = target_list.get(build_branch) |
| 388 | if not target: |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 389 | raise ValueError('Unknown branch: %s' % build_branch) |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 390 | return target |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 391 | |
| 392 | |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 393 | def PrintUprevMetadata(build_branch, stable_candidate, new_ebuild): |
| 394 | """Shows metadata on buildbot page at UprevAndroid step. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 395 | |
| 396 | Args: |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 397 | build_branch: The branch of Android builds. |
| 398 | stable_candidate: The existing stable ebuild. |
| 399 | new_ebuild: The newly written ebuild. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 400 | """ |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 401 | # Examples: |
| 402 | # "android-container-pi revved 6461825-r1 -> 6468247-r1" |
| 403 | # "android-container-pi revved 6461825-r1 -> 6461825-r2 (ebuild update only)" |
| 404 | msg = '%s revved %s -> %s' % (stable_candidate.pkgname, |
| 405 | stable_candidate.version, |
| 406 | new_ebuild.version) |
| 407 | |
| 408 | old_android = stable_candidate.version_no_rev |
| 409 | new_android = new_ebuild.version_no_rev |
| 410 | |
| 411 | if old_android == new_android: |
| 412 | msg += ' (ebuild update only)' |
| 413 | else: |
| 414 | ab_link = ('https://android-build.googleplex.com' |
| 415 | '/builds/%s/branches/%s/cls?end=%s' |
| 416 | % (new_android, build_branch, old_android)) |
| 417 | logging.PrintBuildbotLink('Android changelog', ab_link) |
| 418 | |
| 419 | logging.PrintBuildbotStepText(msg) |
Junichi Uekawa | d21f94d | 2020-07-27 15:50:05 +0900 | [diff] [blame] | 420 | logging.PrintKitchenSetBuildProperty('android_uprev', json.dumps({ |
| 421 | 'branch': build_branch, |
| 422 | 'new': new_ebuild.version, |
| 423 | 'old': stable_candidate.version, |
| 424 | 'pkgname': stable_candidate.pkgname, |
| 425 | })) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 426 | |
| 427 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 428 | def FindDataCollectorArtifacts(gs_context, |
| 429 | android_version, |
| 430 | runtime_artifacts_bucket_url, |
| 431 | version_reference): |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 432 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
| 433 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 434 | This is used from UpdateDataCollectorArtifacts in order to check the |
| 435 | particular version. |
| 436 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 437 | Args: |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 438 | gs_context: context to execute gsutil |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 439 | android_version: The \d+ build id of Android. |
| 440 | runtime_artifacts_bucket_url: root of runtime artifacts |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 441 | build_branch: build branch. Used to determine the pinned version if exists. |
| 442 | version_reference: which version to use as a reference. Could be '${PV}' in |
| 443 | case version of data collector artifacts matches the |
| 444 | Android version or direct version in case of override. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 445 | |
| 446 | Returns: |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 447 | dictionary with filled ebuild variables. This dictionary is empty in case |
| 448 | no artificats are found. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 449 | """ |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 450 | variables = {} |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 451 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 452 | buckets = ['ureadahead_pack', 'gms_core_cache'] |
| 453 | archs = ['arm', 'arm64', 'x86', 'x86_64'] |
| 454 | build_types = ['user', 'userdebug'] |
| 455 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 456 | for bucket in buckets: |
| 457 | for arch in archs: |
| 458 | for build_type in build_types: |
| 459 | path = (f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_' |
Yury Khmel | e1b7440 | 2020-05-18 08:41:35 -0700 | [diff] [blame] | 460 | f'{android_version}.tar') |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 461 | if gs_context.Exists(path): |
| 462 | variables[(f'{arch}_{build_type}_{bucket}').upper()] = ( |
| 463 | f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_' |
| 464 | f'{version_reference}.tar') |
| 465 | |
| 466 | return variables |
| 467 | |
| 468 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 469 | def UpdateDataCollectorArtifacts(android_version, |
| 470 | runtime_artifacts_bucket_url, |
| 471 | build_branch): |
| 472 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
| 473 | |
| 474 | This verifies default android version. In case artificts are not found for |
| 475 | default Android version it tries to find artifacts for pinned version. If |
| 476 | pinned version is provided, it is required artifacts exist for the pinned |
| 477 | version. |
| 478 | |
| 479 | Args: |
| 480 | android_version: The \d+ build id of Android. |
| 481 | runtime_artifacts_bucket_url: root of runtime artifacts |
| 482 | build_branch: build branch. Used to determine the pinned version if exists. |
| 483 | |
| 484 | Returns: |
| 485 | dictionary with filled ebuild variables. |
| 486 | """ |
| 487 | |
| 488 | gs_context = gs.GSContext() |
| 489 | # Check the existing version. If we find any artifacts, use them. |
| 490 | variables = FindDataCollectorArtifacts(gs_context, |
| 491 | android_version, |
| 492 | runtime_artifacts_bucket_url, |
| 493 | '${PV}') |
| 494 | if variables: |
| 495 | # Data artificts were found. |
| 496 | return variables |
| 497 | |
| 498 | # Check pinned version for the current branch. |
| 499 | pin_path = (f'{runtime_artifacts_bucket_url}/{build_branch}_pin_version') |
| 500 | if not gs_context.Exists(pin_path): |
| 501 | # No pinned version. |
| 502 | logging.warning( |
| 503 | 'No data collector artifacts were found for %s', |
| 504 | android_version) |
| 505 | return variables |
| 506 | |
| 507 | pin_version = gs_context.Cat(pin_path, encoding='utf-8').rstrip() |
| 508 | logging.info('Pinned version %s overrides %s', |
| 509 | pin_version, android_version) |
| 510 | variables = FindDataCollectorArtifacts(gs_context, |
| 511 | pin_version, |
| 512 | runtime_artifacts_bucket_url, |
| 513 | pin_version) |
| 514 | if not variables: |
| 515 | # If pin version set it must contain data. |
| 516 | raise Exception('Pinned version %s:%s does not contain artificats' % ( |
| 517 | build_branch, pin_version)) |
| 518 | |
| 519 | return variables |
| 520 | |
| 521 | |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 522 | def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild, |
| 523 | android_package, android_version, package_dir, |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 524 | build_branch, arc_bucket_url, |
| 525 | runtime_artifacts_bucket_url, |
| 526 | build_targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 527 | r"""Uprevs the Android ebuild. |
| 528 | |
| 529 | This is the main function that uprevs from a stable candidate |
| 530 | to its new version. |
| 531 | |
| 532 | Args: |
| 533 | stable_candidate: ebuild that corresponds to the stable ebuild we are |
| 534 | revving from. If None, builds the a new ebuild given the version |
| 535 | with revision set to 1. |
| 536 | unstable_ebuild: ebuild corresponding to the unstable ebuild for Android. |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 537 | android_package: android package name. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 538 | android_version: The \d+ build id of Android. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 539 | package_dir: Path to the android-container package dir. |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 540 | build_branch: branch of Android builds. |
| 541 | arc_bucket_url: URL of the target ARC build gs bucket. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 542 | runtime_artifacts_bucket_url: root of runtime artifacts |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 543 | build_targets: build targets for this particular Android branch. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 544 | |
| 545 | Returns: |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 546 | Tuple[str, List[str], List[str]] if revved, or None |
| 547 | 1. Full portage version atom (including rc's, etc) that was revved. |
| 548 | 2. List of files to be `git add`ed. |
| 549 | 3. List of files to be `git rm`ed. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 550 | """ |
| 551 | def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild): |
| 552 | """Returns True if the new ebuild is redundant. |
| 553 | |
| 554 | This is True if there if the current stable ebuild is the exact same copy |
| 555 | of the new one. |
| 556 | """ |
| 557 | if not stable_ebuild: |
| 558 | return False |
| 559 | |
David Riley | 676f540 | 2016-02-12 17:24:23 -0800 | [diff] [blame] | 560 | if stable_candidate.version_no_rev == new_ebuild.version_no_rev: |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 561 | return filecmp.cmp( |
| 562 | new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False) |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 563 | return False |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 564 | |
| 565 | # Case where we have the last stable candidate with same version just rev. |
David Riley | 676f540 | 2016-02-12 17:24:23 -0800 | [diff] [blame] | 566 | if stable_candidate and stable_candidate.version_no_rev == android_version: |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 567 | new_ebuild_path = '%s-r%d.ebuild' % ( |
| 568 | stable_candidate.ebuild_path_no_revision, |
| 569 | stable_candidate.current_revision + 1) |
| 570 | else: |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 571 | pf = '%s-%s-r1' % (android_package, android_version) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 572 | new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf) |
| 573 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 574 | variables = {'BASE_URL': arc_bucket_url} |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 575 | for build, (target, _) in build_targets.items(): |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 576 | variables[build + '_TARGET'] = '%s-%s' % (build_branch, target) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 577 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 578 | variables.update(UpdateDataCollectorArtifacts( |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 579 | android_version, runtime_artifacts_bucket_url, build_branch)) |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 580 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 581 | portage_util.EBuild.MarkAsStable( |
| 582 | unstable_ebuild.ebuild_path, new_ebuild_path, |
| 583 | variables, make_stable=True) |
| 584 | new_ebuild = portage_util.EBuild(new_ebuild_path) |
| 585 | |
| 586 | # Determine whether this is ebuild is redundant. |
| 587 | if IsTheNewEBuildRedundant(new_ebuild, stable_candidate): |
| 588 | msg = 'Previous ebuild with same version found and ebuild is redundant.' |
| 589 | logging.info(msg) |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 590 | logging.PrintBuildbotStepText('%s %s not revved' |
| 591 | % (stable_candidate.pkgname, |
| 592 | stable_candidate.version)) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 593 | osutils.SafeUnlink(new_ebuild_path) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 594 | return None |
| 595 | |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 596 | # PFQ runs should always be able to find a stable candidate. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 597 | if stable_candidate: |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 598 | PrintUprevMetadata(build_branch, stable_candidate, new_ebuild) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 599 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 600 | files_to_add = [new_ebuild_path] |
| 601 | files_to_remove = [] |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 602 | if stable_candidate and not stable_candidate.IsSticky(): |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 603 | osutils.SafeUnlink(stable_candidate.ebuild_path) |
| 604 | files_to_remove.append(stable_candidate.ebuild_path) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 605 | |
| 606 | # Update ebuild manifest and git add it. |
| 607 | gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force'] |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 608 | cros_build_lib.run(gen_manifest_cmd, extra_env=None, print_cmd=True) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 609 | files_to_add.append('Manifest') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 610 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 611 | return ( |
| 612 | f'{new_ebuild.package}-{new_ebuild.version}', |
| 613 | files_to_add, |
| 614 | files_to_remove, |
| 615 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 616 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 617 | |
| 618 | def _PrepareGitBranch(tracking_branch, overlay_dir): |
| 619 | """Prepares a git branch for the uprev commit.""" |
| 620 | tracking_branch = f'remotes/m/{os.path.basename(tracking_branch)}' |
| 621 | existing_branch = git.GetCurrentBranchOrId(overlay_dir) |
| 622 | work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH, |
| 623 | tracking_branch, |
| 624 | overlay_dir) |
| 625 | work_branch.CreateBranch() |
| 626 | |
| 627 | # In the case of uprevving overlays that have patches applied to them, |
| 628 | # include the patched changes in the stabilizing branch. |
| 629 | if existing_branch: |
| 630 | git.RunGit(overlay_dir, ['rebase', existing_branch]) |
| 631 | |
| 632 | |
| 633 | def _CommitChange(message, android_package_dir, files_to_add, files_to_remove): |
| 634 | """Commit changes to git with list of files to add/remove.""" |
| 635 | git.RunGit(android_package_dir, ['add', '--'] + files_to_add) |
| 636 | git.RunGit(android_package_dir, ['rm', '--'] + files_to_remove) |
| 637 | |
| 638 | portage_util.EBuild.CommitChange(message, android_package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 639 | |
| 640 | |
| 641 | def GetParser(): |
| 642 | """Creates the argument parser.""" |
| 643 | parser = commandline.ArgumentParser() |
| 644 | parser.add_argument('-b', '--boards') |
| 645 | parser.add_argument('--android_bucket_url', |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 646 | default=constants.ANDROID_BUCKET_URL, |
| 647 | type='gs_path') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 648 | parser.add_argument('--android_build_branch', |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 649 | required=True, |
| 650 | help='Android branch to import from. ' |
| 651 | 'Ex: git_mnc-dr-arc-dev') |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 652 | parser.add_argument('--android_package', |
| 653 | default=constants.ANDROID_PACKAGE_NAME) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 654 | parser.add_argument('--arc_bucket_url', |
| 655 | default=constants.ARC_BUCKET_URL, |
| 656 | type='gs_path') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 657 | parser.add_argument('-f', '--force_version', |
| 658 | help='Android build id to use') |
| 659 | parser.add_argument('-s', '--srcroot', |
| 660 | default=os.path.join(os.environ['HOME'], 'trunk', 'src'), |
| 661 | help='Path to the src directory') |
| 662 | parser.add_argument('-t', '--tracking_branch', default='cros/master', |
| 663 | help='Branch we are tracking changes against') |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 664 | parser.add_argument('--runtime_artifacts_bucket_url', |
| 665 | default=_RUNTIME_ARTIFACTS_BUCKET_URL, |
| 666 | type='gs_path') |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 667 | parser.add_argument('--skip_commit', |
| 668 | action='store_true', |
| 669 | help='Skip commiting uprev changes to git') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 670 | return parser |
| 671 | |
| 672 | |
| 673 | def main(argv): |
Hidehiko Abe | c9ecf26 | 2017-07-05 15:17:41 +0900 | [diff] [blame] | 674 | logging.EnableBuildbotMarkers() |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 675 | parser = GetParser() |
| 676 | options = parser.parse_args(argv) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 677 | # HACK(b/179456416): Skip git operations if tracking_branch is empty, which |
| 678 | # happens when called from CQ builders. |
| 679 | # TODO(b/179456416): Make CQ builders specify the --skip_commit option. |
| 680 | if not options.tracking_branch: |
| 681 | options.skip_commit = True |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 682 | options.Freeze() |
| 683 | |
| 684 | overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot}) |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 685 | android_package_dir = os.path.join( |
| 686 | overlay_dir, |
| 687 | portage_util.GetFullAndroidPortagePackageName(options.android_package)) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 688 | version_to_uprev = None |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 689 | |
| 690 | (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 691 | acls = MakeAclDict(android_package_dir) |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 692 | build_targets = MakeBuildTargetDict(options.android_package, |
| 693 | options.android_build_branch) |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 694 | # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt). |
| 695 | version_to_uprev = MirrorArtifacts(options.android_bucket_url, |
| 696 | options.android_build_branch, |
| 697 | options.arc_bucket_url, acls, |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 698 | build_targets, |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 699 | options.force_version) |
| 700 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 701 | stable_candidate = portage_util.BestEBuild(stable_ebuilds) |
| 702 | |
| 703 | if stable_candidate: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 704 | logging.info('Stable candidate found %s', stable_candidate.version) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 705 | else: |
| 706 | logging.info('No stable candidate found.') |
| 707 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 708 | if not options.skip_commit: |
| 709 | _PrepareGitBranch(options.tracking_branch, overlay_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 710 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 711 | revved = MarkAndroidEBuildAsStable( |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 712 | stable_candidate, unstable_ebuild, options.android_package, |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 713 | version_to_uprev, android_package_dir, |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 714 | options.android_build_branch, options.arc_bucket_url, |
| 715 | options.runtime_artifacts_bucket_url, build_targets) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame^] | 716 | |
| 717 | if revved: |
| 718 | android_version_atom, files_to_add, files_to_remove = revved |
| 719 | if not options.skip_commit: |
| 720 | _CommitChange( |
| 721 | _GIT_COMMIT_MESSAGE % {'android_package': options.android_package, |
| 722 | 'android_version': version_to_uprev}, |
| 723 | android_package_dir, |
| 724 | files_to_add, |
| 725 | files_to_remove, |
| 726 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 727 | if options.boards: |
| 728 | cros_mark_as_stable.CleanStalePackages(options.srcroot, |
| 729 | options.boards.split(':'), |
| 730 | [android_version_atom]) |
| 731 | |
| 732 | # Explicit print to communicate to caller. |
| 733 | print('ANDROID_VERSION_ATOM=%s' % android_version_atom) |