Mike Frysinger | e58c0e2 | 2017-10-04 15:43:30 -0400 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 2 | # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """This module uprevs Android for cbuildbot. |
| 7 | |
| 8 | After calling, it prints outs ANDROID_VERSION_ATOM=(version atom string). A |
| 9 | caller could then use this atom with emerge to build the newly uprevved version |
| 10 | of Android e.g. |
| 11 | |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 12 | ./cros_mark_android_as_stable \ |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 13 | --android_build_branch=git_pi-arc \ |
| 14 | --android_package=android-container-pi |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 15 | |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 16 | Returns chromeos-base/android-container-pi-6417892-r1 |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 17 | |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 18 | emerge-eve =chromeos-base/android-container-pi-6417892-r1 |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 19 | """ |
| 20 | |
| 21 | from __future__ import print_function |
| 22 | |
| 23 | import filecmp |
| 24 | import glob |
Junichi Uekawa | d21f94d | 2020-07-27 15:50:05 +0900 | [diff] [blame] | 25 | import json |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 26 | import os |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 27 | import re |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 28 | import time |
Mike Frysinger | 00a0229 | 2020-04-19 06:28:03 -0400 | [diff] [blame] | 29 | import sys |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 30 | |
Aviv Keshet | b7519e1 | 2016-10-04 00:50:00 -0700 | [diff] [blame] | 31 | from chromite.lib import constants |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 32 | from chromite.lib import commandline |
| 33 | from chromite.lib import cros_build_lib |
| 34 | from chromite.lib import cros_logging as logging |
| 35 | from chromite.lib import git |
| 36 | from chromite.lib import gs |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 37 | from chromite.lib import osutils |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 38 | from chromite.lib import portage_util |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 39 | from chromite.lib import repo_util |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 40 | from chromite.scripts import cros_mark_as_stable |
| 41 | |
| 42 | |
Mike Frysinger | 00a0229 | 2020-04-19 06:28:03 -0400 | [diff] [blame] | 43 | assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' |
| 44 | |
| 45 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 46 | # Dir where all the action happens. |
| 47 | _OVERLAY_DIR = '%(srcroot)s/private-overlays/project-cheets-private/' |
| 48 | |
Junichi Uekawa | 6d61ab0 | 2020-04-15 14:52:28 +0900 | [diff] [blame] | 49 | _GIT_COMMIT_MESSAGE = """Marking latest for %(android_package)s ebuild with \ |
| 50 | version %(android_version)s as stable. |
| 51 | |
| 52 | BUG=None |
| 53 | TEST=CQ |
| 54 | """ |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 55 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 56 | _RUNTIME_ARTIFACTS_BUCKET_URL = 'gs://chromeos-arc-images/runtime_artifacts' |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 57 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 58 | def IsBuildIdValid(bucket_url, build_branch, build_id, targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 59 | """Checks that a specific build_id is valid. |
| 60 | |
| 61 | Looks for that build_id for all builds. Confirms that the subpath can |
| 62 | be found and that the zip file is present in that subdirectory. |
| 63 | |
| 64 | Args: |
| 65 | bucket_url: URL of Android build gs bucket |
| 66 | build_branch: branch of Android builds |
| 67 | build_id: A string. The Android build id number to check. |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 68 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 69 | pair. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 70 | |
| 71 | Returns: |
| 72 | Returns subpaths dictionary if build_id is valid. |
| 73 | None if the build_id is not valid. |
| 74 | """ |
| 75 | gs_context = gs.GSContext() |
| 76 | subpaths_dict = {} |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 77 | for build, (target, _) in targets.items(): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 78 | build_dir = '%s-%s' % (build_branch, target) |
| 79 | build_id_path = os.path.join(bucket_url, build_dir, build_id) |
| 80 | |
| 81 | # Find name of subpath. |
| 82 | try: |
| 83 | subpaths = gs_context.List(build_id_path) |
| 84 | except gs.GSNoSuchKey: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 85 | logging.warning( |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 86 | 'Directory [%s] does not contain any subpath, ignoring it.', |
| 87 | build_id_path) |
| 88 | return None |
| 89 | if len(subpaths) > 1: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 90 | logging.warning( |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 91 | 'Directory [%s] contains more than one subpath, ignoring it.', |
| 92 | build_id_path) |
| 93 | return None |
| 94 | |
| 95 | subpath_dir = subpaths[0].url.rstrip('/') |
| 96 | subpath_name = os.path.basename(subpath_dir) |
| 97 | |
| 98 | # Look for a zipfile ending in the build_id number. |
| 99 | try: |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 100 | gs_context.List(subpath_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 101 | except gs.GSNoSuchKey: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 102 | logging.warning( |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 103 | 'Did not find a file for build id [%s] in directory [%s].', |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 104 | build_id, subpath_dir) |
| 105 | return None |
| 106 | |
| 107 | # Record subpath for the build. |
| 108 | subpaths_dict[build] = subpath_name |
| 109 | |
| 110 | # If we got here, it means we found an appropriate build for all platforms. |
| 111 | return subpaths_dict |
| 112 | |
| 113 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 114 | def GetLatestBuild(bucket_url, build_branch, targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 115 | """Searches the gs bucket for the latest green build. |
| 116 | |
| 117 | Args: |
| 118 | bucket_url: URL of Android build gs bucket |
| 119 | build_branch: branch of Android builds |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 120 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 121 | pair. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 122 | |
| 123 | Returns: |
| 124 | Tuple of (latest version string, subpaths dictionary) |
| 125 | If no latest build can be found, returns None, None |
| 126 | """ |
| 127 | gs_context = gs.GSContext() |
| 128 | common_build_ids = None |
| 129 | # Find builds for each target. |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 130 | for target, _ in targets.values(): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 131 | build_dir = '-'.join((build_branch, target)) |
| 132 | base_path = os.path.join(bucket_url, build_dir) |
| 133 | build_ids = [] |
| 134 | for gs_result in gs_context.List(base_path): |
| 135 | # Remove trailing slashes and get the base name, which is the build_id. |
| 136 | build_id = os.path.basename(gs_result.url.rstrip('/')) |
| 137 | if not build_id.isdigit(): |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 138 | logging.warning('Directory [%s] does not look like a valid build_id.', |
| 139 | gs_result.url) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 140 | continue |
| 141 | build_ids.append(build_id) |
| 142 | |
| 143 | # Update current list of builds. |
| 144 | if common_build_ids is None: |
| 145 | # First run, populate it with the first platform. |
| 146 | common_build_ids = set(build_ids) |
| 147 | else: |
| 148 | # Already populated, find the ones that are common. |
| 149 | common_build_ids.intersection_update(build_ids) |
| 150 | |
| 151 | if common_build_ids is None: |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 152 | logging.warning('Did not find a build_id common to all platforms.') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 153 | return None, None |
| 154 | |
| 155 | # Otherwise, find the most recent one that is valid. |
| 156 | for build_id in sorted(common_build_ids, key=int, reverse=True): |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 157 | subpaths = IsBuildIdValid(bucket_url, build_branch, build_id, targets) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 158 | if subpaths: |
| 159 | return build_id, subpaths |
| 160 | |
| 161 | # If not found, no build_id is valid. |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 162 | logging.warning('Did not find a build_id valid on all platforms.') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 163 | return None, None |
| 164 | |
| 165 | |
| 166 | def FindAndroidCandidates(package_dir): |
| 167 | """Return a tuple of Android's unstable ebuild and stable ebuilds. |
| 168 | |
| 169 | Args: |
| 170 | package_dir: The path to where the package ebuild is stored. |
| 171 | |
| 172 | Returns: |
| 173 | Tuple [unstable_ebuild, stable_ebuilds]. |
| 174 | |
| 175 | Raises: |
| 176 | Exception: if no unstable ebuild exists for Android. |
| 177 | """ |
| 178 | stable_ebuilds = [] |
| 179 | unstable_ebuilds = [] |
| 180 | for path in glob.glob(os.path.join(package_dir, '*.ebuild')): |
| 181 | ebuild = portage_util.EBuild(path) |
| 182 | if ebuild.version == '9999': |
| 183 | unstable_ebuilds.append(ebuild) |
| 184 | else: |
| 185 | stable_ebuilds.append(ebuild) |
| 186 | |
| 187 | # Apply some sanity checks. |
| 188 | if not unstable_ebuilds: |
| 189 | raise Exception('Missing 9999 ebuild for %s' % package_dir) |
| 190 | if not stable_ebuilds: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 191 | logging.warning('Missing stable ebuild for %s', package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 192 | |
| 193 | return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds |
| 194 | |
| 195 | |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 196 | def _GetArcBasename(build, basename): |
| 197 | """Tweaks filenames between Android bucket and ARC++ bucket. |
| 198 | |
| 199 | Android builders create build artifacts with the same name for -user and |
| 200 | -userdebug builds, which breaks the android-container ebuild (b/33072485). |
| 201 | When copying the artifacts from the Android bucket to the ARC++ bucket some |
| 202 | artifacts will be renamed from the usual pattern |
| 203 | *cheets_${ARCH}-target_files-S{VERSION}.zip to |
| 204 | cheets_${BUILD_NAME}-target_files-S{VERSION}.zip which will typically look |
| 205 | like cheets_(${LABEL})*${ARCH}_userdebug-target_files-S{VERSION}.zip. |
| 206 | |
| 207 | Args: |
| 208 | build: the build being mirrored, e.g. 'X86', 'ARM', 'X86_USERDEBUG'. |
| 209 | basename: the basename of the artifact to copy. |
| 210 | |
| 211 | Returns: |
| 212 | The basename of the destination. |
| 213 | """ |
| 214 | if build not in constants.ARC_BUILDS_NEED_ARTIFACTS_RENAMED: |
| 215 | return basename |
Yūki Ishii | ef1ada9 | 2018-03-27 15:46:15 +0900 | [diff] [blame] | 216 | if basename in constants.ARC_ARTIFACTS_RENAME_NOT_NEEDED: |
| 217 | return basename |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 218 | to_discard, sep, to_keep = basename.partition('-') |
| 219 | if not sep: |
| 220 | logging.error(('Build %s: Could not find separator "-" in artifact' |
| 221 | ' basename %s'), build, basename) |
| 222 | return basename |
Bernie Thompson | 63ed561 | 2017-08-16 12:27:34 -0700 | [diff] [blame] | 223 | if 'cheets_' in to_discard: |
| 224 | return 'cheets_%s-%s' % (build.lower(), to_keep) |
| 225 | elif 'bertha_' in to_discard: |
| 226 | return 'bertha_%s-%s' % (build.lower(), to_keep) |
| 227 | logging.error('Build %s: Unexpected artifact basename %s', |
| 228 | build, basename) |
| 229 | return basename |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 230 | |
| 231 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 232 | def CopyToArcBucket(android_bucket_url, build_branch, build_id, subpaths, |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 233 | targets, arc_bucket_url, acls): |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 234 | """Copies from source Android bucket to ARC++ specific bucket. |
| 235 | |
| 236 | Copies each build to the ARC bucket eliminating the subpath. |
| 237 | Applies build specific ACLs for each file. |
| 238 | |
| 239 | Args: |
| 240 | android_bucket_url: URL of Android build gs bucket |
| 241 | build_branch: branch of Android builds |
| 242 | build_id: A string. The Android build id number to check. |
| 243 | subpaths: Subpath dictionary for each build to copy. |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 244 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 245 | pair. |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 246 | arc_bucket_url: URL of the target ARC build gs bucket |
| 247 | acls: ACLs dictionary for each build to copy. |
| 248 | """ |
| 249 | gs_context = gs.GSContext() |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 250 | for build, subpath in subpaths.items(): |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 251 | target, pattern = targets[build] |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 252 | build_dir = '%s-%s' % (build_branch, target) |
| 253 | android_dir = os.path.join(android_bucket_url, build_dir, build_id, subpath) |
| 254 | arc_dir = os.path.join(arc_bucket_url, build_dir, build_id) |
| 255 | |
Hidehiko Abe | 12727dd | 2016-05-27 23:23:45 +0900 | [diff] [blame] | 256 | # Copy all target files from android_dir to arc_dir, setting ACLs. |
| 257 | for targetfile in gs_context.List(android_dir): |
| 258 | if re.search(pattern, targetfile.url): |
| 259 | basename = os.path.basename(targetfile.url) |
Nicolas Norvez | b08f54d | 2016-12-05 17:58:54 -0800 | [diff] [blame] | 260 | arc_path = os.path.join(arc_dir, _GetArcBasename(build, basename)) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 261 | acl = acls[build] |
| 262 | needs_copy = True |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 263 | retry_count = 2 |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 264 | |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 265 | # Retry in case race condition when several boards trying to copy the |
| 266 | # same resource |
| 267 | while True: |
| 268 | # Check a pre-existing file with the original source. |
| 269 | if gs_context.Exists(arc_path): |
| 270 | if (gs_context.Stat(targetfile.url).hash_crc32c != |
| 271 | gs_context.Stat(arc_path).hash_crc32c): |
Mike Frysinger | 968c114 | 2020-05-09 00:37:56 -0400 | [diff] [blame] | 272 | logging.warning('Removing incorrect file %s', arc_path) |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 273 | gs_context.Remove(arc_path) |
| 274 | else: |
| 275 | logging.info('Skipping already copied file %s', arc_path) |
| 276 | needs_copy = False |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 277 | |
khmel@google.com | 96c193e | 2018-05-10 14:00:38 -0700 | [diff] [blame] | 278 | # Copy if necessary, and set the ACL unconditionally. |
| 279 | # The Stat() call above doesn't verify the ACL is correct and |
| 280 | # the ChangeACL should be relatively cheap compared to the copy. |
| 281 | # This covers the following caes: |
| 282 | # - handling an interrupted copy from a previous run. |
| 283 | # - rerunning the copy in case one of the googlestorage_acl_X.txt |
| 284 | # files changes (e.g. we add a new variant which reuses a build). |
| 285 | if needs_copy: |
| 286 | logging.info('Copying %s -> %s (acl %s)', |
| 287 | targetfile.url, arc_path, acl) |
| 288 | try: |
| 289 | gs_context.Copy(targetfile.url, arc_path, version=0) |
| 290 | except gs.GSContextPreconditionFailed as error: |
| 291 | if not retry_count: |
| 292 | raise error |
| 293 | # Retry one more time after a short delay |
| 294 | logging.warning('Will retry copying %s -> %s', |
| 295 | targetfile.url, arc_path) |
| 296 | time.sleep(5) |
| 297 | retry_count = retry_count - 1 |
| 298 | continue |
| 299 | gs_context.ChangeACL(arc_path, acl_args_file=acl) |
| 300 | break |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 301 | |
| 302 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 303 | def MirrorArtifacts(android_bucket_url, android_build_branch, arc_bucket_url, |
| 304 | acls, targets, version=None): |
| 305 | """Mirrors artifacts from Android bucket to ARC bucket. |
| 306 | |
| 307 | First, this function identifies which build version should be copied, |
| 308 | if not given. Please see GetLatestBuild() and IsBuildIdValid() for details. |
| 309 | |
| 310 | On build version identified, then copies target artifacts to the ARC bucket, |
| 311 | with setting ACLs. |
| 312 | |
| 313 | Args: |
| 314 | android_bucket_url: URL of Android build gs bucket |
| 315 | android_build_branch: branch of Android builds |
| 316 | arc_bucket_url: URL of the target ARC build gs bucket |
| 317 | acls: ACLs dictionary for each build to copy. |
| 318 | targets: Dict from build key to (targe build suffix, artifact file pattern) |
| 319 | pair. |
| 320 | version: (optional) A string. The Android build id number to check. |
| 321 | If not passed, detect latest good build version. |
| 322 | |
| 323 | Returns: |
| 324 | Mirrored version. |
| 325 | """ |
| 326 | if version: |
| 327 | subpaths = IsBuildIdValid( |
| 328 | android_bucket_url, android_build_branch, version, targets) |
| 329 | if not subpaths: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 330 | logging.error('Requested build %s is not valid', version) |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 331 | else: |
| 332 | version, subpaths = GetLatestBuild( |
| 333 | android_bucket_url, android_build_branch, targets) |
| 334 | |
| 335 | CopyToArcBucket(android_bucket_url, android_build_branch, version, subpaths, |
| 336 | targets, arc_bucket_url, acls) |
khmel@google.com | 778a1cd | 2018-04-13 11:11:58 -0700 | [diff] [blame] | 337 | |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 338 | return version |
| 339 | |
| 340 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 341 | def MakeAclDict(package_dir): |
| 342 | """Creates a dictionary of acl files for each build type. |
| 343 | |
| 344 | Args: |
| 345 | package_dir: The path to where the package acl files are stored. |
| 346 | |
| 347 | Returns: |
| 348 | Returns acls dictionary. |
| 349 | """ |
| 350 | return dict( |
| 351 | (k, os.path.join(package_dir, v)) |
| 352 | for k, v in constants.ARC_BUCKET_ACLS.items() |
| 353 | ) |
| 354 | |
| 355 | |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 356 | def MakeBuildTargetDict(package_name, build_branch): |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 357 | """Creates a dictionary of build targets. |
| 358 | |
Bernie Thompson | 63ed561 | 2017-08-16 12:27:34 -0700 | [diff] [blame] | 359 | Not all targets are common between branches, for example |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 360 | sdk_google_cheets_x86 only exists on N. |
| 361 | This generates a dictionary listing the available build targets for a |
| 362 | specific branch. |
| 363 | |
| 364 | Args: |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 365 | package_name: package name of chromeos arc package. |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 366 | build_branch: branch of Android builds. |
| 367 | |
| 368 | Returns: |
| 369 | Returns build target dictionary. |
| 370 | |
| 371 | Raises: |
| 372 | ValueError: if the Android build branch is invalid. |
| 373 | """ |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 374 | if constants.ANDROID_CONTAINER_PACKAGE_KEYWORD in package_name: |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 375 | target_list = { |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 376 | constants.ANDROID_PI_BUILD_BRANCH: |
| 377 | constants.ANDROID_PI_BUILD_TARGETS, |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 378 | } |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 379 | elif constants.ANDROID_VM_PACKAGE_KEYWORD in package_name: |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 380 | target_list = { |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 381 | constants.ANDROID_VMMST_BUILD_BRANCH: |
| 382 | constants.ANDROID_VMMST_BUILD_TARGETS, |
| 383 | constants.ANDROID_VMRVC_BUILD_BRANCH: |
| 384 | constants.ANDROID_VMRVC_BUILD_TARGETS, |
| 385 | } |
| 386 | else: |
| 387 | raise ValueError('Unknown package: %s' % package_name) |
| 388 | target = target_list.get(build_branch) |
| 389 | if not target: |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 390 | raise ValueError('Unknown branch: %s' % build_branch) |
Federico 'Morg' Pareschi | 041ee65 | 2020-03-10 15:09:42 +0900 | [diff] [blame] | 391 | return target |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 392 | |
| 393 | |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 394 | def PrintUprevMetadata(build_branch, stable_candidate, new_ebuild): |
| 395 | """Shows metadata on buildbot page at UprevAndroid step. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 396 | |
| 397 | Args: |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 398 | build_branch: The branch of Android builds. |
| 399 | stable_candidate: The existing stable ebuild. |
| 400 | new_ebuild: The newly written ebuild. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 401 | """ |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 402 | # Examples: |
| 403 | # "android-container-pi revved 6461825-r1 -> 6468247-r1" |
| 404 | # "android-container-pi revved 6461825-r1 -> 6461825-r2 (ebuild update only)" |
| 405 | msg = '%s revved %s -> %s' % (stable_candidate.pkgname, |
| 406 | stable_candidate.version, |
| 407 | new_ebuild.version) |
| 408 | |
| 409 | old_android = stable_candidate.version_no_rev |
| 410 | new_android = new_ebuild.version_no_rev |
| 411 | |
| 412 | if old_android == new_android: |
| 413 | msg += ' (ebuild update only)' |
| 414 | else: |
| 415 | ab_link = ('https://android-build.googleplex.com' |
| 416 | '/builds/%s/branches/%s/cls?end=%s' |
| 417 | % (new_android, build_branch, old_android)) |
| 418 | logging.PrintBuildbotLink('Android changelog', ab_link) |
| 419 | |
| 420 | logging.PrintBuildbotStepText(msg) |
Junichi Uekawa | d21f94d | 2020-07-27 15:50:05 +0900 | [diff] [blame] | 421 | logging.PrintKitchenSetBuildProperty('android_uprev', json.dumps({ |
| 422 | 'branch': build_branch, |
| 423 | 'new': new_ebuild.version, |
| 424 | 'old': stable_candidate.version, |
| 425 | 'pkgname': stable_candidate.pkgname, |
| 426 | })) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 427 | |
| 428 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 429 | def FindDataCollectorArtifacts(gs_context, |
| 430 | android_version, |
| 431 | runtime_artifacts_bucket_url, |
| 432 | version_reference): |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 433 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
| 434 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 435 | This is used from UpdateDataCollectorArtifacts in order to check the |
| 436 | particular version. |
| 437 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 438 | Args: |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 439 | gs_context: context to execute gsutil |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 440 | android_version: The \d+ build id of Android. |
| 441 | runtime_artifacts_bucket_url: root of runtime artifacts |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 442 | build_branch: build branch. Used to determine the pinned version if exists. |
| 443 | version_reference: which version to use as a reference. Could be '${PV}' in |
| 444 | case version of data collector artifacts matches the |
| 445 | Android version or direct version in case of override. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 446 | |
| 447 | Returns: |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 448 | dictionary with filled ebuild variables. This dictionary is empty in case |
| 449 | no artificats are found. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 450 | """ |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 451 | variables = {} |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 452 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 453 | buckets = ['ureadahead_pack', 'gms_core_cache'] |
| 454 | archs = ['arm', 'arm64', 'x86', 'x86_64'] |
| 455 | build_types = ['user', 'userdebug'] |
| 456 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 457 | for bucket in buckets: |
| 458 | for arch in archs: |
| 459 | for build_type in build_types: |
| 460 | path = (f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_' |
Yury Khmel | e1b7440 | 2020-05-18 08:41:35 -0700 | [diff] [blame] | 461 | f'{android_version}.tar') |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 462 | if gs_context.Exists(path): |
| 463 | variables[(f'{arch}_{build_type}_{bucket}').upper()] = ( |
| 464 | f'{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}_' |
| 465 | f'{version_reference}.tar') |
| 466 | |
| 467 | return variables |
| 468 | |
| 469 | |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 470 | def UpdateDataCollectorArtifacts(android_version, |
| 471 | runtime_artifacts_bucket_url, |
| 472 | build_branch): |
| 473 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
| 474 | |
| 475 | This verifies default android version. In case artificts are not found for |
| 476 | default Android version it tries to find artifacts for pinned version. If |
| 477 | pinned version is provided, it is required artifacts exist for the pinned |
| 478 | version. |
| 479 | |
| 480 | Args: |
| 481 | android_version: The \d+ build id of Android. |
| 482 | runtime_artifacts_bucket_url: root of runtime artifacts |
| 483 | build_branch: build branch. Used to determine the pinned version if exists. |
| 484 | |
| 485 | Returns: |
| 486 | dictionary with filled ebuild variables. |
| 487 | """ |
| 488 | |
| 489 | gs_context = gs.GSContext() |
| 490 | # Check the existing version. If we find any artifacts, use them. |
| 491 | variables = FindDataCollectorArtifacts(gs_context, |
| 492 | android_version, |
| 493 | runtime_artifacts_bucket_url, |
| 494 | '${PV}') |
| 495 | if variables: |
| 496 | # Data artificts were found. |
| 497 | return variables |
| 498 | |
| 499 | # Check pinned version for the current branch. |
| 500 | pin_path = (f'{runtime_artifacts_bucket_url}/{build_branch}_pin_version') |
| 501 | if not gs_context.Exists(pin_path): |
| 502 | # No pinned version. |
| 503 | logging.warning( |
| 504 | 'No data collector artifacts were found for %s', |
| 505 | android_version) |
| 506 | return variables |
| 507 | |
| 508 | pin_version = gs_context.Cat(pin_path, encoding='utf-8').rstrip() |
| 509 | logging.info('Pinned version %s overrides %s', |
| 510 | pin_version, android_version) |
| 511 | variables = FindDataCollectorArtifacts(gs_context, |
| 512 | pin_version, |
| 513 | runtime_artifacts_bucket_url, |
| 514 | pin_version) |
| 515 | if not variables: |
| 516 | # If pin version set it must contain data. |
| 517 | raise Exception('Pinned version %s:%s does not contain artificats' % ( |
| 518 | build_branch, pin_version)) |
| 519 | |
| 520 | return variables |
| 521 | |
| 522 | |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 523 | def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild, |
| 524 | android_package, android_version, package_dir, |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 525 | build_branch, arc_bucket_url, |
| 526 | runtime_artifacts_bucket_url, |
| 527 | build_targets): |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 528 | r"""Uprevs the Android ebuild. |
| 529 | |
| 530 | This is the main function that uprevs from a stable candidate |
| 531 | to its new version. |
| 532 | |
| 533 | Args: |
| 534 | stable_candidate: ebuild that corresponds to the stable ebuild we are |
| 535 | revving from. If None, builds the a new ebuild given the version |
| 536 | with revision set to 1. |
| 537 | unstable_ebuild: ebuild corresponding to the unstable ebuild for Android. |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 538 | android_package: android package name. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 539 | android_version: The \d+ build id of Android. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 540 | package_dir: Path to the android-container package dir. |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 541 | build_branch: branch of Android builds. |
| 542 | arc_bucket_url: URL of the target ARC build gs bucket. |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 543 | runtime_artifacts_bucket_url: root of runtime artifacts |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 544 | build_targets: build targets for this particular Android branch. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 545 | |
| 546 | Returns: |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 547 | Tuple[str, List[str], List[str]] if revved, or None |
| 548 | 1. Full portage version atom (including rc's, etc) that was revved. |
| 549 | 2. List of files to be `git add`ed. |
| 550 | 3. List of files to be `git rm`ed. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 551 | """ |
| 552 | def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild): |
| 553 | """Returns True if the new ebuild is redundant. |
| 554 | |
| 555 | This is True if there if the current stable ebuild is the exact same copy |
| 556 | of the new one. |
| 557 | """ |
| 558 | if not stable_ebuild: |
| 559 | return False |
| 560 | |
David Riley | 676f540 | 2016-02-12 17:24:23 -0800 | [diff] [blame] | 561 | if stable_candidate.version_no_rev == new_ebuild.version_no_rev: |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 562 | return filecmp.cmp( |
| 563 | new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False) |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 564 | return False |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 565 | |
| 566 | # Case where we have the last stable candidate with same version just rev. |
David Riley | 676f540 | 2016-02-12 17:24:23 -0800 | [diff] [blame] | 567 | if stable_candidate and stable_candidate.version_no_rev == android_version: |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 568 | new_ebuild_path = '%s-r%d.ebuild' % ( |
| 569 | stable_candidate.ebuild_path_no_revision, |
| 570 | stable_candidate.current_revision + 1) |
| 571 | else: |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 572 | pf = '%s-%s-r1' % (android_package, android_version) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 573 | new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf) |
| 574 | |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 575 | variables = {'BASE_URL': arc_bucket_url} |
Mike Frysinger | 0bdbc10 | 2019-06-13 15:27:29 -0400 | [diff] [blame] | 576 | for build, (target, _) in build_targets.items(): |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 577 | variables[build + '_TARGET'] = '%s-%s' % (build_branch, target) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 578 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 579 | variables.update(UpdateDataCollectorArtifacts( |
Yury Khmel | b009aeb | 2020-08-19 19:40:00 -0700 | [diff] [blame] | 580 | android_version, runtime_artifacts_bucket_url, build_branch)) |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 581 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 582 | portage_util.EBuild.MarkAsStable( |
| 583 | unstable_ebuild.ebuild_path, new_ebuild_path, |
| 584 | variables, make_stable=True) |
| 585 | new_ebuild = portage_util.EBuild(new_ebuild_path) |
| 586 | |
| 587 | # Determine whether this is ebuild is redundant. |
| 588 | if IsTheNewEBuildRedundant(new_ebuild, stable_candidate): |
| 589 | msg = 'Previous ebuild with same version found and ebuild is redundant.' |
| 590 | logging.info(msg) |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 591 | logging.PrintBuildbotStepText('%s %s not revved' |
| 592 | % (stable_candidate.pkgname, |
| 593 | stable_candidate.version)) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 594 | osutils.SafeUnlink(new_ebuild_path) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 595 | return None |
| 596 | |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 597 | # PFQ runs should always be able to find a stable candidate. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 598 | if stable_candidate: |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 599 | PrintUprevMetadata(build_branch, stable_candidate, new_ebuild) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 600 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 601 | files_to_add = [new_ebuild_path] |
| 602 | files_to_remove = [] |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 603 | if stable_candidate and not stable_candidate.IsSticky(): |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 604 | osutils.SafeUnlink(stable_candidate.ebuild_path) |
| 605 | files_to_remove.append(stable_candidate.ebuild_path) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 606 | |
| 607 | # Update ebuild manifest and git add it. |
| 608 | gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force'] |
Mike Frysinger | 45602c7 | 2019-09-22 02:15:11 -0400 | [diff] [blame] | 609 | cros_build_lib.run(gen_manifest_cmd, extra_env=None, print_cmd=True) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 610 | files_to_add.append('Manifest') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 611 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 612 | return ( |
| 613 | f'{new_ebuild.package}-{new_ebuild.version}', |
| 614 | files_to_add, |
| 615 | files_to_remove, |
| 616 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 617 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 618 | |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 619 | def _PrepareGitBranch(overlay_dir): |
| 620 | """Prepares a git branch for the uprev commit. |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 621 | |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 622 | If the overlay project is currently on a branch (e.g. patches are being |
| 623 | applied), rebase the new branch on top of it. |
| 624 | |
| 625 | Args: |
| 626 | overlay_dir: The overlay directory. |
| 627 | """ |
| 628 | existing_branch = git.GetCurrentBranch(overlay_dir) |
| 629 | repo_util.Repository.MustFind(overlay_dir).StartBranch( |
| 630 | constants.STABLE_EBUILD_BRANCH, projects=['.'], cwd=overlay_dir) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 631 | if existing_branch: |
| 632 | git.RunGit(overlay_dir, ['rebase', existing_branch]) |
| 633 | |
| 634 | |
| 635 | def _CommitChange(message, android_package_dir, files_to_add, files_to_remove): |
| 636 | """Commit changes to git with list of files to add/remove.""" |
| 637 | git.RunGit(android_package_dir, ['add', '--'] + files_to_add) |
| 638 | git.RunGit(android_package_dir, ['rm', '--'] + files_to_remove) |
| 639 | |
| 640 | portage_util.EBuild.CommitChange(message, android_package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 641 | |
| 642 | |
| 643 | def GetParser(): |
| 644 | """Creates the argument parser.""" |
| 645 | parser = commandline.ArgumentParser() |
| 646 | parser.add_argument('-b', '--boards') |
| 647 | parser.add_argument('--android_bucket_url', |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 648 | default=constants.ANDROID_BUCKET_URL, |
| 649 | type='gs_path') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 650 | parser.add_argument('--android_build_branch', |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 651 | required=True, |
| 652 | help='Android branch to import from. ' |
| 653 | 'Ex: git_mnc-dr-arc-dev') |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 654 | parser.add_argument('--android_package', |
| 655 | default=constants.ANDROID_PACKAGE_NAME) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 656 | parser.add_argument('--arc_bucket_url', |
| 657 | default=constants.ARC_BUCKET_URL, |
| 658 | type='gs_path') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 659 | parser.add_argument('-f', '--force_version', |
| 660 | help='Android build id to use') |
| 661 | parser.add_argument('-s', '--srcroot', |
| 662 | default=os.path.join(os.environ['HOME'], 'trunk', 'src'), |
| 663 | help='Path to the src directory') |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 664 | # TODO(crbug/1074145): Remove once CQ specifies --skip_commit. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 665 | parser.add_argument('-t', '--tracking_branch', default='cros/master', |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 666 | help='DEPRECATED') |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 667 | parser.add_argument('--runtime_artifacts_bucket_url', |
| 668 | default=_RUNTIME_ARTIFACTS_BUCKET_URL, |
| 669 | type='gs_path') |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 670 | parser.add_argument('--skip_commit', |
| 671 | action='store_true', |
| 672 | help='Skip commiting uprev changes to git') |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 673 | return parser |
| 674 | |
| 675 | |
| 676 | def main(argv): |
Hidehiko Abe | c9ecf26 | 2017-07-05 15:17:41 +0900 | [diff] [blame] | 677 | logging.EnableBuildbotMarkers() |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 678 | parser = GetParser() |
| 679 | options = parser.parse_args(argv) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 680 | # HACK(b/179456416): Skip git operations if tracking_branch is empty, which |
| 681 | # happens when called from CQ builders. |
| 682 | # TODO(b/179456416): Make CQ builders specify the --skip_commit option. |
| 683 | if not options.tracking_branch: |
| 684 | options.skip_commit = True |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 685 | options.Freeze() |
| 686 | |
| 687 | overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot}) |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 688 | android_package_dir = os.path.join( |
| 689 | overlay_dir, |
| 690 | portage_util.GetFullAndroidPortagePackageName(options.android_package)) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 691 | version_to_uprev = None |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 692 | |
| 693 | (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates(android_package_dir) |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 694 | acls = MakeAclDict(android_package_dir) |
Qijiang Fan | 6588cc9 | 2019-11-20 13:26:04 +0900 | [diff] [blame] | 695 | build_targets = MakeBuildTargetDict(options.android_package, |
| 696 | options.android_build_branch) |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 697 | # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt). |
| 698 | version_to_uprev = MirrorArtifacts(options.android_bucket_url, |
| 699 | options.android_build_branch, |
| 700 | options.arc_bucket_url, acls, |
Nicolas Norvez | 4bd854f | 2017-05-23 10:04:45 -0700 | [diff] [blame] | 701 | build_targets, |
Hidehiko Abe | 1ebc25d | 2016-07-28 02:24:37 +0900 | [diff] [blame] | 702 | options.force_version) |
| 703 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 704 | stable_candidate = portage_util.BestEBuild(stable_ebuilds) |
| 705 | |
| 706 | if stable_candidate: |
Lann Martin | ffb9516 | 2018-08-28 12:02:54 -0600 | [diff] [blame] | 707 | logging.info('Stable candidate found %s', stable_candidate.version) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 708 | else: |
| 709 | logging.info('No stable candidate found.') |
| 710 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 711 | if not options.skip_commit: |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame^] | 712 | _PrepareGitBranch(overlay_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 713 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 714 | revved = MarkAndroidEBuildAsStable( |
Hidehiko Abe | 4fd94ae | 2017-01-24 18:59:55 +0900 | [diff] [blame] | 715 | stable_candidate, unstable_ebuild, options.android_package, |
David Riley | 73f00d9 | 2016-02-16 18:54:20 -0800 | [diff] [blame] | 716 | version_to_uprev, android_package_dir, |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 717 | options.android_build_branch, options.arc_bucket_url, |
| 718 | options.runtime_artifacts_bucket_url, build_targets) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 719 | |
| 720 | if revved: |
| 721 | android_version_atom, files_to_add, files_to_remove = revved |
| 722 | if not options.skip_commit: |
| 723 | _CommitChange( |
| 724 | _GIT_COMMIT_MESSAGE % {'android_package': options.android_package, |
| 725 | 'android_version': version_to_uprev}, |
| 726 | android_package_dir, |
| 727 | files_to_add, |
| 728 | files_to_remove, |
| 729 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 730 | if options.boards: |
| 731 | cros_mark_as_stable.CleanStalePackages(options.srcroot, |
| 732 | options.boards.split(':'), |
| 733 | [android_version_atom]) |
| 734 | |
| 735 | # Explicit print to communicate to caller. |
| 736 | print('ANDROID_VERSION_ATOM=%s' % android_version_atom) |