Mike Frysinger | f1ba7ad | 2022-09-12 05:42:57 -0400 | [diff] [blame] | 1 | # Copyright 2016 The ChromiumOS Authors |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """This module uprevs Android for cbuildbot. |
| 6 | |
Shao-Chuan Lee | 84bf9a2 | 2021-11-19 17:42:11 +0900 | [diff] [blame] | 7 | After calling, it prints out a JSON representing the result, with the new |
| 8 | Android version atom string included. A caller could then use this atom with |
| 9 | emerge to build the newly uprevved version of Android e.g. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 10 | |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 11 | ./cros_mark_android_as_stable \ |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 12 | --android_build_branch=git_pi-arc \ |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 13 | --android_package=android-container-pi |
Shuhei Takahashi | 6d02c19 | 2017-04-05 14:01:24 +0900 | [diff] [blame] | 14 | |
Shao-Chuan Lee | 84bf9a2 | 2021-11-19 17:42:11 +0900 | [diff] [blame] | 15 | Returns {"android_atom": "chromeos-base/android-container-pi-6417892-r1"} |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 16 | |
Shao-Chuan Lee | 9c39e0c | 2020-04-24 11:40:34 +0900 | [diff] [blame] | 17 | emerge-eve =chromeos-base/android-container-pi-6417892-r1 |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 18 | """ |
| 19 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 20 | import filecmp |
| 21 | import glob |
Junichi Uekawa | d21f94d | 2020-07-27 15:50:05 +0900 | [diff] [blame] | 22 | import json |
Chris McDonald | b55b703 | 2021-06-17 16:41:32 -0600 | [diff] [blame] | 23 | import logging |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 24 | import os |
| 25 | |
Chris McDonald | b55b703 | 2021-06-17 16:41:32 -0600 | [diff] [blame] | 26 | from chromite.cbuildbot import cbuildbot_alerts |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 27 | from chromite.lib import commandline |
Chris McDonald | b55b703 | 2021-06-17 16:41:32 -0600 | [diff] [blame] | 28 | from chromite.lib import constants |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 29 | from chromite.lib import cros_build_lib |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 30 | from chromite.lib import git |
| 31 | from chromite.lib import gs |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 32 | from chromite.lib import osutils |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 33 | from chromite.lib import portage_util |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame] | 34 | from chromite.lib import repo_util |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 35 | from chromite.scripts import cros_mark_as_stable |
Shao-Chuan Lee | 7fd0ca1 | 2021-03-19 15:57:40 +0900 | [diff] [blame] | 36 | from chromite.service import android |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 37 | from chromite.service import packages |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 38 | |
| 39 | |
| 40 | # Dir where all the action happens. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 41 | _OVERLAY_DIR = "%(srcroot)s/private-overlays/project-cheets-private/" |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 42 | |
Junichi Uekawa | 6d61ab0 | 2020-04-15 14:52:28 +0900 | [diff] [blame] | 43 | _GIT_COMMIT_MESSAGE = """Marking latest for %(android_package)s ebuild with \ |
| 44 | version %(android_version)s as stable. |
| 45 | |
| 46 | BUG=None |
| 47 | TEST=CQ |
| 48 | """ |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 49 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 50 | _RUNTIME_ARTIFACTS_BUCKET_URL = "gs://chromeos-arc-images/runtime_artifacts" |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 51 | |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 52 | |
| 53 | def FindAndroidCandidates(package_dir): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 54 | """Return a tuple of Android's unstable ebuild and stable ebuilds. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 55 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 56 | Args: |
| 57 | package_dir: The path to where the package ebuild is stored. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 58 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 59 | Returns: |
| 60 | Tuple [unstable_ebuild, stable_ebuilds]. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 61 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 62 | Raises: |
| 63 | Exception: if no unstable ebuild exists for Android. |
| 64 | """ |
| 65 | stable_ebuilds = [] |
| 66 | unstable_ebuilds = [] |
| 67 | for path in glob.glob(os.path.join(package_dir, "*.ebuild")): |
| 68 | ebuild = portage_util.EBuild(path) |
| 69 | if ebuild.version == "9999": |
| 70 | unstable_ebuilds.append(ebuild) |
| 71 | else: |
| 72 | stable_ebuilds.append(ebuild) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 73 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 74 | # Apply some confidence checks. |
| 75 | if not unstable_ebuilds: |
| 76 | raise Exception("Missing 9999 ebuild for %s" % package_dir) |
| 77 | if not stable_ebuilds: |
| 78 | logging.warning("Missing stable ebuild for %s", package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 79 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 80 | return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 81 | |
| 82 | |
Shao-Chuan Lee | 085e3d7 | 2020-05-11 16:00:42 +0900 | [diff] [blame] | 83 | def PrintUprevMetadata(build_branch, stable_candidate, new_ebuild): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 84 | """Shows metadata on buildbot page at UprevAndroid step. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 85 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 86 | Args: |
| 87 | build_branch: The branch of Android builds. |
| 88 | stable_candidate: The existing stable ebuild. |
| 89 | new_ebuild: The newly written ebuild. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 90 | """ |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 91 | # Examples: |
| 92 | # "android-container-pi revved 6461825-r1 -> 6468247-r1" |
| 93 | # "android-container-pi revved 6461825-r1 -> 6461825-r2 (ebuild update only)" |
| 94 | msg = "%s revved %s -> %s" % ( |
| 95 | stable_candidate.pkgname, |
| 96 | stable_candidate.version, |
| 97 | new_ebuild.version, |
| 98 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 99 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 100 | old_android = stable_candidate.version_no_rev |
| 101 | new_android = new_ebuild.version_no_rev |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 102 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 103 | if old_android == new_android: |
| 104 | msg += " (ebuild update only)" |
| 105 | else: |
| 106 | ab_link = ( |
| 107 | "https://android-build.googleplex.com" |
| 108 | "/builds/%s/branches/%s/cls?end=%s" |
| 109 | % (new_android, build_branch, old_android) |
| 110 | ) |
| 111 | cbuildbot_alerts.PrintBuildbotLink("Android changelog", ab_link) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 112 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 113 | cbuildbot_alerts.PrintBuildbotStepText(msg) |
| 114 | cbuildbot_alerts.PrintKitchenSetBuildProperty( |
| 115 | "android_uprev", |
| 116 | json.dumps( |
| 117 | { |
| 118 | "branch": build_branch, |
| 119 | "new": new_ebuild.version, |
| 120 | "old": stable_candidate.version, |
| 121 | "pkgname": stable_candidate.pkgname, |
| 122 | } |
| 123 | ), |
| 124 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 125 | |
khmel@chromium.org | d3ec3d7 | 2020-04-29 15:57:35 -0700 | [diff] [blame] | 126 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 127 | def FindDataCollectorArtifacts( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 128 | gs_context, |
| 129 | android_version, |
| 130 | package_name, |
| 131 | runtime_artifacts_bucket_url, |
| 132 | version_reference, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 133 | ): |
| 134 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 135 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 136 | This is used from UpdateDataCollectorArtifacts in order to check the |
| 137 | particular version. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 138 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 139 | Args: |
| 140 | gs_context: context to execute gsutil |
| 141 | android_version: The \d+ build id of Android. |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 142 | package_name: android package name. Used as folder to locate the cache. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 143 | runtime_artifacts_bucket_url: root of runtime artifacts |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 144 | version_reference: which version to use as a reference. Could be '${PV}' in |
| 145 | case version of data collector artifacts matches the |
| 146 | Android version or direct version in case of override. |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 147 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 148 | Returns: |
| 149 | dictionary with filled ebuild variables. This dictionary is empty in case |
| 150 | no artificats are found. |
| 151 | """ |
| 152 | variables = {} |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 153 | |
Yury Khmel | 697863b | 2022-10-25 09:02:57 -0700 | [diff] [blame] | 154 | buckets = ["ureadahead_pack_host", "gms_core_cache", "tts_cache"] |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 155 | archs = ["arm", "arm64", "x86", "x86_64"] |
| 156 | build_types = ["user", "userdebug"] |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 157 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 158 | for bucket in buckets: |
| 159 | for arch in archs: |
| 160 | for build_type in build_types: |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 161 | # TODO(b/255854925): remove path without |package_name|. |
| 162 | # |package_name| is required to separate artefacts for bertha |
| 163 | # and cheets. |
| 164 | root_paths = [ |
| 165 | f"{runtime_artifacts_bucket_url}/{package_name}/{bucket}_{arch}_{build_type}", |
| 166 | f"{runtime_artifacts_bucket_url}/{bucket}_{arch}_{build_type}", |
| 167 | ] |
| 168 | |
| 169 | for _, root_path in enumerate(root_paths): |
| 170 | path = f"{root_path}_{android_version}.tar" |
| 171 | if gs_context.Exists(path): |
| 172 | variables[ |
| 173 | (f"{arch}_{build_type}_{bucket}").upper() |
| 174 | ] = f"{root_path}_{version_reference}.tar" |
| 175 | break |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 176 | |
| 177 | return variables |
| 178 | |
| 179 | |
| 180 | def UpdateDataCollectorArtifacts( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 181 | android_version, runtime_artifacts_bucket_url, package_name |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 182 | ): |
| 183 | r"""Finds and includes into variables artifacts from arc.DataCollector. |
| 184 | |
| 185 | This verifies default android version. In case artificts are not found for |
| 186 | default Android version it tries to find artifacts for pinned version. If |
| 187 | pinned version is provided, it is required artifacts exist for the pinned |
| 188 | version. |
| 189 | |
| 190 | Args: |
| 191 | android_version: The \d+ build id of Android. |
| 192 | runtime_artifacts_bucket_url: root of runtime artifacts |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 193 | package_name: android package name. Used to determine the pinned version if exists. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 194 | |
| 195 | Returns: |
| 196 | dictionary with filled ebuild variables. |
| 197 | """ |
| 198 | |
| 199 | gs_context = gs.GSContext() |
| 200 | # Check the existing version. If we find any artifacts, use them. |
| 201 | variables = FindDataCollectorArtifacts( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 202 | gs_context, |
| 203 | android_version, |
| 204 | package_name, |
| 205 | runtime_artifacts_bucket_url, |
| 206 | "${PV}", |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 207 | ) |
| 208 | if variables: |
| 209 | # Data artificts were found. |
| 210 | return variables |
| 211 | |
| 212 | # Check pinned version for the current branch. |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 213 | milestone = packages.determine_milestone_version() |
| 214 | pin_path = f"{runtime_artifacts_bucket_url}/{package_name}/M{milestone}_pin_version" |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 215 | if not gs_context.Exists(pin_path): |
| 216 | # No pinned version. |
| 217 | logging.warning( |
| 218 | "No data collector artifacts were found for %s", android_version |
| 219 | ) |
| 220 | return variables |
| 221 | |
| 222 | pin_version = gs_context.Cat(pin_path, encoding="utf-8").rstrip() |
| 223 | logging.info("Pinned version %s overrides %s", pin_version, android_version) |
| 224 | variables = FindDataCollectorArtifacts( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 225 | gs_context, |
| 226 | pin_version, |
| 227 | package_name, |
| 228 | runtime_artifacts_bucket_url, |
| 229 | pin_version, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 230 | ) |
| 231 | if not variables: |
| 232 | # If pin version set it must contain data. |
| 233 | raise Exception( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 234 | "Pinned version %s does not contain artificats" % pin_path |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 235 | ) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 236 | return variables |
| 237 | |
| 238 | |
| 239 | def MarkAndroidEBuildAsStable( |
| 240 | stable_candidate, |
| 241 | unstable_ebuild, |
| 242 | android_package, |
| 243 | android_version, |
| 244 | package_dir, |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 245 | build_branch, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 246 | arc_bucket_url, |
| 247 | runtime_artifacts_bucket_url, |
| 248 | ): |
| 249 | r"""Uprevs the Android ebuild. |
| 250 | |
| 251 | This is the main function that uprevs from a stable candidate |
| 252 | to its new version. |
| 253 | |
| 254 | Args: |
| 255 | stable_candidate: ebuild that corresponds to the stable ebuild we are |
| 256 | revving from. If None, builds the a new ebuild given the version |
| 257 | with revision set to 1. |
| 258 | unstable_ebuild: ebuild corresponding to the unstable ebuild for Android. |
| 259 | android_package: android package name. |
| 260 | android_version: The \d+ build id of Android. |
| 261 | package_dir: Path to the android-container package dir. |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 262 | build_branch: branch of Android builds. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 263 | arc_bucket_url: URL of the target ARC build gs bucket. |
| 264 | runtime_artifacts_bucket_url: root of runtime artifacts |
| 265 | |
| 266 | Returns: |
| 267 | Tuple[str, List[str], List[str]] if revved, or None |
| 268 | 1. Full portage version atom (including rc's, etc) that was revved. |
| 269 | 2. List of files to be `git add`ed. |
| 270 | 3. List of files to be `git rm`ed. |
| 271 | """ |
| 272 | |
| 273 | def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild): |
| 274 | """Returns True if the new ebuild is redundant. |
| 275 | |
| 276 | This is True if there if the current stable ebuild is the exact same copy |
| 277 | of the new one. |
| 278 | """ |
| 279 | if not stable_ebuild: |
| 280 | return False |
| 281 | |
| 282 | if stable_candidate.version_no_rev == new_ebuild.version_no_rev: |
| 283 | return filecmp.cmp( |
| 284 | new_ebuild.ebuild_path, stable_ebuild.ebuild_path, shallow=False |
| 285 | ) |
| 286 | return False |
| 287 | |
| 288 | # Case where we have the last stable candidate with same version just rev. |
| 289 | if stable_candidate and stable_candidate.version_no_rev == android_version: |
| 290 | new_ebuild_path = "%s-r%d.ebuild" % ( |
| 291 | stable_candidate.ebuild_path_no_revision, |
| 292 | stable_candidate.current_revision + 1, |
| 293 | ) |
| 294 | else: |
| 295 | pf = "%s-%s-r1" % (android_package, android_version) |
| 296 | new_ebuild_path = os.path.join(package_dir, "%s.ebuild" % pf) |
| 297 | |
Shao-Chuan Lee | ca2cbcc | 2022-11-02 08:28:31 +0900 | [diff] [blame] | 298 | build_targets = android.GetAndroidEbuildTargetsForPackage(android_package) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 299 | variables = {"BASE_URL": arc_bucket_url} |
| 300 | for var, target in build_targets.items(): |
Shao-Chuan Lee | 20c51d8 | 2022-10-26 21:56:56 +0900 | [diff] [blame] | 301 | # TODO(b/255705023): Have MirrorArtifacts generate the mapping for us. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 302 | variables[var] = f"{build_branch}-linux-{target}" |
| 303 | |
| 304 | variables.update( |
| 305 | UpdateDataCollectorArtifacts( |
Yury Khmel | c0a1844 | 2022-11-01 16:56:22 -0700 | [diff] [blame] | 306 | android_version, |
| 307 | runtime_artifacts_bucket_url, |
| 308 | android_package, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 309 | ) |
| 310 | ) |
| 311 | |
| 312 | portage_util.EBuild.MarkAsStable( |
| 313 | unstable_ebuild.ebuild_path, |
| 314 | new_ebuild_path, |
| 315 | variables, |
| 316 | make_stable=True, |
| 317 | ) |
| 318 | new_ebuild = portage_util.EBuild(new_ebuild_path) |
| 319 | |
| 320 | # Determine whether this is ebuild is redundant. |
| 321 | if IsTheNewEBuildRedundant(new_ebuild, stable_candidate): |
| 322 | msg = "Previous ebuild with same version found and ebuild is redundant." |
| 323 | logging.info(msg) |
| 324 | cbuildbot_alerts.PrintBuildbotStepText( |
| 325 | "%s %s not revved" |
| 326 | % (stable_candidate.pkgname, stable_candidate.version) |
| 327 | ) |
| 328 | osutils.SafeUnlink(new_ebuild_path) |
| 329 | return None |
| 330 | |
| 331 | # PFQ runs should always be able to find a stable candidate. |
| 332 | if stable_candidate: |
| 333 | PrintUprevMetadata(build_branch, stable_candidate, new_ebuild) |
| 334 | |
| 335 | files_to_add = [new_ebuild_path] |
| 336 | files_to_remove = [] |
| 337 | if stable_candidate and not stable_candidate.IsSticky(): |
| 338 | osutils.SafeUnlink(stable_candidate.ebuild_path) |
| 339 | files_to_remove.append(stable_candidate.ebuild_path) |
| 340 | |
| 341 | # Update ebuild manifest and git add it. |
| 342 | gen_manifest_cmd = ["ebuild", new_ebuild_path, "manifest", "--force"] |
| 343 | cros_build_lib.run(gen_manifest_cmd, extra_env=None, print_cmd=True) |
| 344 | files_to_add.append(os.path.join(package_dir, "Manifest")) |
| 345 | |
| 346 | return ( |
| 347 | f"{new_ebuild.package}-{new_ebuild.version}", |
| 348 | files_to_add, |
| 349 | files_to_remove, |
| 350 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 351 | |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 352 | |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame] | 353 | def _PrepareGitBranch(overlay_dir): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 354 | """Prepares a git branch for the uprev commit. |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 355 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 356 | If the overlay project is currently on a branch (e.g. patches are being |
| 357 | applied), rebase the new branch on top of it. |
Shao-Chuan Lee | 007dbe8 | 2021-02-09 14:05:39 +0900 | [diff] [blame] | 358 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 359 | Args: |
| 360 | overlay_dir: The overlay directory. |
| 361 | """ |
| 362 | existing_branch = git.GetCurrentBranch(overlay_dir) |
| 363 | repo_util.Repository.MustFind(overlay_dir).StartBranch( |
| 364 | constants.STABLE_EBUILD_BRANCH, projects=["."], cwd=overlay_dir |
| 365 | ) |
| 366 | if existing_branch: |
| 367 | git.RunGit(overlay_dir, ["rebase", existing_branch]) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 368 | |
| 369 | |
| 370 | def _CommitChange(message, android_package_dir, files_to_add, files_to_remove): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 371 | """Commit changes to git with list of files to add/remove.""" |
| 372 | git.RunGit(android_package_dir, ["add", "--"] + files_to_add) |
| 373 | if files_to_remove: |
| 374 | git.RunGit(android_package_dir, ["rm", "--"] + files_to_remove) |
Shao-Chuan Lee | 301a419 | 2021-02-08 11:53:49 +0900 | [diff] [blame] | 375 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 376 | portage_util.EBuild.CommitChange(message, android_package_dir) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 377 | |
| 378 | |
| 379 | def GetParser(): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 380 | """Creates the argument parser.""" |
| 381 | parser = commandline.ArgumentParser() |
| 382 | parser.add_argument("-b", "--boards") |
| 383 | parser.add_argument( |
| 384 | "--android_bucket_url", |
| 385 | default=android.ANDROID_BUCKET_URL, |
| 386 | type="gs_path", |
| 387 | ) |
| 388 | parser.add_argument( |
| 389 | "--android_build_branch", |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 390 | help="Android branch to import from, overriding default", |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 391 | ) |
| 392 | parser.add_argument( |
| 393 | "--android_package", |
| 394 | required=True, |
Shao-Chuan Lee | ca2cbcc | 2022-11-02 08:28:31 +0900 | [diff] [blame] | 395 | choices=android.GetAllAndroidPackages(), |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 396 | help="Android package to uprev", |
| 397 | ) |
| 398 | parser.add_argument( |
| 399 | "--arc_bucket_url", default=constants.ARC_BUCKET_URL, type="gs_path" |
| 400 | ) |
| 401 | parser.add_argument("-f", "--force_version", help="Android build id to use") |
| 402 | parser.add_argument( |
| 403 | "-s", |
| 404 | "--srcroot", |
| 405 | default=os.path.join(constants.SOURCE_ROOT, "src"), |
| 406 | help="Path to the src directory", |
| 407 | ) |
| 408 | parser.add_argument( |
| 409 | "--runtime_artifacts_bucket_url", |
| 410 | default=_RUNTIME_ARTIFACTS_BUCKET_URL, |
| 411 | type="gs_path", |
| 412 | ) |
| 413 | parser.add_argument( |
| 414 | "--skip_commit", |
| 415 | action="store_true", |
| 416 | help="Skip commiting uprev changes to git", |
| 417 | ) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 418 | return parser |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 419 | |
| 420 | |
| 421 | def main(argv): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 422 | cbuildbot_alerts.EnableBuildbotMarkers() |
| 423 | parser = GetParser() |
| 424 | options = parser.parse_args(argv) |
| 425 | options.Freeze() |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 426 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 427 | overlay_dir = os.path.abspath(_OVERLAY_DIR % {"srcroot": options.srcroot}) |
| 428 | android_package_dir = android.GetAndroidPackageDir( |
| 429 | options.android_package, overlay_dir=overlay_dir |
| 430 | ) |
Shao-Chuan Lee | 3aa7652 | 2021-12-03 17:18:58 +0900 | [diff] [blame] | 431 | |
| 432 | if not options.skip_commit: |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 433 | _PrepareGitBranch(overlay_dir) |
Shao-Chuan Lee | 3aa7652 | 2021-12-03 17:18:58 +0900 | [diff] [blame] | 434 | |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 435 | # Use default Android branch if not overridden. |
| 436 | android_build_branch = ( |
Shao-Chuan Lee | 20c51d8 | 2022-10-26 21:56:56 +0900 | [diff] [blame] | 437 | options.android_build_branch |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 438 | or android.GetAndroidBranchForPackage(options.android_package) |
| 439 | ) |
Shao-Chuan Lee | dea458f | 2021-11-25 23:46:53 +0900 | [diff] [blame] | 440 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 441 | (unstable_ebuild, stable_ebuilds) = FindAndroidCandidates( |
| 442 | android_package_dir |
| 443 | ) |
| 444 | # Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt). |
| 445 | version_to_uprev = android.MirrorArtifacts( |
Shao-Chuan Lee | 20c51d8 | 2022-10-26 21:56:56 +0900 | [diff] [blame] | 446 | options.android_package, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 447 | options.android_bucket_url, |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 448 | android_build_branch, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 449 | options.arc_bucket_url, |
| 450 | android_package_dir, |
| 451 | options.force_version, |
| 452 | ) |
David Riley | c0da9d9 | 2016-02-01 12:11:01 -0800 | [diff] [blame] | 453 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 454 | stable_candidate = portage_util.BestEBuild(stable_ebuilds) |
Shao-Chuan Lee | dea458f | 2021-11-25 23:46:53 +0900 | [diff] [blame] | 455 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 456 | if stable_candidate: |
| 457 | logging.info("Stable candidate found %s", stable_candidate.version) |
| 458 | else: |
| 459 | logging.info("No stable candidate found.") |
| 460 | |
| 461 | revved = MarkAndroidEBuildAsStable( |
| 462 | stable_candidate, |
| 463 | unstable_ebuild, |
| 464 | options.android_package, |
| 465 | version_to_uprev, |
| 466 | android_package_dir, |
Shao-Chuan Lee | 7e507e6 | 2022-11-15 02:17:34 +0000 | [diff] [blame] | 467 | android_build_branch, |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 468 | options.arc_bucket_url, |
| 469 | options.runtime_artifacts_bucket_url, |
| 470 | ) |
| 471 | |
| 472 | output = dict(revved=bool(revved)) |
| 473 | |
| 474 | if revved: |
| 475 | android_atom, files_to_add, files_to_remove = revved |
| 476 | if not options.skip_commit: |
| 477 | _CommitChange( |
| 478 | _GIT_COMMIT_MESSAGE |
| 479 | % { |
| 480 | "android_package": options.android_package, |
| 481 | "android_version": version_to_uprev, |
| 482 | }, |
| 483 | android_package_dir, |
| 484 | files_to_add, |
| 485 | files_to_remove, |
| 486 | ) |
| 487 | if options.boards: |
| 488 | cros_mark_as_stable.CleanStalePackages( |
| 489 | options.srcroot, options.boards.split(":"), [android_atom] |
| 490 | ) |
| 491 | |
| 492 | output["android_atom"] = android_atom |
| 493 | # This field is read by the PUpr uprev handler for creating CLs. We cannot |
| 494 | # return absolute paths because this script runs inside chroot but the uprev |
| 495 | # handler runs outside. |
| 496 | # Here we return paths relative to |overlay_dir|. |
| 497 | output["modified_files"] = [ |
| 498 | os.path.relpath(f, overlay_dir) |
| 499 | for f in files_to_add + files_to_remove |
| 500 | ] |
| 501 | |
| 502 | # The output is being parsed by service.packages.uprev_android and has to be |
| 503 | # in its own single line. When invoked from chromite API endpoints, entering |
| 504 | # chroot can generate junk messages on stdout, so we prefix our output with a |
| 505 | # line break to further ensure that. |
| 506 | print("\n" + json.dumps(output, sort_keys=True)) |