Mike Frysinger | f1ba7ad | 2022-09-12 05:42:57 -0400 | [diff] [blame] | 1 | # Copyright 2013 The ChromiumOS Authors |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Mike Frysinger | ad8c6ca | 2014-02-03 11:28:45 -0500 | [diff] [blame] | 5 | """Fast alternative to `emerge-$BOARD autotest-all` |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 6 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 7 | Simple script to be run inside the chroot. Used as a fast approximation of |
| 8 | emerge-$board autotest-all, by simply rsync'ing changes from trunk to sysroot. |
| 9 | """ |
| 10 | |
Aviv Keshet | e7b2019 | 2013-04-24 14:05:53 -0700 | [diff] [blame] | 11 | import argparse |
Chris McDonald | 59650c3 | 2021-07-20 15:29:28 -0600 | [diff] [blame] | 12 | from collections import namedtuple |
Aviv Keshet | fe54a8a | 2013-09-16 15:49:03 -0700 | [diff] [blame] | 13 | import glob |
Chris McDonald | 59650c3 | 2021-07-20 15:29:28 -0600 | [diff] [blame] | 14 | import logging |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 15 | import os |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 16 | import re |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 17 | import sys |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 18 | |
Alex Klein | 727e320 | 2020-05-29 11:13:55 -0600 | [diff] [blame] | 19 | from chromite.lib import commandline |
Aviv Keshet | b7519e1 | 2016-10-04 00:50:00 -0700 | [diff] [blame] | 20 | from chromite.lib import constants |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 21 | from chromite.lib import cros_build_lib |
| 22 | from chromite.lib import git |
Aviv Keshet | 557e688 | 2013-04-25 13:26:09 -0700 | [diff] [blame] | 23 | from chromite.lib import osutils |
Alex Klein | 18a60af | 2020-06-11 12:08:47 -0600 | [diff] [blame] | 24 | from chromite.lib.parser import package_info |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 25 | |
Chris McDonald | 59650c3 | 2021-07-20 15:29:28 -0600 | [diff] [blame] | 26 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 27 | if cros_build_lib.IsInsideChroot(): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 28 | # pylint: disable=import-error |
| 29 | import portage |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 30 | |
Mike Frysinger | e65f375 | 2014-12-08 00:46:39 -0500 | [diff] [blame] | 31 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 32 | INCLUDE_PATTERNS_FILENAME = "autotest-quickmerge-includepatterns" |
| 33 | AUTOTEST_SYMLINK = "autotest_lib" |
| 34 | AUTOTEST_PROJECT_NAME = "chromiumos/third_party/autotest" |
| 35 | AUTOTEST_EBUILD = "chromeos-base/autotest" |
| 36 | DOWNGRADE_EBUILDS = ["chromeos-base/autotest"] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 37 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 38 | IGNORE_SUBDIRS = ["ExternalSource", "logs", "results", "site-packages"] |
Aviv Keshet | c73cfc3 | 2013-06-14 16:18:53 -0700 | [diff] [blame] | 39 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 40 | # Data structure describing a single rsync filesystem change. |
| 41 | # |
| 42 | # change_description: An 11 character string, the rsync change description |
| 43 | # for the particular file. |
| 44 | # absolute_path: The absolute path of the created or modified file. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 45 | ItemizedChange = namedtuple( |
| 46 | "ItemizedChange", ["change_description", "absolute_path"] |
| 47 | ) |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 48 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 49 | # Data structure describing the rsync new/modified files or directories. |
| 50 | # |
| 51 | # new_files: A list of ItemizedChange objects for new files. |
| 52 | # modified_files: A list of ItemizedChange objects for modified files. |
| 53 | # new_directories: A list of ItemizedChange objects for new directories. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 54 | ItemizedChangeReport = namedtuple( |
| 55 | "ItemizedChangeReport", ["new_files", "modified_files", "new_directories"] |
| 56 | ) |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 57 | |
Mike Frysinger | e65f375 | 2014-12-08 00:46:39 -0500 | [diff] [blame] | 58 | |
Aviv Keshet | 84bdfc5 | 2013-06-04 13:19:38 -0700 | [diff] [blame] | 59 | class PortagePackageAPIError(Exception): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 60 | """Exception thrown when unable to retrieve a portage package API.""" |
Aviv Keshet | 84bdfc5 | 2013-06-04 13:19:38 -0700 | [diff] [blame] | 61 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 62 | |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 63 | def GetStalePackageNames(change_list, autotest_sysroot): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 64 | """Given a rsync change report, returns the names of stale test packages. |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 65 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 66 | This function pulls out test package names for client-side tests, stored |
| 67 | within the client/site_tests directory tree, that had any files added or |
| 68 | modified and for whom any existing bzipped test packages may now be stale. |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 69 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 70 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 71 | change_list: A list of ItemizedChange objects corresponding to changed |
| 72 | or modified files. |
| 73 | autotest_sysroot: Absolute path of autotest in the sysroot, |
| 74 | e.g. '/build/lumpy/usr/local/build/autotest' |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 75 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 76 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 77 | A list of test package names, e.g. ['factory_Leds', |
| 78 | 'login_UserPolicyKeys']. May contain duplicate entries if multiple files |
| 79 | within a test directory were modified. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 80 | """ |
| 81 | exp = os.path.abspath(autotest_sysroot) + r"/client/site_tests/(.*?)/.*" |
| 82 | matches = [re.match(exp, change.absolute_path) for change in change_list] |
| 83 | return [match.group(1) for match in matches if match] |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 84 | |
| 85 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 86 | def ItemizeChangesFromRsyncOutput(rsync_output, destination_path): |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 87 | """Convert output of an rsync with `-i` to a ItemizedChangeReport object. |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 88 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 89 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 90 | rsync_output: String stdout of rsync command that was run with `-i` |
| 91 | option. |
| 92 | destination_path: String absolute path of the destination directory for |
| 93 | the rsync operations. This argument is necessary because rsync's |
| 94 | output only gives the relative path of touched/added files. |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 95 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 96 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 97 | ItemizedChangeReport object giving the absolute paths of files that were |
| 98 | created or modified by rsync. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 99 | """ |
| 100 | modified_matches = re.findall(r"([.>]f[^+]{9}) (.*)", rsync_output) |
| 101 | new_matches = re.findall(r"(>f\+{9}) (.*)", rsync_output) |
| 102 | new_symlink_matches = re.findall(r"(cL\+{9}) (.*) -> .*", rsync_output) |
| 103 | new_dir_matches = re.findall(r"(cd\+{9}) (.*)", rsync_output) |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 104 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 105 | absolute_modified = [ |
| 106 | ItemizedChange(c, os.path.join(destination_path, f)) |
| 107 | for (c, f) in modified_matches |
| 108 | ] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 109 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 110 | # Note: new symlinks are treated as new files. |
| 111 | absolute_new = [ |
| 112 | ItemizedChange(c, os.path.join(destination_path, f)) |
| 113 | for (c, f) in new_matches + new_symlink_matches |
| 114 | ] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 115 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 116 | absolute_new_dir = [ |
| 117 | ItemizedChange(c, os.path.join(destination_path, f)) |
| 118 | for (c, f) in new_dir_matches |
| 119 | ] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 120 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 121 | return ItemizedChangeReport( |
| 122 | new_files=absolute_new, |
| 123 | modified_files=absolute_modified, |
| 124 | new_directories=absolute_new_dir, |
| 125 | ) |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 126 | |
| 127 | |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 128 | def GetPackageAPI(portage_root, package_cp): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 129 | """Gets portage API handles for the given package. |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 130 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 131 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 132 | portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy' |
| 133 | package_cp: A string similar to 'chromeos-base/autotest-tests'. |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 134 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 135 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 136 | Returns (package, vartree) tuple, where |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 137 | package is of type portage.dbapi.vartree.dblink |
| 138 | vartree is of type portage.dbapi.vartree.vartree |
| 139 | """ |
| 140 | if portage_root is None: |
| 141 | # pylint: disable=no-member |
| 142 | portage_root = portage.root |
| 143 | # Ensure that portage_root ends with trailing slash. |
| 144 | portage_root = os.path.join(portage_root, "") |
| 145 | |
| 146 | # Create a vartree object corresponding to portage_root. |
| 147 | trees = portage.create_trees(portage_root, portage_root) |
| 148 | vartree = trees[portage_root]["vartree"] |
| 149 | |
| 150 | # List the matching installed packages in cpv format. |
| 151 | matching_packages = vartree.dbapi.cp_list(package_cp) |
| 152 | |
| 153 | if not matching_packages: |
| 154 | raise PortagePackageAPIError( |
| 155 | "No matching package for %s in portage_root " |
| 156 | "%s" % (package_cp, portage_root) |
| 157 | ) |
| 158 | |
| 159 | if len(matching_packages) > 1: |
| 160 | raise PortagePackageAPIError( |
| 161 | "Too many matching packages for %s in " |
| 162 | "portage_root %s" % (package_cp, portage_root) |
| 163 | ) |
| 164 | |
| 165 | # Convert string match to package dblink. |
| 166 | package_cpv = matching_packages[0] |
| 167 | package_split = package_info.SplitCPV(package_cpv) |
Mike Frysinger | e65f375 | 2014-12-08 00:46:39 -0500 | [diff] [blame] | 168 | # pylint: disable=no-member |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 169 | package = portage.dblink( |
| 170 | package_split.category, |
| 171 | package_split.pv, |
| 172 | settings=vartree.settings, |
| 173 | vartree=vartree, |
| 174 | ) |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 175 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 176 | return package, vartree |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 177 | |
| 178 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 179 | def DowngradePackageVersion(portage_root, package_cp, downgrade_to_version="0"): |
| 180 | """Downgrade the specified portage package version. |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 181 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 182 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 183 | portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy' |
| 184 | package_cp: A string similar to 'chromeos-base/autotest-tests'. |
| 185 | downgrade_to_version: String version to downgrade to. Default: '0' |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 186 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 187 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 188 | bool: True on success. False on failure (nonzero return code from `mv` |
| 189 | command). |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 190 | """ |
| 191 | try: |
| 192 | package, _ = GetPackageAPI(portage_root, package_cp) |
| 193 | except PortagePackageAPIError: |
| 194 | # Unable to fetch a corresponding portage package API for this |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 195 | # package_cp (either no such package, or name ambiguous and matches). |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 196 | # So, just fail out. |
| 197 | return False |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 198 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 199 | source_directory = package.dbdir |
| 200 | destination_path = os.path.join( |
| 201 | package.dbroot, package_cp + "-" + downgrade_to_version |
| 202 | ) |
| 203 | if os.path.abspath(source_directory) == os.path.abspath(destination_path): |
| 204 | return True |
| 205 | command = ["mv", source_directory, destination_path] |
| 206 | code = cros_build_lib.sudo_run(command, check=False).returncode |
| 207 | return code == 0 |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 208 | |
| 209 | |
Aviv Keshet | e7b2019 | 2013-04-24 14:05:53 -0700 | [diff] [blame] | 210 | def UpdatePackageContents(change_report, package_cp, portage_root=None): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 211 | """Add newly created files/directors to package contents. |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 212 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 213 | Given an ItemizedChangeReport, add the newly created files and directories |
| 214 | to the CONTENTS of an installed portage package, such that these files are |
| 215 | considered owned by that package. |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 216 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 217 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 218 | change_report: ItemizedChangeReport object for the changes to be |
| 219 | made to the package. |
| 220 | package_cp: A string similar to 'chromeos-base/autotest-tests' giving |
| 221 | the package category and name of the package to be altered. |
| 222 | portage_root: Portage root path, corresponding to the board that |
| 223 | we are working on. Defaults to '/'. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 224 | """ |
| 225 | package, vartree = GetPackageAPI(portage_root, package_cp) |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 226 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 227 | # Append new contents to package contents dictionary. |
| 228 | contents = package.getcontents().copy() |
| 229 | for _, filename in change_report.new_files: |
| 230 | contents.setdefault(filename, ("obj", "0", "0")) |
| 231 | for _, dirname in change_report.new_directories: |
| 232 | # Strip trailing slashes if present. |
| 233 | contents.setdefault(dirname.rstrip("/"), ("dir",)) |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 234 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 235 | # Write new contents dictionary to file. |
| 236 | vartree.dbapi.writeContentsToContentsFile(package, contents) |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 237 | |
| 238 | |
Aviv Keshet | 1927675 | 2013-05-16 11:12:23 -0700 | [diff] [blame] | 239 | def RemoveBzipPackages(autotest_sysroot): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 240 | """Remove all bzipped test/dep/profiler packages from sysroot autotest. |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 241 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 242 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 243 | autotest_sysroot: Absolute path of autotest in the sysroot, |
| 244 | e.g. '/build/lumpy/usr/local/build/autotest' |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 245 | """ |
| 246 | osutils.RmDir( |
| 247 | os.path.join(autotest_sysroot, "packages"), ignore_missing=True |
| 248 | ) |
| 249 | osutils.SafeUnlink(os.path.join(autotest_sysroot, "packages.checksum")) |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 250 | |
| 251 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 252 | def RsyncQuickmerge( |
| 253 | source_path, |
| 254 | sysroot_autotest_path, |
| 255 | include_pattern_file=None, |
| 256 | pretend=False, |
| 257 | overwrite=False, |
| 258 | ): |
| 259 | """Run rsync quickmerge command, with specified arguments. |
Aviv Keshet | e7b2019 | 2013-04-24 14:05:53 -0700 | [diff] [blame] | 260 | |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 261 | Command will take form: |
| 262 | `rsync -a [options] --exclude=**.pyc --exclude=**.pyo |
| 263 | [optional --include-from include_pattern_file] |
| 264 | --exclude=* [source_path] [sysroot_autotest_path]` |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 265 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 266 | Args: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 267 | source_path: Directory to rsync from. |
| 268 | sysroot_autotest_path: Directory to rsync too. |
| 269 | include_pattern_file: Optional pattern of files to include in rsync. |
| 270 | pretend: True to use the '-n' option to rsync, to perform dry run. |
| 271 | overwrite: True to omit '-u' option, overwrite all files in sysroot, |
| 272 | not just older files. |
Aviv Keshet | 557e688 | 2013-04-25 13:26:09 -0700 | [diff] [blame] | 273 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 274 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 275 | The cros_build_lib.CompletedProcess object resulting from the rsync |
| 276 | command. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 277 | """ |
| 278 | command = ["rsync", "-a"] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 279 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 280 | # For existing files, preserve destination permissions. This ensures that |
| 281 | # existing files end up with the file permissions set by the ebuilds. |
| 282 | # If this script copies over a file that does not exist in the destination |
| 283 | # tree, it will set the least restrictive permissions allowed in the |
| 284 | # destination tree. This could happen if the file copied is not installed by |
| 285 | # *any* ebuild, or if the ebuild that installs the file was never emerged. |
| 286 | command += ["--no-p", "--chmod=ugo=rwX"] |
Prathmesh Prabhu | 137266a | 2014-02-11 20:02:18 -0800 | [diff] [blame] | 287 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 288 | if pretend: |
| 289 | command += ["-n"] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 290 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 291 | if not overwrite: |
| 292 | command += ["-u"] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 293 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 294 | command += ["-i"] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 295 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 296 | command += ["--exclude=**.pyc"] |
| 297 | command += ["--exclude=**.pyo"] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 298 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 299 | # Always exclude the autotest symlink to avoid a possible recursion hole. |
| 300 | # The order here is (unfortunately) extremely important. |
| 301 | if AUTOTEST_SYMLINK not in source_path: |
| 302 | command += ["--exclude=%s/" % AUTOTEST_SYMLINK] |
Derek Beckett | 1f19c9c | 2020-12-14 12:25:53 -0800 | [diff] [blame] | 303 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 304 | # Exclude files with a specific substring in their name, because |
| 305 | # they create an ambiguous itemized report. (see unit test file for details) |
| 306 | command += ["--exclude=** -> *"] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 307 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 308 | # Order seems important here, and this include must come before the possible |
| 309 | # exclude below... |
| 310 | if include_pattern_file: |
| 311 | command += ["--include-from=%s" % include_pattern_file] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 312 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 313 | if AUTOTEST_SYMLINK in source_path: |
| 314 | command += ["-l"] |
| 315 | else: |
| 316 | command += ["-L", "--exclude=*"] |
Po-Hsien Wang | 745ac35 | 2018-04-13 12:25:41 -0700 | [diff] [blame] | 317 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 318 | command += [source_path, sysroot_autotest_path] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 319 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 320 | return cros_build_lib.sudo_run(command, stdout=True, encoding="utf-8") |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 321 | |
| 322 | |
| 323 | def ParseArguments(argv): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 324 | """Parse command line arguments |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 325 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 326 | Returns: |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 327 | parsed arguments. |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 328 | """ |
| 329 | parser = commandline.ArgumentParser( |
| 330 | description="Perform a fast approximation to emerge-$board " |
| 331 | "autotest-all, by rsyncing source tree to sysroot." |
| 332 | ) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 333 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 334 | default_board = cros_build_lib.GetDefaultBoard() |
| 335 | parser.add_argument( |
| 336 | "--board", |
| 337 | metavar="BOARD", |
| 338 | default=default_board, |
| 339 | help="Board to perform quickmerge for. Default: " |
| 340 | + (default_board or "Not configured."), |
| 341 | ) |
| 342 | parser.add_argument( |
| 343 | "--pretend", |
| 344 | action="store_true", |
| 345 | help="Dry run only, do not modify sysroot autotest.", |
| 346 | ) |
| 347 | parser.add_argument( |
| 348 | "--overwrite", |
| 349 | action="store_true", |
| 350 | help="Overwrite existing files even if newer.", |
| 351 | ) |
| 352 | parser.add_argument("--force", action="store_true", help=argparse.SUPPRESS) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 353 | |
Alex Klein | 9e7b29e | 2023-04-11 16:10:31 -0600 | [diff] [blame] | 354 | # Used only if test_that is calling autotest_quickmerge and has detected |
| 355 | # that the sysroot autotest path is still in usr/local/autotest (ie the |
| 356 | # build pre-dates https://chromium-review.googlesource.com/#/c/62880/ ) |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 357 | parser.add_argument( |
| 358 | "--legacy_path", action="store_true", help=argparse.SUPPRESS |
| 359 | ) |
Aviv Keshet | 474469d | 2013-07-22 12:54:52 -0700 | [diff] [blame] | 360 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 361 | return parser.parse_args(argv) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 362 | |
| 363 | |
Derek Beckett | 1f19c9c | 2020-12-14 12:25:53 -0800 | [diff] [blame] | 364 | def _maybe_add_autotest_symlink(src_paths, path, dest_path): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 365 | """If the symlink folders exists, add them to the src to quickmerge.""" |
| 366 | autotest_client_symlink = os.path.join(path, "client", AUTOTEST_SYMLINK) |
| 367 | if os.path.exists(autotest_client_symlink): |
| 368 | src_paths.append( |
| 369 | (autotest_client_symlink, os.path.join(dest_path, "client/")) |
| 370 | ) |
| 371 | autotest_main_symlink = os.path.join(path, AUTOTEST_SYMLINK) |
| 372 | if os.path.exists(autotest_main_symlink): |
| 373 | src_paths.append((autotest_main_symlink, dest_path)) |
Derek Beckett | 1f19c9c | 2020-12-14 12:25:53 -0800 | [diff] [blame] | 374 | |
| 375 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 376 | def main(argv): |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 377 | cros_build_lib.AssertInsideChroot() |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 378 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 379 | args = ParseArguments(argv) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 380 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 381 | if osutils.IsNonRootUser(): |
| 382 | try: |
| 383 | cros_build_lib.sudo_run([sys.executable] + sys.argv) |
| 384 | except cros_build_lib.RunCommandError: |
| 385 | return 1 |
| 386 | return 0 |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 387 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 388 | if not args.board: |
| 389 | print("No board specified. Aborting.") |
| 390 | return 1 |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 391 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 392 | manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT) |
| 393 | checkout = manifest.FindCheckout(AUTOTEST_PROJECT_NAME) |
| 394 | brillo_autotest_src_path = os.path.join(checkout.GetPath(absolute=True), "") |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 395 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 396 | script_path = os.path.dirname(__file__) |
| 397 | include_pattern_file = os.path.join(script_path, INCLUDE_PATTERNS_FILENAME) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 398 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 399 | # TODO: Determine the following string programatically. |
| 400 | sysroot_path = os.path.join("/build", args.board, "") |
| 401 | sysroot_autotest_path = os.path.join( |
| 402 | sysroot_path, constants.AUTOTEST_BUILD_PATH, "" |
| 403 | ) |
| 404 | if args.legacy_path: |
| 405 | sysroot_autotest_path = os.path.join( |
| 406 | sysroot_path, "usr/local/autotest", "" |
| 407 | ) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 408 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 409 | # Generate the list of source paths to copy. |
| 410 | src_paths = {os.path.abspath(brillo_autotest_src_path)} |
| 411 | for quickmerge_file in glob.glob( |
| 412 | os.path.join(sysroot_autotest_path, "quickmerge", "*", "*") |
| 413 | ): |
| 414 | try: |
| 415 | path = osutils.ReadFile(quickmerge_file).strip() |
| 416 | if path and os.path.exists(path): |
| 417 | src_paths.add(os.path.abspath(path)) |
| 418 | except IOError: |
| 419 | logging.error( |
| 420 | "Could not quickmerge for project: %s", |
| 421 | os.path.basename(quickmerge_file), |
| 422 | ) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 423 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 424 | # Autotest uses a circular symlink in client that *must* be added after the |
| 425 | # other sections of Autotest. |
| 426 | src_paths = list(src_paths) |
Derek Beckett | 1f19c9c | 2020-12-14 12:25:53 -0800 | [diff] [blame] | 427 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 428 | # All destination paths up to this point are the same, but other sources |
| 429 | # added below might have a different one. |
| 430 | src_dest_paths = [ |
| 431 | (src_path + "/", sysroot_autotest_path) for src_path in src_paths |
| 432 | ] |
Derek Beckett | 1f19c9c | 2020-12-14 12:25:53 -0800 | [diff] [blame] | 433 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 434 | _maybe_add_autotest_symlink( |
| 435 | src_dest_paths, brillo_autotest_src_path, sysroot_autotest_path |
| 436 | ) |
| 437 | num_new_files = 0 |
| 438 | num_modified_files = 0 |
| 439 | for src_path, dest_path in src_dest_paths: |
| 440 | rsync_output = RsyncQuickmerge( |
| 441 | src_path, |
| 442 | dest_path, |
| 443 | include_pattern_file, |
| 444 | args.pretend, |
| 445 | args.overwrite, |
| 446 | ) |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 447 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 448 | if args.verbose: |
| 449 | logging.info(rsync_output.stdout) |
| 450 | change_report = ItemizeChangesFromRsyncOutput( |
| 451 | rsync_output.stdout, sysroot_autotest_path |
| 452 | ) |
| 453 | num_new_files = num_new_files + len(change_report.new_files) |
| 454 | num_modified_files = num_modified_files + len( |
| 455 | change_report.modified_files |
| 456 | ) |
| 457 | if not args.pretend: |
| 458 | logging.info("Updating portage database.") |
| 459 | UpdatePackageContents(change_report, AUTOTEST_EBUILD, sysroot_path) |
| 460 | |
Simran Basi | 348fccc | 2015-04-28 12:39:01 -0700 | [diff] [blame] | 461 | if not args.pretend: |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 462 | for logfile in glob.glob( |
| 463 | os.path.join(sysroot_autotest_path, "packages", "*.log") |
| 464 | ): |
| 465 | try: |
| 466 | # Open file in a try-except block, for atomicity, instead of |
| 467 | # doing existence check. |
Mike Frysinger | 31fdddd | 2023-02-24 15:50:55 -0500 | [diff] [blame] | 468 | with open(logfile, "r", encoding="utf-8") as f: |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 469 | package_cp = f.readline().strip() |
| 470 | DOWNGRADE_EBUILDS.append(package_cp) |
| 471 | except IOError: |
| 472 | pass |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 473 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 474 | for ebuild in DOWNGRADE_EBUILDS: |
| 475 | if not DowngradePackageVersion(sysroot_path, ebuild): |
| 476 | logging.warning( |
| 477 | "Unable to downgrade package %s version number.", ebuild |
| 478 | ) |
| 479 | RemoveBzipPackages(sysroot_autotest_path) |
Aviv Keshet | fe54a8a | 2013-09-16 15:49:03 -0700 | [diff] [blame] | 480 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 481 | sentinel_filename = os.path.join( |
| 482 | sysroot_autotest_path, ".quickmerge_sentinel" |
| 483 | ) |
| 484 | cros_build_lib.run(["touch", sentinel_filename]) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 485 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 486 | if args.pretend: |
| 487 | logging.info( |
| 488 | "The following message is pretend only. No filesystem " |
| 489 | "changes made." |
| 490 | ) |
| 491 | logging.info( |
| 492 | "Quickmerge complete. Created or modified %s files.", |
| 493 | num_new_files + num_modified_files, |
| 494 | ) |
Aviv Keshet | c73cfc3 | 2013-06-14 16:18:53 -0700 | [diff] [blame] | 495 | |
Alex Klein | 1699fab | 2022-09-08 08:46:06 -0600 | [diff] [blame] | 496 | return 0 |