Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | |
| 8 | """ |
| 9 | Simple script to be run inside the chroot. Used as a fast approximation of |
| 10 | emerge-$board autotest-all, by simply rsync'ing changes from trunk to sysroot. |
| 11 | """ |
| 12 | |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 13 | import logging |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 14 | import os |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 15 | import re |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 16 | import sys |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 17 | from collections import namedtuple |
| 18 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 19 | from chromite.buildbot import constants |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 20 | from chromite.buildbot import portage_utilities |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 21 | from chromite.lib import cros_build_lib |
| 22 | from chromite.lib import git |
| 23 | |
| 24 | import argparse |
| 25 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 26 | if cros_build_lib.IsInsideChroot(): |
| 27 | # Only import portage after we've checked that we're inside the chroot. |
| 28 | import portage |
| 29 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 30 | INCLUDE_PATTERNS_FILENAME = 'autotest-quickmerge-includepatterns' |
| 31 | AUTOTEST_PROJECT_NAME = 'chromiumos/third_party/autotest' |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 32 | AUTOTEST_TESTS_EBUILD = 'chromeos-base/autotest-tests' |
Aviv Keshet | 3cc4e9e | 2013-04-24 10:47:23 -0700 | [diff] [blame^] | 33 | DOWNGRADE_EBUILDS = ['chromeos-base/autotest', |
| 34 | 'chromeos-base/autotest-tests', |
| 35 | 'chromeos-base/autotest-chrome', |
| 36 | 'chromeos-base/autotest-factory', |
| 37 | 'chromeos-base/autotest-telemetry', |
| 38 | 'chromeos-base/autotest-tests-ltp', |
| 39 | 'chromeos-base/autotest-tests-ownershipapi'] |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 40 | |
| 41 | # Data structure describing a single rsync filesystem change. |
| 42 | # |
| 43 | # change_description: An 11 character string, the rsync change description |
| 44 | # for the particular file. |
| 45 | # absolute_path: The absolute path of the created or modified file. |
| 46 | ItemizedChange = namedtuple('ItemizedChange', ['change_description', |
| 47 | 'absolute_path']) |
| 48 | |
| 49 | |
| 50 | # Data structure describing the rsync new/modified files or directories. |
| 51 | # |
| 52 | # new_files: A list of ItemizedChange objects for new files. |
| 53 | # modified_files: A list of ItemizedChange objects for modified files. |
| 54 | # new_directories: A list of ItemizedChange objects for new directories. |
| 55 | ItemizedChangeReport = namedtuple('ItemizedChangeReport', |
| 56 | ['new_files', 'modified_files', |
| 57 | 'new_directories']) |
| 58 | |
| 59 | |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 60 | def GetStalePackageNames(change_list, autotest_sysroot): |
| 61 | """ |
| 62 | Given a rsync change report, returns the names of stale test packages. |
| 63 | |
| 64 | This function pulls out test package names for client-side tests, stored |
| 65 | within the client/site_tests directory tree, that had any files added or |
| 66 | modified and for whom any existing bzipped test packages may now be stale. |
| 67 | |
| 68 | Arguments: |
| 69 | change_list: A list of ItemizedChange objects corresponding to changed |
| 70 | or modified files. |
| 71 | autotest_sysroot: Absolute path of autotest in the sysroot, |
| 72 | e.g. '/build/lumpy/usr/local/autotest' |
| 73 | |
| 74 | Returns: |
| 75 | A list of test package names, eg ['factory_Leds', 'login_UserPolicyKeys']. |
| 76 | May contain duplicate entries if multiple files within a test directory |
| 77 | were modified. |
| 78 | """ |
| 79 | exp = os.path.abspath(autotest_sysroot) + r'/client/site_tests/(.*?)/.*' |
| 80 | matches = [re.match(exp, change.absolute_path) for change in change_list] |
| 81 | return [match.group(1) for match in matches if match] |
| 82 | |
| 83 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 84 | def ItemizeChangesFromRsyncOutput(rsync_output, destination_path): |
| 85 | """Convert the output of an rsync with `-i` to a ItemizedChangeReport object. |
| 86 | |
| 87 | Arguments: |
| 88 | rsync_output: String stdout of rsync command that was run with `-i` option. |
| 89 | destination_path: String absolute path of the destination directory for the |
| 90 | rsync operations. This argument is necessary because |
| 91 | rsync's output only gives the relative path of |
| 92 | touched/added files. |
| 93 | |
| 94 | Returns: |
| 95 | ItemizedChangeReport object giving the absolute paths of files that were |
| 96 | created or modified by rsync. |
| 97 | """ |
| 98 | modified_matches = re.findall(r'([.>]f[^+]{9}) (.*)', rsync_output) |
| 99 | new_matches = re.findall(r'(>f\+{9}) (.*)', rsync_output) |
| 100 | new_symlink_matches = re.findall(r'(cL\+{9}) (.*) -> .*', rsync_output) |
| 101 | new_dir_matches = re.findall(r'(cd\+{9}) (.*)', rsync_output) |
| 102 | |
| 103 | absolute_modified = [ItemizedChange(c, os.path.join(destination_path, f)) |
| 104 | for (c, f) in modified_matches] |
| 105 | |
| 106 | # Note: new symlinks are treated as new files. |
| 107 | absolute_new = [ItemizedChange(c, os.path.join(destination_path, f)) |
| 108 | for (c, f) in new_matches + new_symlink_matches] |
| 109 | |
| 110 | absolute_new_dir = [ItemizedChange(c, os.path.join(destination_path, f)) |
| 111 | for (c, f) in new_dir_matches] |
| 112 | |
| 113 | return ItemizedChangeReport(new_files=absolute_new, |
| 114 | modified_files=absolute_modified, |
| 115 | new_directories=absolute_new_dir) |
| 116 | |
| 117 | |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 118 | def GetPackageAPI(portage_root, package_cp): |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 119 | """ |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 120 | Gets portage API handles for the given package. |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 121 | |
| 122 | Arguments: |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 123 | portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy' |
| 124 | package_cp: A string similar to 'chromeos-base/autotest-tests'. |
| 125 | |
| 126 | Returns: |
| 127 | Returns (package, vartree) tuple, where |
| 128 | package is of type portage.dbapi.vartree.dblink |
| 129 | vartree is of type portage.dbapi.vartree.vartree |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 130 | """ |
| 131 | if portage_root is None: |
| 132 | portage_root = portage.root # pylint: disable-msg=E1101 |
| 133 | # Ensure that portage_root ends with trailing slash. |
| 134 | portage_root = os.path.join(portage_root, '') |
| 135 | |
| 136 | # Create vartree object corresponding to portage_root |
| 137 | trees = portage.create_trees(portage_root, portage_root) |
| 138 | vartree = trees[portage_root]['vartree'] |
| 139 | |
| 140 | # List matching installed packages in cpv format |
| 141 | matching_packages = vartree.dbapi.cp_list(package_cp) |
| 142 | |
| 143 | if not matching_packages: |
| 144 | raise ValueError('No matching package for %s in portage_root %s' % ( |
| 145 | package_cp, portage_root)) |
| 146 | |
| 147 | if len(matching_packages) > 1: |
| 148 | raise ValueError('Too many matching packages for %s in portage_root ' |
| 149 | '%s' % (package_cp, portage_root)) |
| 150 | |
| 151 | # Convert string match to package dblink |
| 152 | package_cpv = matching_packages[0] |
| 153 | package_split = portage_utilities.SplitCPV(package_cpv) |
| 154 | package = portage.dblink(package_split.category, # pylint: disable-msg=E1101 |
| 155 | package_split.pv, settings=vartree.settings, |
| 156 | vartree=vartree) |
| 157 | |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 158 | return package, vartree |
| 159 | |
| 160 | |
| 161 | def DowngradePackageVersion(portage_root, package_cp, |
| 162 | downgrade_to_version='0'): |
| 163 | """ |
| 164 | Downgrade the specified portage package version. |
| 165 | |
| 166 | Arguments: |
| 167 | portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy' |
| 168 | package_cp: A string similar to 'chromeos-base/autotest-tests'. |
| 169 | downgrade_to_version: String version to downgrade to. Default: '0' |
| 170 | |
| 171 | Returns: |
| 172 | Returns the return value of the `mv` command used to perform operation. |
| 173 | """ |
| 174 | package, _ = GetPackageAPI(portage_root, package_cp) |
| 175 | |
| 176 | source_directory = package.dbdir |
| 177 | destination_path = os.path.join( |
| 178 | package.dbroot, package_cp + '-' + downgrade_to_version) |
| 179 | if os.path.abspath(source_directory) == os.path.abspath(destination_path): |
| 180 | return 0 |
| 181 | command = ['mv', source_directory, destination_path] |
| 182 | return cros_build_lib.SudoRunCommand(command).returncode |
| 183 | |
| 184 | |
| 185 | def UpdatePackageContents(change_report, package_cp, |
| 186 | portage_root=None): |
| 187 | """ |
| 188 | Add newly created files/directors to package contents. |
| 189 | |
| 190 | Given an ItemizedChangeReport, add the newly created files and directories |
| 191 | to the CONTENTS of an installed portage package, such that these files are |
| 192 | considered owned by that package. |
| 193 | |
| 194 | Arguments: |
| 195 | changereport: ItemizedChangeReport object for the changes to be |
| 196 | made to the package. |
| 197 | package_cp: A string similar to 'chromeos-base/autotest-tests' giving |
| 198 | the package category and name of the package to be altered. |
| 199 | portage_root: Portage root path, corresponding to the board that |
| 200 | we are working on. Defaults to '/' |
| 201 | """ |
| 202 | package, vartree = GetPackageAPI(portage_root, package_cp) |
| 203 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 204 | # Append new contents to package contents dictionary |
| 205 | contents = package.getcontents().copy() |
| 206 | for _, filename in change_report.new_files: |
| 207 | contents.setdefault(filename, (u'obj', '0', '0')) |
| 208 | for _, dirname in change_report.new_directories: |
| 209 | # String trailing slashes if present. |
| 210 | dirname = dirname.rstrip('/') |
| 211 | contents.setdefault(dirname, (u'dir',)) |
| 212 | |
| 213 | # Write new contents dictionary to file |
| 214 | vartree.dbapi.writeContentsToContentsFile(package, contents) |
| 215 | |
| 216 | |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 217 | def RemoveTestPackages(stale_packages, autotest_sysroot): |
| 218 | """ |
| 219 | Remove bzipped test packages from sysroot. |
| 220 | |
| 221 | Arguments: |
| 222 | stale_packages: List of test packages names to be removed. |
| 223 | e.g. ['factory_Leds', 'login_UserPolicyKeys'] |
| 224 | autotest_sysroot: Absolute path of autotest in the sysroot, |
| 225 | e.g. '/build/lumpy/usr/local/autotest' |
| 226 | """ |
| 227 | for package in set(stale_packages): |
| 228 | package_filename = 'test-' + package + '.tar.bz2' |
| 229 | package_file_fullpath = os.path.join(autotest_sysroot, 'packages', |
| 230 | package_filename) |
| 231 | try: |
| 232 | os.remove(package_file_fullpath) |
| 233 | logging.info('Removed stale %s', package_file_fullpath) |
| 234 | except OSError as err: |
| 235 | # Suppress no-such-file exceptions. Raise all others. |
| 236 | if err.errno != 2: |
| 237 | raise |
| 238 | |
| 239 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 240 | def RsyncQuickmerge(source_path, sysroot_autotest_path, |
| 241 | include_pattern_file=None, pretend=False, |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 242 | overwrite=False): |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 243 | """Run rsync quickmerge command, with specified arguments. |
| 244 | Command will take form `rsync -a [options] --exclude=**.pyc |
| 245 | --exclude=**.pyo |
| 246 | [optional --include-from argument] |
| 247 | --exclude=* [source_path] [sysroot_autotest_path]` |
| 248 | |
| 249 | Arguments: |
| 250 | pretend: True to use the '-n' option to rsync, to perform dry run. |
| 251 | overwrite: True to omit '-u' option, overwrite all files in sysroot, |
| 252 | not just older files. |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 253 | """ |
| 254 | command = ['rsync', '-a'] |
| 255 | |
| 256 | if pretend: |
| 257 | command += ['-n'] |
| 258 | |
| 259 | if not overwrite: |
| 260 | command += ['-u'] |
| 261 | |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 262 | command += ['-i'] |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 263 | |
| 264 | command += ['--exclude=**.pyc'] |
| 265 | command += ['--exclude=**.pyo'] |
| 266 | |
Aviv Keshet | 787ffcd | 2013-04-08 15:14:56 -0700 | [diff] [blame] | 267 | # Exclude files with a specific substring in their name, because |
| 268 | # they create an ambiguous itemized report. (see unit test file for details) |
| 269 | command += ['--exclude=** -> *'] |
| 270 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 271 | if include_pattern_file: |
| 272 | command += ['--include-from=%s' % include_pattern_file] |
| 273 | |
| 274 | command += ['--exclude=*'] |
| 275 | |
| 276 | command += [source_path, sysroot_autotest_path] |
| 277 | |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 278 | return cros_build_lib.SudoRunCommand(command, redirect_stdout=True) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 279 | |
| 280 | |
| 281 | def ParseArguments(argv): |
| 282 | """Parse command line arguments |
| 283 | |
| 284 | Returns: parsed arguments. |
| 285 | """ |
| 286 | parser = argparse.ArgumentParser(description='Perform a fast approximation ' |
| 287 | 'to emerge-$board autotest-all, by ' |
| 288 | 'rsyncing source tree to sysroot.') |
| 289 | |
| 290 | parser.add_argument('--board', metavar='BOARD', default=None, required=True) |
| 291 | parser.add_argument('--pretend', action='store_true', |
| 292 | help='Dry run only, do not modify sysroot autotest.') |
| 293 | parser.add_argument('--overwrite', action='store_true', |
| 294 | help='Overwrite existing files even if newer.') |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 295 | parser.add_argument('--verbose', action='store_true', |
| 296 | help='Print detailed change report.') |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 297 | |
| 298 | return parser.parse_args(argv) |
| 299 | |
| 300 | |
| 301 | def main(argv): |
| 302 | cros_build_lib.AssertInsideChroot() |
| 303 | |
| 304 | args = ParseArguments(argv) |
| 305 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 306 | if not os.geteuid()==0: |
| 307 | try: |
| 308 | cros_build_lib.SudoRunCommand([sys.executable] + sys.argv) |
| 309 | except cros_build_lib.RunCommandError: |
| 310 | return 1 |
| 311 | return 0 |
| 312 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 313 | if not args.board: |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 314 | print 'No board specified. Aborting.' |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 315 | return 1 |
| 316 | |
| 317 | manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT) |
| 318 | source_path = manifest.GetProjectPath(AUTOTEST_PROJECT_NAME, absolute=True) |
| 319 | source_path = os.path.join(source_path, '') |
| 320 | |
| 321 | script_path = os.path.dirname(__file__) |
| 322 | include_pattern_file = os.path.join(script_path, INCLUDE_PATTERNS_FILENAME) |
| 323 | |
| 324 | # TODO: Determine the following string programatically. |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 325 | sysroot_path = os.path.join('/build', args.board, '') |
| 326 | sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local', |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 327 | 'autotest', '') |
| 328 | |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 329 | rsync_output = RsyncQuickmerge(source_path, sysroot_autotest_path, |
| 330 | include_pattern_file, args.pretend, args.overwrite) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 331 | |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 332 | if args.verbose: |
| 333 | logging.info(rsync_output.output) |
| 334 | |
Aviv Keshet | 60968ec | 2013-04-11 18:44:14 -0700 | [diff] [blame] | 335 | change_report = ItemizeChangesFromRsyncOutput(rsync_output.output, |
| 336 | sysroot_autotest_path) |
| 337 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 338 | if not args.pretend: |
| 339 | UpdatePackageContents(change_report, AUTOTEST_TESTS_EBUILD, |
| 340 | sysroot_path) |
Aviv Keshet | 3cc4e9e | 2013-04-24 10:47:23 -0700 | [diff] [blame^] | 341 | for ebuild in DOWNGRADE_EBUILDS: |
| 342 | if DowngradePackageVersion(sysroot_path, ebuild) != 0: |
| 343 | logging.warning('Unable to downgrade package %s version number.', |
| 344 | ebuild) |
Aviv Keshet | 75d6596 | 2013-04-17 16:15:23 -0700 | [diff] [blame] | 345 | stale_packages = GetStalePackageNames( |
| 346 | change_report.new_files + change_report.modified_files, |
| 347 | sysroot_autotest_path) |
| 348 | RemoveTestPackages(stale_packages, sysroot_autotest_path) |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 349 | |
Aviv Keshet | 940c17f | 2013-04-11 18:41:42 -0700 | [diff] [blame] | 350 | if args.pretend: |
Aviv Keshet | e00caeb | 2013-04-17 14:03:25 -0700 | [diff] [blame] | 351 | logging.info('The following message is pretend only. No filesystem ' |
| 352 | 'changes made.') |
| 353 | logging.info('Quickmerge complete. Created or modified %s files.', |
| 354 | len(change_report.new_files) + len(change_report.modified_files)) |
| 355 | |
| 356 | return 0 |
| 357 | |
Aviv Keshet | b1238c3 | 2013-04-01 11:42:13 -0700 | [diff] [blame] | 358 | |
| 359 | if __name__ == '__main__': |
| 360 | sys.exit(main(sys.argv)) |