blob: f0656c359c2e11c7079daabf4bb9137308f8150d [file] [log] [blame]
Aviv Keshetb1238c32013-04-01 11:42:13 -07001#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""
9Simple script to be run inside the chroot. Used as a fast approximation of
10emerge-$board autotest-all, by simply rsync'ing changes from trunk to sysroot.
11"""
12
Aviv Keshete00caeb2013-04-17 14:03:25 -070013import logging
Aviv Keshetb1238c32013-04-01 11:42:13 -070014import os
Aviv Keshet787ffcd2013-04-08 15:14:56 -070015import re
Aviv Keshetb1238c32013-04-01 11:42:13 -070016import sys
Aviv Keshet787ffcd2013-04-08 15:14:56 -070017from collections import namedtuple
18
Aviv Keshetb1238c32013-04-01 11:42:13 -070019from chromite.buildbot import constants
Aviv Keshet940c17f2013-04-11 18:41:42 -070020from chromite.buildbot import portage_utilities
Aviv Keshetb1238c32013-04-01 11:42:13 -070021from chromite.lib import cros_build_lib
22from chromite.lib import git
23
24import argparse
25
Aviv Keshet940c17f2013-04-11 18:41:42 -070026if cros_build_lib.IsInsideChroot():
27 # Only import portage after we've checked that we're inside the chroot.
28 import portage
29
Aviv Keshetb1238c32013-04-01 11:42:13 -070030INCLUDE_PATTERNS_FILENAME = 'autotest-quickmerge-includepatterns'
31AUTOTEST_PROJECT_NAME = 'chromiumos/third_party/autotest'
Aviv Keshet940c17f2013-04-11 18:41:42 -070032AUTOTEST_TESTS_EBUILD = 'chromeos-base/autotest-tests'
Aviv Keshet3cc4e9e2013-04-24 10:47:23 -070033DOWNGRADE_EBUILDS = ['chromeos-base/autotest',
34 'chromeos-base/autotest-tests',
35 'chromeos-base/autotest-chrome',
36 'chromeos-base/autotest-factory',
37 'chromeos-base/autotest-telemetry',
38 'chromeos-base/autotest-tests-ltp',
39 'chromeos-base/autotest-tests-ownershipapi']
Aviv Keshet787ffcd2013-04-08 15:14:56 -070040
41# Data structure describing a single rsync filesystem change.
42#
43# change_description: An 11 character string, the rsync change description
44# for the particular file.
45# absolute_path: The absolute path of the created or modified file.
46ItemizedChange = namedtuple('ItemizedChange', ['change_description',
47 'absolute_path'])
48
49
50# Data structure describing the rsync new/modified files or directories.
51#
52# new_files: A list of ItemizedChange objects for new files.
53# modified_files: A list of ItemizedChange objects for modified files.
54# new_directories: A list of ItemizedChange objects for new directories.
55ItemizedChangeReport = namedtuple('ItemizedChangeReport',
56 ['new_files', 'modified_files',
57 'new_directories'])
58
59
Aviv Keshet75d65962013-04-17 16:15:23 -070060def GetStalePackageNames(change_list, autotest_sysroot):
61 """
62 Given a rsync change report, returns the names of stale test packages.
63
64 This function pulls out test package names for client-side tests, stored
65 within the client/site_tests directory tree, that had any files added or
66 modified and for whom any existing bzipped test packages may now be stale.
67
68 Arguments:
69 change_list: A list of ItemizedChange objects corresponding to changed
70 or modified files.
71 autotest_sysroot: Absolute path of autotest in the sysroot,
72 e.g. '/build/lumpy/usr/local/autotest'
73
74 Returns:
75 A list of test package names, eg ['factory_Leds', 'login_UserPolicyKeys'].
76 May contain duplicate entries if multiple files within a test directory
77 were modified.
78 """
79 exp = os.path.abspath(autotest_sysroot) + r'/client/site_tests/(.*?)/.*'
80 matches = [re.match(exp, change.absolute_path) for change in change_list]
81 return [match.group(1) for match in matches if match]
82
83
Aviv Keshet787ffcd2013-04-08 15:14:56 -070084def ItemizeChangesFromRsyncOutput(rsync_output, destination_path):
85 """Convert the output of an rsync with `-i` to a ItemizedChangeReport object.
86
87 Arguments:
88 rsync_output: String stdout of rsync command that was run with `-i` option.
89 destination_path: String absolute path of the destination directory for the
90 rsync operations. This argument is necessary because
91 rsync's output only gives the relative path of
92 touched/added files.
93
94 Returns:
95 ItemizedChangeReport object giving the absolute paths of files that were
96 created or modified by rsync.
97 """
98 modified_matches = re.findall(r'([.>]f[^+]{9}) (.*)', rsync_output)
99 new_matches = re.findall(r'(>f\+{9}) (.*)', rsync_output)
100 new_symlink_matches = re.findall(r'(cL\+{9}) (.*) -> .*', rsync_output)
101 new_dir_matches = re.findall(r'(cd\+{9}) (.*)', rsync_output)
102
103 absolute_modified = [ItemizedChange(c, os.path.join(destination_path, f))
104 for (c, f) in modified_matches]
105
106 # Note: new symlinks are treated as new files.
107 absolute_new = [ItemizedChange(c, os.path.join(destination_path, f))
108 for (c, f) in new_matches + new_symlink_matches]
109
110 absolute_new_dir = [ItemizedChange(c, os.path.join(destination_path, f))
111 for (c, f) in new_dir_matches]
112
113 return ItemizedChangeReport(new_files=absolute_new,
114 modified_files=absolute_modified,
115 new_directories=absolute_new_dir)
116
117
Aviv Keshete00caeb2013-04-17 14:03:25 -0700118def GetPackageAPI(portage_root, package_cp):
Aviv Keshet940c17f2013-04-11 18:41:42 -0700119 """
Aviv Keshete00caeb2013-04-17 14:03:25 -0700120 Gets portage API handles for the given package.
Aviv Keshet940c17f2013-04-11 18:41:42 -0700121
122 Arguments:
Aviv Keshete00caeb2013-04-17 14:03:25 -0700123 portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy'
124 package_cp: A string similar to 'chromeos-base/autotest-tests'.
125
126 Returns:
127 Returns (package, vartree) tuple, where
128 package is of type portage.dbapi.vartree.dblink
129 vartree is of type portage.dbapi.vartree.vartree
Aviv Keshet940c17f2013-04-11 18:41:42 -0700130 """
131 if portage_root is None:
132 portage_root = portage.root # pylint: disable-msg=E1101
133 # Ensure that portage_root ends with trailing slash.
134 portage_root = os.path.join(portage_root, '')
135
136 # Create vartree object corresponding to portage_root
137 trees = portage.create_trees(portage_root, portage_root)
138 vartree = trees[portage_root]['vartree']
139
140 # List matching installed packages in cpv format
141 matching_packages = vartree.dbapi.cp_list(package_cp)
142
143 if not matching_packages:
144 raise ValueError('No matching package for %s in portage_root %s' % (
145 package_cp, portage_root))
146
147 if len(matching_packages) > 1:
148 raise ValueError('Too many matching packages for %s in portage_root '
149 '%s' % (package_cp, portage_root))
150
151 # Convert string match to package dblink
152 package_cpv = matching_packages[0]
153 package_split = portage_utilities.SplitCPV(package_cpv)
154 package = portage.dblink(package_split.category, # pylint: disable-msg=E1101
155 package_split.pv, settings=vartree.settings,
156 vartree=vartree)
157
Aviv Keshete00caeb2013-04-17 14:03:25 -0700158 return package, vartree
159
160
161def DowngradePackageVersion(portage_root, package_cp,
162 downgrade_to_version='0'):
163 """
164 Downgrade the specified portage package version.
165
166 Arguments:
167 portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy'
168 package_cp: A string similar to 'chromeos-base/autotest-tests'.
169 downgrade_to_version: String version to downgrade to. Default: '0'
170
171 Returns:
172 Returns the return value of the `mv` command used to perform operation.
173 """
174 package, _ = GetPackageAPI(portage_root, package_cp)
175
176 source_directory = package.dbdir
177 destination_path = os.path.join(
178 package.dbroot, package_cp + '-' + downgrade_to_version)
179 if os.path.abspath(source_directory) == os.path.abspath(destination_path):
180 return 0
181 command = ['mv', source_directory, destination_path]
182 return cros_build_lib.SudoRunCommand(command).returncode
183
184
185def UpdatePackageContents(change_report, package_cp,
186 portage_root=None):
187 """
188 Add newly created files/directors to package contents.
189
190 Given an ItemizedChangeReport, add the newly created files and directories
191 to the CONTENTS of an installed portage package, such that these files are
192 considered owned by that package.
193
194 Arguments:
195 changereport: ItemizedChangeReport object for the changes to be
196 made to the package.
197 package_cp: A string similar to 'chromeos-base/autotest-tests' giving
198 the package category and name of the package to be altered.
199 portage_root: Portage root path, corresponding to the board that
200 we are working on. Defaults to '/'
201 """
202 package, vartree = GetPackageAPI(portage_root, package_cp)
203
Aviv Keshet940c17f2013-04-11 18:41:42 -0700204 # Append new contents to package contents dictionary
205 contents = package.getcontents().copy()
206 for _, filename in change_report.new_files:
207 contents.setdefault(filename, (u'obj', '0', '0'))
208 for _, dirname in change_report.new_directories:
209 # String trailing slashes if present.
210 dirname = dirname.rstrip('/')
211 contents.setdefault(dirname, (u'dir',))
212
213 # Write new contents dictionary to file
214 vartree.dbapi.writeContentsToContentsFile(package, contents)
215
216
Aviv Keshet75d65962013-04-17 16:15:23 -0700217def RemoveTestPackages(stale_packages, autotest_sysroot):
218 """
219 Remove bzipped test packages from sysroot.
220
221 Arguments:
222 stale_packages: List of test packages names to be removed.
223 e.g. ['factory_Leds', 'login_UserPolicyKeys']
224 autotest_sysroot: Absolute path of autotest in the sysroot,
225 e.g. '/build/lumpy/usr/local/autotest'
226 """
227 for package in set(stale_packages):
228 package_filename = 'test-' + package + '.tar.bz2'
229 package_file_fullpath = os.path.join(autotest_sysroot, 'packages',
230 package_filename)
231 try:
232 os.remove(package_file_fullpath)
233 logging.info('Removed stale %s', package_file_fullpath)
234 except OSError as err:
235 # Suppress no-such-file exceptions. Raise all others.
236 if err.errno != 2:
237 raise
238
239
Aviv Keshetb1238c32013-04-01 11:42:13 -0700240def RsyncQuickmerge(source_path, sysroot_autotest_path,
241 include_pattern_file=None, pretend=False,
Aviv Keshet60968ec2013-04-11 18:44:14 -0700242 overwrite=False):
Aviv Keshetb1238c32013-04-01 11:42:13 -0700243 """Run rsync quickmerge command, with specified arguments.
244 Command will take form `rsync -a [options] --exclude=**.pyc
245 --exclude=**.pyo
246 [optional --include-from argument]
247 --exclude=* [source_path] [sysroot_autotest_path]`
248
249 Arguments:
250 pretend: True to use the '-n' option to rsync, to perform dry run.
251 overwrite: True to omit '-u' option, overwrite all files in sysroot,
252 not just older files.
Aviv Keshetb1238c32013-04-01 11:42:13 -0700253 """
254 command = ['rsync', '-a']
255
256 if pretend:
257 command += ['-n']
258
259 if not overwrite:
260 command += ['-u']
261
Aviv Keshet60968ec2013-04-11 18:44:14 -0700262 command += ['-i']
Aviv Keshetb1238c32013-04-01 11:42:13 -0700263
264 command += ['--exclude=**.pyc']
265 command += ['--exclude=**.pyo']
266
Aviv Keshet787ffcd2013-04-08 15:14:56 -0700267 # Exclude files with a specific substring in their name, because
268 # they create an ambiguous itemized report. (see unit test file for details)
269 command += ['--exclude=** -> *']
270
Aviv Keshetb1238c32013-04-01 11:42:13 -0700271 if include_pattern_file:
272 command += ['--include-from=%s' % include_pattern_file]
273
274 command += ['--exclude=*']
275
276 command += [source_path, sysroot_autotest_path]
277
Aviv Keshet60968ec2013-04-11 18:44:14 -0700278 return cros_build_lib.SudoRunCommand(command, redirect_stdout=True)
Aviv Keshetb1238c32013-04-01 11:42:13 -0700279
280
281def ParseArguments(argv):
282 """Parse command line arguments
283
284 Returns: parsed arguments.
285 """
286 parser = argparse.ArgumentParser(description='Perform a fast approximation '
287 'to emerge-$board autotest-all, by '
288 'rsyncing source tree to sysroot.')
289
290 parser.add_argument('--board', metavar='BOARD', default=None, required=True)
291 parser.add_argument('--pretend', action='store_true',
292 help='Dry run only, do not modify sysroot autotest.')
293 parser.add_argument('--overwrite', action='store_true',
294 help='Overwrite existing files even if newer.')
Aviv Keshete00caeb2013-04-17 14:03:25 -0700295 parser.add_argument('--verbose', action='store_true',
296 help='Print detailed change report.')
Aviv Keshetb1238c32013-04-01 11:42:13 -0700297
298 return parser.parse_args(argv)
299
300
301def main(argv):
302 cros_build_lib.AssertInsideChroot()
303
304 args = ParseArguments(argv)
305
Aviv Keshet940c17f2013-04-11 18:41:42 -0700306 if not os.geteuid()==0:
307 try:
308 cros_build_lib.SudoRunCommand([sys.executable] + sys.argv)
309 except cros_build_lib.RunCommandError:
310 return 1
311 return 0
312
Aviv Keshetb1238c32013-04-01 11:42:13 -0700313 if not args.board:
Aviv Keshete00caeb2013-04-17 14:03:25 -0700314 print 'No board specified. Aborting.'
Aviv Keshetb1238c32013-04-01 11:42:13 -0700315 return 1
316
317 manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
318 source_path = manifest.GetProjectPath(AUTOTEST_PROJECT_NAME, absolute=True)
319 source_path = os.path.join(source_path, '')
320
321 script_path = os.path.dirname(__file__)
322 include_pattern_file = os.path.join(script_path, INCLUDE_PATTERNS_FILENAME)
323
324 # TODO: Determine the following string programatically.
Aviv Keshet940c17f2013-04-11 18:41:42 -0700325 sysroot_path = os.path.join('/build', args.board, '')
326 sysroot_autotest_path = os.path.join(sysroot_path, 'usr', 'local',
Aviv Keshetb1238c32013-04-01 11:42:13 -0700327 'autotest', '')
328
Aviv Keshet60968ec2013-04-11 18:44:14 -0700329 rsync_output = RsyncQuickmerge(source_path, sysroot_autotest_path,
330 include_pattern_file, args.pretend, args.overwrite)
Aviv Keshetb1238c32013-04-01 11:42:13 -0700331
Aviv Keshete00caeb2013-04-17 14:03:25 -0700332 if args.verbose:
333 logging.info(rsync_output.output)
334
Aviv Keshet60968ec2013-04-11 18:44:14 -0700335 change_report = ItemizeChangesFromRsyncOutput(rsync_output.output,
336 sysroot_autotest_path)
337
Aviv Keshet940c17f2013-04-11 18:41:42 -0700338 if not args.pretend:
339 UpdatePackageContents(change_report, AUTOTEST_TESTS_EBUILD,
340 sysroot_path)
Aviv Keshet3cc4e9e2013-04-24 10:47:23 -0700341 for ebuild in DOWNGRADE_EBUILDS:
342 if DowngradePackageVersion(sysroot_path, ebuild) != 0:
343 logging.warning('Unable to downgrade package %s version number.',
344 ebuild)
Aviv Keshet75d65962013-04-17 16:15:23 -0700345 stale_packages = GetStalePackageNames(
346 change_report.new_files + change_report.modified_files,
347 sysroot_autotest_path)
348 RemoveTestPackages(stale_packages, sysroot_autotest_path)
Aviv Keshetb1238c32013-04-01 11:42:13 -0700349
Aviv Keshet940c17f2013-04-11 18:41:42 -0700350 if args.pretend:
Aviv Keshete00caeb2013-04-17 14:03:25 -0700351 logging.info('The following message is pretend only. No filesystem '
352 'changes made.')
353 logging.info('Quickmerge complete. Created or modified %s files.',
354 len(change_report.new_files) + len(change_report.modified_files))
355
356 return 0
357
Aviv Keshetb1238c32013-04-01 11:42:13 -0700358
359if __name__ == '__main__':
360 sys.exit(main(sys.argv))