blob: 2d5db92101590ea4d9b8418384c8a75ac9f1a2f6 [file] [log] [blame]
Mike Frysinger110750a2012-03-26 14:19:20 -04001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Chris Sosadad0d322011-01-31 16:37:33 -08002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""This module uprevs a given package's ebuild to the next revision."""
6
Mike Frysinger383367e2014-09-16 15:06:17 -04007from __future__ import print_function
8
Chris Sosa62ad8522011-03-08 17:46:17 -08009import optparse
Chris Sosadad0d322011-01-31 16:37:33 -080010import os
Chris Sosadad0d322011-01-31 16:37:33 -080011import sys
12
Don Garrett88b8d782014-05-13 17:30:55 -070013from chromite.cbuildbot import constants
Chris Sosac13bba52011-05-24 15:14:09 -070014from chromite.lib import cros_build_lib
David James97d95872012-11-16 15:09:56 -080015from chromite.lib import git
Mike Frysingerfddaeb52012-11-20 11:17:31 -050016from chromite.lib import osutils
David James6450a0a2012-12-04 07:59:53 -080017from chromite.lib import parallel
Alex Deymo075c2292014-09-04 18:31:50 -070018from chromite.lib import portage_util
Chris Sosac13bba52011-05-24 15:14:09 -070019
David James15ed1302013-04-25 09:21:19 -070020
David James29e86d52013-04-19 09:41:29 -070021# Commit message for uprevving Portage packages.
22_GIT_COMMIT_MESSAGE = 'Marking 9999 ebuild for %s as stable.'
23
Chris Sosadad0d322011-01-31 16:37:33 -080024# Dictionary of valid commands with usage information.
25COMMAND_DICTIONARY = {
Mike Frysinger5cd8c742013-10-11 14:43:01 -040026 'commit': 'Marks given ebuilds as stable locally',
27 'push': 'Pushes previous marking of ebuilds to remote repo',
28}
Chris Sosadad0d322011-01-31 16:37:33 -080029
Chris Sosadad0d322011-01-31 16:37:33 -080030
Chris Sosadad0d322011-01-31 16:37:33 -080031# ======================= Global Helper Functions ========================
32
33
David Jamescc09c9b2012-01-26 22:10:13 -080034def CleanStalePackages(boards, package_atoms):
Chris Sosabf153872011-04-28 14:21:09 -070035 """Cleans up stale package info from a previous build.
Mike Frysinger5cd8c742013-10-11 14:43:01 -040036
Chris Sosabf153872011-04-28 14:21:09 -070037 Args:
David Jamescc09c9b2012-01-26 22:10:13 -080038 boards: Boards to clean the packages from.
Mike Frysingerde5ab0e2013-03-21 20:48:36 -040039 package_atoms: A list of package atoms to unmerge.
Chris Sosabf153872011-04-28 14:21:09 -070040 """
David James15ed1302013-04-25 09:21:19 -070041 if package_atoms:
42 cros_build_lib.Info('Cleaning up stale packages %s.' % package_atoms)
43
Mike Frysingerb7ab9b82012-04-04 16:22:43 -040044 # First unmerge all the packages for a board, then eclean it.
45 # We need these two steps to run in order (unmerge/eclean),
46 # but we can let all the boards run in parallel.
47 def _CleanStalePackages(board):
48 if board:
49 suffix = '-' + board
50 runcmd = cros_build_lib.RunCommand
51 else:
52 suffix = ''
53 runcmd = cros_build_lib.SudoRunCommand
Chris Sosadad0d322011-01-31 16:37:33 -080054
David James59a0a2b2013-03-22 14:04:44 -070055 emerge, eclean = 'emerge' + suffix, 'eclean' + suffix
56 if not osutils.FindMissingBinaries([emerge, eclean]):
David James63841a82014-01-16 14:39:24 -080057 if package_atoms:
58 # If nothing was found to be unmerged, emerge will exit(1).
59 result = runcmd([emerge, '-q', '--unmerge'] + package_atoms,
60 extra_env={'CLEAN_DELAY': '0'}, error_code_ok=True)
61 if not result.returncode in (0, 1):
62 raise cros_build_lib.RunCommandError('unexpected error', result)
David James59a0a2b2013-03-22 14:04:44 -070063 runcmd([eclean, '-d', 'packages'],
64 redirect_stdout=True, redirect_stderr=True)
Mike Frysingerb7ab9b82012-04-04 16:22:43 -040065
66 tasks = []
David Jamescc09c9b2012-01-26 22:10:13 -080067 for board in boards:
Mike Frysingerb7ab9b82012-04-04 16:22:43 -040068 tasks.append([board])
69 tasks.append([None])
70
David James6450a0a2012-12-04 07:59:53 -080071 parallel.RunTasksInProcessPool(_CleanStalePackages, tasks)
Chris Sosadad0d322011-01-31 16:37:33 -080072
73
Mike Frysinger2ebe3732012-05-08 17:04:12 -040074# TODO(build): This code needs to be gutted and rebased to cros_build_lib.
75def _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
Chris Sosadad0d322011-01-31 16:37:33 -080076 """Returns true if there are local commits."""
David James97d95872012-11-16 15:09:56 -080077 current_branch = git.GetCurrentBranch(cwd)
Brian Harring609dc4e2012-05-07 02:17:44 -070078
79 if current_branch != stable_branch:
Chris Sosadad0d322011-01-31 16:37:33 -080080 return False
David James97d95872012-11-16 15:09:56 -080081 output = git.RunGit(
Brian Harring609dc4e2012-05-07 02:17:44 -070082 cwd, ['rev-parse', 'HEAD', tracking_branch]).output.split()
Brian Harring5b86b5e2012-05-25 18:27:40 -070083 return output[0] != output[1]
Chris Sosadad0d322011-01-31 16:37:33 -080084
85
David James1b363582012-12-17 11:53:11 -080086def _CheckSaneArguments(command, options):
Chris Sosadad0d322011-01-31 16:37:33 -080087 """Checks to make sure the flags are sane. Dies if arguments are not sane."""
88 if not command in COMMAND_DICTIONARY.keys():
89 _PrintUsageAndDie('%s is not a valid command' % command)
Chris Sosa62ad8522011-03-08 17:46:17 -080090 if not options.packages and command == 'commit' and not options.all:
Chris Sosadad0d322011-01-31 16:37:33 -080091 _PrintUsageAndDie('Please specify at least one package')
Mike Frysinger8fd67dc2012-12-03 23:51:18 -050092 if options.boards:
93 cros_build_lib.AssertInsideChroot()
Chris Sosa62ad8522011-03-08 17:46:17 -080094 if not os.path.isdir(options.srcroot):
Chris Sosadad0d322011-01-31 16:37:33 -080095 _PrintUsageAndDie('srcroot is not a valid path')
Chris Sosa62ad8522011-03-08 17:46:17 -080096 options.srcroot = os.path.abspath(options.srcroot)
Chris Sosadad0d322011-01-31 16:37:33 -080097
98
99def _PrintUsageAndDie(error_message=''):
100 """Prints optional error_message the usage and returns an error exit code."""
101 command_usage = 'Commands: \n'
102 # Add keys and usage information from dictionary.
103 commands = sorted(COMMAND_DICTIONARY.keys())
104 for command in commands:
105 command_usage += ' %s: %s\n' % (command, COMMAND_DICTIONARY[command])
106 commands_str = '|'.join(commands)
Chris Sosac13bba52011-05-24 15:14:09 -0700107 cros_build_lib.Warning('Usage: %s FLAGS [%s]\n\n%s' % (
108 sys.argv[0], commands_str, command_usage))
Chris Sosadad0d322011-01-31 16:37:33 -0800109 if error_message:
Chris Sosac13bba52011-05-24 15:14:09 -0700110 cros_build_lib.Die(error_message)
Chris Sosadad0d322011-01-31 16:37:33 -0800111 else:
112 sys.exit(1)
113
114
Chris Sosadad0d322011-01-31 16:37:33 -0800115# ======================= End Global Helper Functions ========================
116
117
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400118def PushChange(stable_branch, tracking_branch, dryrun, cwd):
Chris Sosadad0d322011-01-31 16:37:33 -0800119 """Pushes commits in the stable_branch to the remote git repository.
120
David Jamesee2da622012-02-23 09:32:16 -0800121 Pushes local commits from calls to CommitChange to the remote git
David James66009462012-03-25 10:08:38 -0700122 repository specified by current working directory. If changes are
123 found to commit, they will be merged to the merge branch and pushed.
124 In that case, the local repository will be left on the merge branch.
Chris Sosadad0d322011-01-31 16:37:33 -0800125
126 Args:
127 stable_branch: The local branch with commits we want to push.
128 tracking_branch: The tracking branch of the local branch.
Chris Sosa62ad8522011-03-08 17:46:17 -0800129 dryrun: Use git push --dryrun to emulate a push.
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400130 cwd: The directory to run commands in.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500131
Chris Sosadad0d322011-01-31 16:37:33 -0800132 Raises:
Mike Frysinger5cd8c742013-10-11 14:43:01 -0400133 OSError: Error occurred while pushing.
Chris Sosadad0d322011-01-31 16:37:33 -0800134 """
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400135 if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
Brian Harring609dc4e2012-05-07 02:17:44 -0700136 cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
Chris Sosadad0d322011-01-31 16:37:33 -0800137 return
138
David James66009462012-03-25 10:08:38 -0700139 # For the commit queue, our local branch may contain commits that were
140 # just tested and pushed during the CommitQueueCompletion stage. Sync
141 # and rebase our local branch on top of the remote commits.
David James97d95872012-11-16 15:09:56 -0800142 remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
143 git.SyncPushBranch(cwd, remote, push_branch)
David James66009462012-03-25 10:08:38 -0700144
145 # Check whether any local changes remain after the sync.
Brian Harring609dc4e2012-05-07 02:17:44 -0700146 if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
147 cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
David James66009462012-03-25 10:08:38 -0700148 return
149
Matt Tennantcb522052013-11-25 14:23:43 -0800150 # Add a failsafe check here. Only CLs from the 'chrome-bot' user should
151 # be involved here. If any other CLs are found then complain.
152 # In dryruns extra CLs are normal, though, and can be ignored.
153 bad_cl_cmd = ['log', '--format=short', '--perl-regexp',
154 '--author', '^(?!chrome-bot)', '%s..%s' % (
155 push_branch, stable_branch)]
156 bad_cls = git.RunGit(cwd, bad_cl_cmd).output
157 if bad_cls.strip() and not dryrun:
158 cros_build_lib.Error('The Uprev stage found changes from users other'
159 ' than chrome-bot:\n\n%s', bad_cls)
160 raise AssertionError('Unexpected CLs found during uprev stage.')
161
David James67d73252013-09-19 17:33:12 -0700162 description = git.RunGit(cwd,
163 ['log', '--format=format:%s%n%n%b', '%s..%s' % (
164 push_branch, stable_branch)]).output
Chris Sosadad0d322011-01-31 16:37:33 -0800165 description = 'Marking set of ebuilds as stable\n\n%s' % description
Brian Harring609dc4e2012-05-07 02:17:44 -0700166 cros_build_lib.Info('For %s, using description %s', cwd, description)
David James97d95872012-11-16 15:09:56 -0800167 git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
168 git.RunGit(cwd, ['merge', '--squash', stable_branch])
169 git.RunGit(cwd, ['commit', '-m', description])
170 git.RunGit(cwd, ['config', 'push.default', 'tracking'])
171 git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
Chris Sosadad0d322011-01-31 16:37:33 -0800172
173
174class GitBranch(object):
175 """Wrapper class for a git branch."""
176
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400177 def __init__(self, branch_name, tracking_branch, cwd):
David Jamesc7c4ff52013-09-18 17:57:13 -0700178 """Sets up variables but does not create the branch.
179
Mike Frysinger5cd8c742013-10-11 14:43:01 -0400180 Args:
David Jamesc7c4ff52013-09-18 17:57:13 -0700181 branch_name: The name of the branch.
182 tracking_branch: The associated tracking branch.
183 cwd: The git repository to work in.
184 """
Chris Sosadad0d322011-01-31 16:37:33 -0800185 self.branch_name = branch_name
186 self.tracking_branch = tracking_branch
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400187 self.cwd = cwd
Chris Sosadad0d322011-01-31 16:37:33 -0800188
189 def CreateBranch(self):
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400190 self.Checkout()
Chris Sosadad0d322011-01-31 16:37:33 -0800191
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400192 def Checkout(self, branch=None):
Chris Sosadad0d322011-01-31 16:37:33 -0800193 """Function used to check out to another GitBranch."""
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400194 if not branch:
195 branch = self.branch_name
196 if branch == self.tracking_branch or self.Exists(branch):
197 git_cmd = ['git', 'checkout', '-f', branch]
Chris Sosadad0d322011-01-31 16:37:33 -0800198 else:
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400199 git_cmd = ['repo', 'start', branch, '.']
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800200 cros_build_lib.RunCommand(git_cmd, print_cmd=False, cwd=self.cwd,
201 capture_output=True)
Chris Sosadad0d322011-01-31 16:37:33 -0800202
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400203 def Exists(self, branch=None):
Chris Sosadad0d322011-01-31 16:37:33 -0800204 """Returns True if the branch exists."""
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400205 if not branch:
206 branch = self.branch_name
David James67d73252013-09-19 17:33:12 -0700207 branches = git.RunGit(self.cwd, ['branch']).output
Mike Frysinger2ebe3732012-05-08 17:04:12 -0400208 return branch in branches.split()
Chris Sosadad0d322011-01-31 16:37:33 -0800209
210
David James1b363582012-12-17 11:53:11 -0800211def main(_argv):
Chris Sosa62ad8522011-03-08 17:46:17 -0800212 parser = optparse.OptionParser('cros_mark_as_stable OPTIONS packages')
213 parser.add_option('--all', action='store_true',
214 help='Mark all packages as stable.')
Mike Frysinger587bd562012-11-19 16:41:39 -0500215 parser.add_option('-b', '--boards', default='',
David Jamescc09c9b2012-01-26 22:10:13 -0800216 help='Colon-separated list of boards')
Chris Sosa62ad8522011-03-08 17:46:17 -0800217 parser.add_option('--drop_file',
218 help='File to list packages that were revved.')
219 parser.add_option('--dryrun', action='store_true',
220 help='Passes dry-run to git push if pushing a change.')
221 parser.add_option('-o', '--overlays',
222 help='Colon-separated list of overlays to modify.')
223 parser.add_option('-p', '--packages',
224 help='Colon separated list of packages to rev.')
225 parser.add_option('-r', '--srcroot',
Mike Frysingerfddaeb52012-11-20 11:17:31 -0500226 default=os.path.join(constants.SOURCE_ROOT, 'src'),
Chris Sosa62ad8522011-03-08 17:46:17 -0800227 help='Path to root src directory.')
Chris Sosa62ad8522011-03-08 17:46:17 -0800228 parser.add_option('--verbose', action='store_true',
229 help='Prints out debug info.')
230 (options, args) = parser.parse_args()
Chris Sosadad0d322011-01-31 16:37:33 -0800231
Alex Deymo075c2292014-09-04 18:31:50 -0700232 portage_util.EBuild.VERBOSE = options.verbose
Chris Sosa62ad8522011-03-08 17:46:17 -0800233
234 if len(args) != 1:
Ryan Cui05a31ba2011-05-31 17:47:37 -0700235 _PrintUsageAndDie('Must specify a valid command [commit, push]')
Chris Sosa62ad8522011-03-08 17:46:17 -0800236
237 command = args[0]
238 package_list = None
239 if options.packages:
240 package_list = options.packages.split(':')
241
David James1b363582012-12-17 11:53:11 -0800242 _CheckSaneArguments(command, options)
Chris Sosa62ad8522011-03-08 17:46:17 -0800243 if options.overlays:
Chris Sosadad0d322011-01-31 16:37:33 -0800244 overlays = {}
Chris Sosa62ad8522011-03-08 17:46:17 -0800245 for path in options.overlays.split(':'):
Ryan Cui05a31ba2011-05-31 17:47:37 -0700246 if not os.path.isdir(path):
Chris Sosac13bba52011-05-24 15:14:09 -0700247 cros_build_lib.Die('Cannot find overlay: %s' % path)
Chris Sosadad0d322011-01-31 16:37:33 -0800248 overlays[path] = []
249 else:
Chris Sosac13bba52011-05-24 15:14:09 -0700250 cros_build_lib.Warning('Missing --overlays argument')
Chris Sosadad0d322011-01-31 16:37:33 -0800251 overlays = {
Chris Sosa62ad8522011-03-08 17:46:17 -0800252 '%s/private-overlays/chromeos-overlay' % options.srcroot: [],
253 '%s/third_party/chromiumos-overlay' % options.srcroot: []
Chris Sosadad0d322011-01-31 16:37:33 -0800254 }
255
David James97d95872012-11-16 15:09:56 -0800256 manifest = git.ManifestCheckout.Cached(options.srcroot)
Ryan Cui4656a3c2011-05-24 12:30:30 -0700257
David James84e953c2013-04-23 18:44:06 -0700258 if command == 'commit':
Alex Deymo075c2292014-09-04 18:31:50 -0700259 portage_util.BuildEBuildDictionary(overlays, options.all, package_list)
David James84e953c2013-04-23 18:44:06 -0700260
David James15ed1302013-04-25 09:21:19 -0700261 # Contains the array of packages we actually revved.
262 revved_packages = []
263 new_package_atoms = []
David Jamesa8457b52011-05-28 00:03:20 -0700264
David James15ed1302013-04-25 09:21:19 -0700265 # Slight optimization hack: process the chromiumos overlay before any other
266 # cros-workon overlay first so we can do background cache generation in it.
267 # A perfect solution would walk all the overlays, figure out any dependencies
268 # between them (with layout.conf), and then process them in dependency order.
269 # However, this operation isn't slow enough to warrant that level of
270 # complexity, so we'll just special case the main overlay.
271 #
272 # Similarly, generate the cache in the portage-stable tree asap. We know
273 # we won't have any cros-workon packages in there, so generating the cache
274 # is the only thing it'll be doing. The chromiumos overlay instead might
275 # have revbumping to do before it can generate the cache.
276 keys = overlays.keys()
277 for overlay in ('/third_party/chromiumos-overlay',
278 '/third_party/portage-stable'):
279 for k in keys:
280 if k.endswith(overlay):
281 keys.remove(k)
282 keys.insert(0, k)
283 break
Chris Sosadad0d322011-01-31 16:37:33 -0800284
Alex Deymo075c2292014-09-04 18:31:50 -0700285 with parallel.BackgroundTaskRunner(portage_util.RegenCache) as queue:
David James15ed1302013-04-25 09:21:19 -0700286 for overlay in keys:
287 ebuilds = overlays[overlay]
288 if not os.path.isdir(overlay):
Mike Frysinger5cd8c742013-10-11 14:43:01 -0400289 cros_build_lib.Warning('Skipping %s' % overlay)
David James15ed1302013-04-25 09:21:19 -0700290 continue
Chris Sosadad0d322011-01-31 16:37:33 -0800291
David James15ed1302013-04-25 09:21:19 -0700292 # Note we intentionally work from the non push tracking branch;
293 # everything built thus far has been against it (meaning, http mirrors),
294 # thus we should honor that. During the actual push, the code switches
295 # to the correct urls, and does an appropriate rebasing.
296 tracking_branch = git.GetTrackingBranchViaManifest(
297 overlay, manifest=manifest)[1]
Brian Harring609dc4e2012-05-07 02:17:44 -0700298
David James15ed1302013-04-25 09:21:19 -0700299 if command == 'push':
300 PushChange(constants.STABLE_EBUILD_BRANCH, tracking_branch,
301 options.dryrun, cwd=overlay)
302 elif command == 'commit':
David James98b77f52013-11-19 10:11:56 -0800303 existing_commit = git.GetGitRepoRevision(overlay)
David James15ed1302013-04-25 09:21:19 -0700304 work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch,
305 cwd=overlay)
306 work_branch.CreateBranch()
307 if not work_branch.Exists():
308 cros_build_lib.Die('Unable to create stabilizing branch in %s' %
309 overlay)
310
311 # In the case of uprevving overlays that have patches applied to them,
312 # include the patched changes in the stabilizing branch.
David James98b77f52013-11-19 10:11:56 -0800313 git.RunGit(overlay, ['rebase', existing_commit])
David James15ed1302013-04-25 09:21:19 -0700314
315 messages = []
316 for ebuild in ebuilds:
317 if options.verbose:
318 cros_build_lib.Info('Working on %s', ebuild.package)
319 try:
320 new_package = ebuild.RevWorkOnEBuild(options.srcroot, manifest)
321 if new_package:
322 revved_packages.append(ebuild.package)
323 new_package_atoms.append('=%s' % new_package)
324 messages.append(_GIT_COMMIT_MESSAGE % ebuild.package)
325 except (OSError, IOError):
Mike Frysinger5cd8c742013-10-11 14:43:01 -0400326 cros_build_lib.Warning(
327 'Cannot rev %s\n'
328 'Note you will have to go into %s '
329 'and reset the git repo yourself.' % (ebuild.package, overlay))
David James15ed1302013-04-25 09:21:19 -0700330 raise
331
332 if messages:
Alex Deymo075c2292014-09-04 18:31:50 -0700333 portage_util.EBuild.CommitChange('\n\n'.join(messages), overlay)
David James15ed1302013-04-25 09:21:19 -0700334
335 if cros_build_lib.IsInsideChroot():
336 # Regenerate caches if need be. We do this all the time to
337 # catch when users make changes without updating cache files.
338 queue.put([overlay])
339
340 if command == 'commit':
341 if cros_build_lib.IsInsideChroot():
342 CleanStalePackages(options.boards.split(':'), new_package_atoms)
343 if options.drop_file:
344 osutils.WriteFile(options.drop_file, ' '.join(revved_packages))