blob: 1ff3f198109d017584171964924b35fc69ee36ca [file] [log] [blame]
Brian Harring3fec5a82012-03-01 05:57:03 -08001#!/usr/bin/python
2
3# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Main builder code for Chromium OS.
8
9Used by Chromium OS buildbot configuration for all Chromium OS builds including
10full and pre-flight-queue builds.
11"""
12
13import distutils.version
14import glob
Chris Sosa4f6ffaf2012-05-01 17:05:44 -070015import logging
David James58e0c092012-03-04 20:31:12 -080016import multiprocessing
Brian Harring3fec5a82012-03-01 05:57:03 -080017import optparse
18import os
19import pprint
20import sys
Ryan Cui54da0702012-04-19 18:38:08 -070021import time
Brian Harring3fec5a82012-03-01 05:57:03 -080022
23from chromite.buildbot import builderstage as bs
24from chromite.buildbot import cbuildbot_background as background
25from chromite.buildbot import cbuildbot_config
26from chromite.buildbot import cbuildbot_stages as stages
27from chromite.buildbot import cbuildbot_results as results_lib
Brian Harring3fec5a82012-03-01 05:57:03 -080028from chromite.buildbot import constants
29from chromite.buildbot import gerrit_helper
30from chromite.buildbot import patch as cros_patch
31from chromite.buildbot import remote_try
32from chromite.buildbot import repository
33from chromite.buildbot import tee
Ryan Cui16d9e1f2012-05-11 10:50:18 -070034from chromite.buildbot import trybot_patch_pool
Brian Harring3fec5a82012-03-01 05:57:03 -080035
Brian Harringc92a7012012-02-29 10:11:34 -080036from chromite.lib import cgroups
Brian Harringa184efa2012-03-04 11:51:25 -080037from chromite.lib import cleanup
Brian Harring3fec5a82012-03-01 05:57:03 -080038from chromite.lib import cros_build_lib as cros_lib
Brian Harringaf019fb2012-05-10 15:06:13 -070039from chromite.lib import osutils
Brian Harring3fec5a82012-03-01 05:57:03 -080040from chromite.lib import sudo
41
Ryan Cuiadd49122012-03-21 22:19:58 -070042
Brian Harring3fec5a82012-03-01 05:57:03 -080043cros_lib.STRICT_SUDO = True
44
45_DEFAULT_LOG_DIR = 'cbuildbot_logs'
46_BUILDBOT_LOG_FILE = 'cbuildbot.log'
47_DEFAULT_EXT_BUILDROOT = 'trybot'
48_DEFAULT_INT_BUILDROOT = 'trybot-internal'
Brian Harring3fec5a82012-03-01 05:57:03 -080049_DISTRIBUTED_TYPES = [constants.COMMIT_QUEUE_TYPE, constants.PFQ_TYPE,
50 constants.CANARY_TYPE, constants.CHROME_PFQ_TYPE,
51 constants.PALADIN_TYPE]
Brian Harring351ce442012-03-09 16:38:14 -080052_BUILDBOT_REQUIRED_BINARIES = ('pbzip2',)
Brian Harring3fec5a82012-03-01 05:57:03 -080053
54
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070055def _PrintValidConfigs(display_all=False):
Brian Harring3fec5a82012-03-01 05:57:03 -080056 """Print a list of valid buildbot configs.
57
58 Arguments:
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070059 display_all: Print all configs. Otherwise, prints only configs with
60 trybot_list=True.
Brian Harring3fec5a82012-03-01 05:57:03 -080061 """
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070062 def _GetSortKey(config_name):
63 config_dict = cbuildbot_config.config[config_name]
64 return (not config_dict['trybot_list'], config_dict['description'],
65 config_name)
66
Brian Harring3fec5a82012-03-01 05:57:03 -080067 COLUMN_WIDTH = 45
68 print 'config'.ljust(COLUMN_WIDTH), 'description'
69 print '------'.ljust(COLUMN_WIDTH), '-----------'
70 config_names = cbuildbot_config.config.keys()
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070071 config_names.sort(key=_GetSortKey)
Brian Harring3fec5a82012-03-01 05:57:03 -080072 for name in config_names:
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070073 if display_all or cbuildbot_config.config[name]['trybot_list']:
74 desc = cbuildbot_config.config[name].get('description')
75 desc = desc if desc else ''
Brian Harring3fec5a82012-03-01 05:57:03 -080076 print name.ljust(COLUMN_WIDTH), desc
77
78
79def _GetConfig(config_name):
80 """Gets the configuration for the build"""
81 if not cbuildbot_config.config.has_key(config_name):
82 print 'Non-existent configuration %s specified.' % config_name
83 print 'Please specify one of:'
84 _PrintValidConfigs()
85 sys.exit(1)
86
87 result = cbuildbot_config.config[config_name]
88
89 return result
90
91
92def _GetChromiteTrackingBranch():
David James66009462012-03-25 10:08:38 -070093 """Returns the remote branch associated with chromite."""
Brian Harring3fec5a82012-03-01 05:57:03 -080094 cwd = os.path.dirname(os.path.realpath(__file__))
David James66009462012-03-25 10:08:38 -070095 branch = cros_lib.GetCurrentBranch(cwd)
96 if branch:
97 tracking_branch = cros_lib.GetTrackingBranch(branch, cwd)[1]
98 if tracking_branch.startswith('refs/heads/'):
99 return tracking_branch.replace('refs/heads/', '')
100 # If we are not on a branch, or if the tracking branch is a revision,
David James8b3c1bf2012-03-28 09:10:16 -0700101 # use the push branch. For repo repositories, this will be the manifest
102 # branch configured for this project. For other repositories, we'll just
103 # guess 'master', since there's no easy way to find out what branch
104 # we're on.
105 return cros_lib.GetPushBranch(cwd)[1]
Brian Harring3fec5a82012-03-01 05:57:03 -0800106
107
108def _CheckBuildRootBranch(buildroot, tracking_branch):
109 """Make sure buildroot branch is the same as Chromite branch."""
110 manifest_branch = cros_lib.GetManifestDefaultBranch(buildroot)
111 if manifest_branch != tracking_branch:
112 cros_lib.Die('Chromite is not on same branch as buildroot checkout\n' +
113 'Chromite is on branch %s.\n' % tracking_branch +
114 'Buildroot checked out to %s\n' % manifest_branch)
115
116
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700117def AcquirePoolFromOptions(options, target_manifest_branch):
118 """Generate patch objects from passed in options.
Brian Harring3fec5a82012-03-01 05:57:03 -0800119
120 Args:
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700121 options: The options object generated by optparse.
Brian Harring3fec5a82012-03-01 05:57:03 -0800122
Ryan Cuif7f24692012-05-18 16:35:33 -0700123 Returns:
124 trybot_patch_pool.TrybotPatchPool object.
125
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700126 Raises:
127 gerrit_helper.GerritException, cros_patch.PatchException
Brian Harring3fec5a82012-03-01 05:57:03 -0800128 """
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700129 gerrit_patches = []
130 local_patches = []
131 remote_patches = []
Brian Harring3fec5a82012-03-01 05:57:03 -0800132
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700133 if options.gerrit_patches:
134 gerrit_patches = gerrit_helper.GetGerritPatchInfo(
135 options.gerrit_patches)
136 for patch in gerrit_patches:
137 if patch.IsAlreadyMerged():
138 cros_lib.Warning('Patch %s has already been merged.' % str(patch))
Brian Harring3fec5a82012-03-01 05:57:03 -0800139
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700140 if options.local_patches:
141 local_patches = cros_patch.PrepareLocalPatches(
Ryan Cui5ba7e152012-05-10 14:36:52 -0700142 options.sourceroot,
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700143 options.local_patches,
144 target_manifest_branch)
Brian Harring3fec5a82012-03-01 05:57:03 -0800145
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700146 if options.remote_patches:
147 remote_patches = cros_patch.PrepareRemotePatches(
148 options.remote_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800149
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700150 return trybot_patch_pool.TrybotPatchPool(gerrit_patches, local_patches,
151 remote_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800152
153
154def _IsIncrementalBuild(buildroot, clobber):
155 """Returns True if we are reusing an existing buildroot."""
156 repo_dir = os.path.join(buildroot, '.repo')
157 return not clobber and os.path.isdir(repo_dir)
158
159
160class Builder(object):
161 """Parent class for all builder types.
162
163 This class functions as a parent class for various build types. It's intended
164 use is builder_instance.Run().
165
166 Vars:
Brian Harring3fec5a82012-03-01 05:57:03 -0800167 build_config: The configuration dictionary from cbuildbot_config.
168 options: The options provided from optparse in main().
Brian Harring3fec5a82012-03-01 05:57:03 -0800169 archive_url: Where our artifacts for this builder will be archived.
170 tracking_branch: The tracking branch for this build.
171 release_tag: The associated "chrome os version" of this build.
Brian Harring3fec5a82012-03-01 05:57:03 -0800172 """
173
Ryan Cuif7f24692012-05-18 16:35:33 -0700174 def __init__(self, options, build_config, target_manifest_branch):
Brian Harring3fec5a82012-03-01 05:57:03 -0800175 """Initializes instance variables. Must be called by all subclasses."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800176 self.build_config = build_config
177 self.options = options
178
179 # TODO, Remove here and in config after bug chromium-os:14649 is fixed.
180 if self.build_config['chromeos_official']:
181 os.environ['CHROMEOS_OFFICIAL'] = '1'
182
David James58e0c092012-03-04 20:31:12 -0800183 self.archive_stages = {}
Brian Harring3fec5a82012-03-01 05:57:03 -0800184 self.archive_urls = {}
185 self.release_tag = None
Ryan Cuif7f24692012-05-18 16:35:33 -0700186 self.target_manifest_branch = target_manifest_branch
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700187 self.patch_pool = trybot_patch_pool.GetEmptyPool()
Brian Harring3fec5a82012-03-01 05:57:03 -0800188
Ryan Cuif7f24692012-05-18 16:35:33 -0700189 bs.BuilderStage.SetManifestBranch(target_manifest_branch)
190
Brian Harring3fec5a82012-03-01 05:57:03 -0800191 def Initialize(self):
192 """Runs through the initialization steps of an actual build."""
Ryan Cuif7f24692012-05-18 16:35:33 -0700193 if self.options.resume:
194 results_lib.LoadCheckpoint(self.options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800195
196 # Check branch matching early.
197 if _IsIncrementalBuild(self.options.buildroot, self.options.clobber):
Brian Harringeb237932012-05-07 02:08:06 -0700198 _CheckBuildRootBranch(self.options.buildroot, self.target_manifest_branch)
Brian Harring3fec5a82012-03-01 05:57:03 -0800199
200 self._RunStage(stages.CleanUpStage)
201
202 def _GetStageInstance(self, stage, *args, **kwargs):
203 """Helper function to get an instance given the args.
204
David James944a48e2012-03-07 12:19:03 -0800205 Useful as almost all stages just take in options and build_config.
Brian Harring3fec5a82012-03-01 05:57:03 -0800206 """
David James944a48e2012-03-07 12:19:03 -0800207 config = kwargs.pop('config', self.build_config)
208 return stage(self.options, config, *args, **kwargs)
Brian Harring3fec5a82012-03-01 05:57:03 -0800209
210 def _SetReleaseTag(self):
211 """Sets the release tag from the manifest_manager.
212
213 Must be run after sync stage as syncing enables us to have a release tag.
214 """
215 # Extract version we have decided to build into self.release_tag.
216 manifest_manager = stages.ManifestVersionedSyncStage.manifest_manager
217 if manifest_manager:
218 self.release_tag = manifest_manager.current_version
219
220 def _RunStage(self, stage, *args, **kwargs):
221 """Wrapper to run a stage."""
222 stage_instance = self._GetStageInstance(stage, *args, **kwargs)
223 return stage_instance.Run()
224
225 def GetSyncInstance(self):
226 """Returns an instance of a SyncStage that should be run.
227
228 Subclasses must override this method.
229 """
230 raise NotImplementedError()
231
232 def RunStages(self):
233 """Subclasses must override this method. Runs the appropriate code."""
234 raise NotImplementedError()
235
Brian Harring3fec5a82012-03-01 05:57:03 -0800236 def _ShouldReExecuteInBuildRoot(self):
237 """Returns True if this build should be re-executed in the buildroot."""
238 abs_buildroot = os.path.abspath(self.options.buildroot)
239 return not os.path.abspath(__file__).startswith(abs_buildroot)
240
241 def _ReExecuteInBuildroot(self, sync_instance):
242 """Reexecutes self in buildroot and returns True if build succeeds.
243
244 This allows the buildbot code to test itself when changes are patched for
245 buildbot-related code. This is a no-op if the buildroot == buildroot
246 of the running chromite checkout.
247
248 Args:
249 sync_instance: Instance of the sync stage that was run to sync.
250
251 Returns:
252 True if the Build succeeded.
253 """
Brian Harring3fec5a82012-03-01 05:57:03 -0800254 if not self.options.resume:
Ryan Cuif7f24692012-05-18 16:35:33 -0700255 results_lib.WriteCheckpoint(self.options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800256
257 # Re-write paths to use absolute paths.
258 # Suppress any timeout options given from the commandline in the
259 # invoked cbuildbot; our timeout will enforce it instead.
Brian Harringf11bf682012-05-14 15:53:43 -0700260 args_to_append = ['--resume', '--timeout', '0', '--notee', '--nocgroups',
261 '--buildroot', os.path.abspath(self.options.buildroot)]
Brian Harring3fec5a82012-03-01 05:57:03 -0800262
263 if self.options.chrome_root:
264 args_to_append += ['--chrome_root',
265 os.path.abspath(self.options.chrome_root)]
266
267 if stages.ManifestVersionedSyncStage.manifest_manager:
268 ver = stages.ManifestVersionedSyncStage.manifest_manager.current_version
269 args_to_append += ['--version', ver]
270
271 if isinstance(sync_instance, stages.CommitQueueSyncStage):
272 vp_file = sync_instance.SaveValidationPool()
273 args_to_append += ['--validation_pool', vp_file]
274
275 # Re-run the command in the buildroot.
276 # Finally, be generous and give the invoked cbuildbot 30s to shutdown
277 # when something occurs. It should exit quicker, but the sigterm may
278 # hit while the system is particularly busy.
279 return_obj = cros_lib.RunCommand(
Ryan Cuif7f24692012-05-18 16:35:33 -0700280 [constants.PATH_TO_CBUILDBOT] + sys.argv[1:] + args_to_append,
Brian Harring3fec5a82012-03-01 05:57:03 -0800281 cwd=self.options.buildroot, error_code_ok=True, kill_timeout=30)
282 return return_obj.returncode == 0
283
Ryan Cuif7f24692012-05-18 16:35:33 -0700284 def _InitializeTrybotPatchPool(self):
285 """Generate patch pool from patches specified on the command line.
286
287 Do this only if we need to patch changes later on.
288 """
289 changes_stage = stages.PatchChangesStage.StageNamePrefix()
290 check_func = results_lib.Results.PreviouslyCompletedRecord
291 if not check_func(changes_stage) or self.options.bootstrap:
292 self.patch_pool = AcquirePoolFromOptions(self.options,
293 self.target_manifest_branch)
294
295 def _GetBootstrapStage(self):
296 """Constructs and returns the BootStrapStage object.
297
298 We return None when there are no chromite patches to test, and
299 --test-bootstrap wasn't passed in.
300 """
301 stage = None
302 chromite_pool = self.patch_pool.Filter(project=constants.CHROMITE_PROJECT)
303 if chromite_pool or self.options.test_bootstrap:
304 stage = stages.BootstrapStage(self.options, self.build_config,
305 chromite_pool)
306 return stage
307
Brian Harring3fec5a82012-03-01 05:57:03 -0800308 def Run(self):
Ryan Cuif7f24692012-05-18 16:35:33 -0700309 """Main runner for this builder class. Runs build and prints summary.
310
311 Returns:
312 Whether the build succeeded.
313 """
314 self._InitializeTrybotPatchPool()
315
316 if self.options.bootstrap:
317 bootstrap_stage = self._GetBootstrapStage()
318 if bootstrap_stage:
319 # BootstrapStage blocks on re-execution of cbuildbot.
320 bootstrap_stage.Run()
321 return bootstrap_stage.returncode == 0
322
Brian Harring3fec5a82012-03-01 05:57:03 -0800323 print_report = True
David James3d4d3502012-04-09 15:12:06 -0700324 exception_thrown = False
Brian Harring3fec5a82012-03-01 05:57:03 -0800325 success = True
326 try:
327 self.Initialize()
328 sync_instance = self.GetSyncInstance()
329 sync_instance.Run()
330 self._SetReleaseTag()
331
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700332 if self.patch_pool:
333 self._RunStage(stages.PatchChangesStage, self.patch_pool)
Brian Harring3fec5a82012-03-01 05:57:03 -0800334
335 if self._ShouldReExecuteInBuildRoot():
336 print_report = False
337 success = self._ReExecuteInBuildroot(sync_instance)
338 else:
339 self.RunStages()
David James3d4d3502012-04-09 15:12:06 -0700340 except Exception:
341 exception_thrown = True
342 raise
Brian Harring3fec5a82012-03-01 05:57:03 -0800343 finally:
344 if print_report:
Ryan Cuif7f24692012-05-18 16:35:33 -0700345 results_lib.WriteCheckpoint(self.options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800346 print '\n\n\n@@@BUILD_STEP Report@@@\n'
347 results_lib.Results.Report(sys.stdout, self.archive_urls,
348 self.release_tag)
349 success = results_lib.Results.BuildSucceededSoFar()
David James3d4d3502012-04-09 15:12:06 -0700350 if exception_thrown and success:
351 success = False
352 print >> sys.stderr, """
353@@@STEP_FAILURE@@@
354Exception thrown, but all stages marked successful. This is an internal error,
355because the stage that threw the exception should be marked as failing."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800356
357 return success
358
359
360class SimpleBuilder(Builder):
361 """Builder that performs basic vetting operations."""
362
363 def GetSyncInstance(self):
364 """Sync to lkgm or TOT as necessary.
365
366 Returns: the instance of the sync stage that was run.
367 """
368 if self.options.lkgm or self.build_config['use_lkgm']:
369 sync_stage = self._GetStageInstance(stages.LKGMSyncStage)
370 else:
371 sync_stage = self._GetStageInstance(stages.SyncStage)
372
373 return sync_stage
374
David James58e0c092012-03-04 20:31:12 -0800375 def _RunBackgroundStagesForBoard(self, board):
376 """Run background board-specific stages for the specified board."""
David James58e0c092012-03-04 20:31:12 -0800377 archive_stage = self.archive_stages[board]
David James944a48e2012-03-07 12:19:03 -0800378 configs = self.build_config['board_specific_configs']
379 config = configs.get(board, self.build_config)
380 stage_list = [[stages.VMTestStage, board, archive_stage],
381 [stages.ChromeTestStage, board, archive_stage],
382 [stages.UnitTestStage, board],
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700383 [stages.UploadPrebuiltsStage, board, archive_stage]]
Brian Harring3fec5a82012-03-01 05:57:03 -0800384
David James58e0c092012-03-04 20:31:12 -0800385 # We can not run hw tests without archiving the payloads.
386 if self.options.archive:
David James944a48e2012-03-07 12:19:03 -0800387 for suite in config['hw_tests']:
388 stage_list.append([stages.HWTestStage, board, archive_stage, suite])
Chris Sosab50dc932012-03-01 14:00:58 -0800389
David James944a48e2012-03-07 12:19:03 -0800390 steps = [self._GetStageInstance(*x, config=config).Run for x in stage_list]
391 background.RunParallelSteps(steps + [archive_stage.Run])
Brian Harring3fec5a82012-03-01 05:57:03 -0800392
393 def RunStages(self):
394 """Runs through build process."""
395 self._RunStage(stages.BuildBoardStage)
396
397 # TODO(sosa): Split these out into classes.
Brian Harring3fec5a82012-03-01 05:57:03 -0800398 if self.build_config['build_type'] == constants.CHROOT_BUILDER_TYPE:
399 self._RunStage(stages.SDKTestStage)
400 self._RunStage(stages.UploadPrebuiltsStage,
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700401 constants.CHROOT_BUILDER_BOARD, None)
Brian Harring3fec5a82012-03-01 05:57:03 -0800402 elif self.build_config['build_type'] == constants.REFRESH_PACKAGES_TYPE:
403 self._RunStage(stages.RefreshPackageStatusStage)
404 else:
405 self._RunStage(stages.UprevStage)
Brian Harring3fec5a82012-03-01 05:57:03 -0800406
David James944a48e2012-03-07 12:19:03 -0800407 configs = self.build_config['board_specific_configs']
David James58e0c092012-03-04 20:31:12 -0800408 for board in self.build_config['boards']:
David James944a48e2012-03-07 12:19:03 -0800409 config = configs.get(board, self.build_config)
410 archive_stage = self._GetStageInstance(stages.ArchiveStage, board,
411 config=config)
David James58e0c092012-03-04 20:31:12 -0800412 self.archive_stages[board] = archive_stage
413
David James944a48e2012-03-07 12:19:03 -0800414 # Set up a process pool to run test/archive stages in the background.
415 # This process runs task(board) for each board added to the queue.
David James58e0c092012-03-04 20:31:12 -0800416 queue = multiprocessing.Queue()
417 task = self._RunBackgroundStagesForBoard
418 with background.BackgroundTaskRunner(queue, task):
David James944a48e2012-03-07 12:19:03 -0800419 for board in self.build_config['boards']:
David James58e0c092012-03-04 20:31:12 -0800420 # Run BuildTarget in the foreground.
David James944a48e2012-03-07 12:19:03 -0800421 archive_stage = self.archive_stages[board]
422 config = configs.get(board, self.build_config)
423 self._RunStage(stages.BuildTargetStage, board, archive_stage,
Chris Sosa1a87b3e2012-04-12 13:20:42 -0700424 self.release_tag, config=config)
David James58e0c092012-03-04 20:31:12 -0800425 self.archive_urls[board] = archive_stage.GetDownloadUrl()
426
David James944a48e2012-03-07 12:19:03 -0800427 # Kick off task(board) in the background.
David James58e0c092012-03-04 20:31:12 -0800428 queue.put([board])
429
Brian Harring3fec5a82012-03-01 05:57:03 -0800430
431class DistributedBuilder(SimpleBuilder):
432 """Build class that has special logic to handle distributed builds.
433
434 These builds sync using git/manifest logic in manifest_versions. In general
435 they use a non-distributed builder code for the bulk of the work.
436 """
Ryan Cuif7f24692012-05-18 16:35:33 -0700437 def __init__(self, *args, **kwargs):
Brian Harring3fec5a82012-03-01 05:57:03 -0800438 """Initializes a buildbot builder.
439
440 Extra variables:
441 completion_stage_class: Stage used to complete a build. Set in the Sync
442 stage.
443 """
Ryan Cuif7f24692012-05-18 16:35:33 -0700444 super(DistributedBuilder, self).__init__(*args, **kwargs)
Brian Harring3fec5a82012-03-01 05:57:03 -0800445 self.completion_stage_class = None
446
447 def GetSyncInstance(self):
448 """Syncs the tree using one of the distributed sync logic paths.
449
450 Returns: the instance of the sync stage that was run.
451 """
452 # Determine sync class to use. CQ overrides PFQ bits so should check it
453 # first.
454 if cbuildbot_config.IsCQType(self.build_config['build_type']):
455 sync_stage = self._GetStageInstance(stages.CommitQueueSyncStage)
456 self.completion_stage_class = stages.CommitQueueCompletionStage
457 elif cbuildbot_config.IsPFQType(self.build_config['build_type']):
458 sync_stage = self._GetStageInstance(stages.LKGMCandidateSyncStage)
459 self.completion_stage_class = stages.LKGMCandidateSyncCompletionStage
460 else:
461 sync_stage = self._GetStageInstance(stages.ManifestVersionedSyncStage)
462 self.completion_stage_class = stages.ManifestVersionedSyncCompletionStage
463
464 return sync_stage
465
466 def Publish(self, was_build_successful):
467 """Completes build by publishing any required information."""
468 completion_stage = self._GetStageInstance(self.completion_stage_class,
469 was_build_successful)
470 completion_stage.Run()
471 name = completion_stage.name
472 if not results_lib.Results.WasStageSuccessful(name):
473 should_publish_changes = False
474 else:
475 should_publish_changes = (self.build_config['master'] and
476 was_build_successful)
477
478 if should_publish_changes:
479 self._RunStage(stages.PublishUprevChangesStage)
480
481 def RunStages(self):
482 """Runs simple builder logic and publishes information to overlays."""
483 was_build_successful = False
484 try:
David Jamesf55709e2012-03-13 09:10:15 -0700485 super(DistributedBuilder, self).RunStages()
486 was_build_successful = results_lib.Results.BuildSucceededSoFar()
Brian Harring3fec5a82012-03-01 05:57:03 -0800487 except SystemExit as ex:
488 # If a stage calls sys.exit(0), it's exiting with success, so that means
489 # we should mark ourselves as successful.
490 if ex.code == 0:
491 was_build_successful = True
492 raise
493 finally:
494 self.Publish(was_build_successful)
495
Brian Harring3fec5a82012-03-01 05:57:03 -0800496
497def _ConfirmBuildRoot(buildroot):
498 """Confirm with user the inferred buildroot, and mark it as confirmed."""
499 warning = 'Using default directory %s as buildroot' % buildroot
500 response = cros_lib.YesNoPrompt(default=cros_lib.NO, warning=warning,
501 full=True)
502 if response == cros_lib.NO:
503 print('Please specify a buildroot with the --buildroot option.')
504 sys.exit(0)
505
506 if not os.path.exists(buildroot):
507 os.mkdir(buildroot)
508
509 repository.CreateTrybotMarker(buildroot)
510
511
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700512def _ConfirmRemoteBuildbotRun():
513 """Confirm user wants to run with --buildbot --remote."""
514 warning = ('You are about to launch a PRODUCTION job! This is *NOT* a '
515 'trybot run! Are you sure?')
516 response = cros_lib.YesNoPrompt(default=cros_lib.NO, warning=warning,
517 full=True)
518
519 if response == cros_lib.NO:
520 print('Please specify --pass-through="--debug".')
521 sys.exit(0)
522
523
Ryan Cui5ba7e152012-05-10 14:36:52 -0700524def _DetermineDefaultBuildRoot(sourceroot, internal_build):
Brian Harring3fec5a82012-03-01 05:57:03 -0800525 """Default buildroot to be under the directory that contains current checkout.
526
527 Arguments:
528 internal_build: Whether the build is an internal build
Ryan Cui5ba7e152012-05-10 14:36:52 -0700529 sourceroot: Use specified sourceroot.
Brian Harring3fec5a82012-03-01 05:57:03 -0800530 """
Ryan Cui5ba7e152012-05-10 14:36:52 -0700531 if not repository.IsARepoRoot(sourceroot):
532 cros_lib.Die('Could not find root of local checkout at %s. Please specify '
533 'using the --sourceroot option.' % sourceroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800534
535 # Place trybot buildroot under the directory containing current checkout.
Ryan Cui5ba7e152012-05-10 14:36:52 -0700536 top_level = os.path.dirname(os.path.realpath(sourceroot))
Brian Harring3fec5a82012-03-01 05:57:03 -0800537 if internal_build:
538 buildroot = os.path.join(top_level, _DEFAULT_INT_BUILDROOT)
539 else:
540 buildroot = os.path.join(top_level, _DEFAULT_EXT_BUILDROOT)
541
542 return buildroot
543
544
545def _BackupPreviousLog(log_file, backup_limit=25):
546 """Rename previous log.
547
548 Args:
549 log_file: The absolute path to the previous log.
550 """
551 if os.path.exists(log_file):
552 old_logs = sorted(glob.glob(log_file + '.*'),
553 key=distutils.version.LooseVersion)
554
555 if len(old_logs) >= backup_limit:
556 os.remove(old_logs[0])
557
558 last = 0
559 if old_logs:
560 last = int(old_logs.pop().rpartition('.')[2])
561
562 os.rename(log_file, log_file + '.' + str(last + 1))
563
David James944a48e2012-03-07 12:19:03 -0800564def _RunBuildStagesWrapper(options, build_config):
Brian Harring3fec5a82012-03-01 05:57:03 -0800565 """Helper function that wraps RunBuildStages()."""
566 def IsDistributedBuilder():
567 """Determines whether the build_config should be a DistributedBuilder."""
568 if not options.buildbot:
569 return False
570 elif build_config['build_type'] in _DISTRIBUTED_TYPES:
571 chrome_rev = build_config['chrome_rev']
572 if options.chrome_rev: chrome_rev = options.chrome_rev
573 # We don't do distributed logic to TOT Chrome PFQ's, nor local
574 # chrome roots (e.g. chrome try bots)
575 if chrome_rev not in [constants.CHROME_REV_TOT,
576 constants.CHROME_REV_LOCAL,
577 constants.CHROME_REV_SPEC]:
578 return True
579
580 return False
581
Brian Harringd166aaf2012-05-14 18:31:53 -0700582 cros_lib.Info("cbuildbot executed with args %s"
583 % ' '.join(map(repr, sys.argv)))
Brian Harring3fec5a82012-03-01 05:57:03 -0800584
Ryan Cuif7f24692012-05-18 16:35:33 -0700585 target_manifest_branch = _GetChromiteTrackingBranch()
586
587 target = DistributedBuilder if IsDistributedBuilder() else SimpleBuilder
588 buildbot = target(options, build_config, target_manifest_branch)
Brian Harringd166aaf2012-05-14 18:31:53 -0700589 if not buildbot.Run():
590 sys.exit(1)
Brian Harring3fec5a82012-03-01 05:57:03 -0800591
592
593# Parser related functions
Ryan Cui5ba7e152012-05-10 14:36:52 -0700594def _CheckLocalPatches(sourceroot, local_patches):
Brian Harring3fec5a82012-03-01 05:57:03 -0800595 """Do an early quick check of the passed-in patches.
596
597 If the branch of a project is not specified we append the current branch the
598 project is on.
Ryan Cui5ba7e152012-05-10 14:36:52 -0700599
600 Args:
601 sourceroot: The checkout where patches are coming from.
Brian Harring3fec5a82012-03-01 05:57:03 -0800602 """
Ryan Cuicedd8a52012-03-22 02:28:35 -0700603 verified_patches = []
604 for patch in local_patches:
Brian Harring3fec5a82012-03-01 05:57:03 -0800605 components = patch.split(':')
606 if len(components) > 2:
607 msg = 'Specify local patches in project[:branch] format.'
608 raise optparse.OptionValueError(msg)
609
610 # validate project
611 project = components[0]
Ryan Cui5ba7e152012-05-10 14:36:52 -0700612 if not cros_lib.DoesProjectExist(sourceroot, project):
Brian Harring3fec5a82012-03-01 05:57:03 -0800613 raise optparse.OptionValueError('Project %s does not exist.' % project)
614
Ryan Cui5ba7e152012-05-10 14:36:52 -0700615 project_dir = cros_lib.GetProjectDir(sourceroot, project)
Brian Harring3fec5a82012-03-01 05:57:03 -0800616
617 # If no branch was specified, we use the project's current branch.
618 if len(components) == 1:
619 branch = cros_lib.GetCurrentBranch(project_dir)
620 if not branch:
621 raise optparse.OptionValueError('project %s is not on a branch!'
622 % project)
623 # Append branch information to patch
624 patch = '%s:%s' % (project, branch)
625 else:
626 branch = components[1]
627 if not cros_lib.DoesLocalBranchExist(project_dir, branch):
628 raise optparse.OptionValueError('Project %s does not have branch %s'
629 % (project, branch))
630
Ryan Cuicedd8a52012-03-22 02:28:35 -0700631 verified_patches.append(patch)
Brian Harring3fec5a82012-03-01 05:57:03 -0800632
Ryan Cuicedd8a52012-03-22 02:28:35 -0700633 return verified_patches
Brian Harring3fec5a82012-03-01 05:57:03 -0800634
635
Brian Harring3fec5a82012-03-01 05:57:03 -0800636def _CheckChromeVersionOption(_option, _opt_str, value, parser):
637 """Upgrade other options based on chrome_version being passed."""
638 value = value.strip()
639
640 if parser.values.chrome_rev is None and value:
641 parser.values.chrome_rev = constants.CHROME_REV_SPEC
642
643 parser.values.chrome_version = value
644
645
646def _CheckChromeRootOption(_option, _opt_str, value, parser):
647 """Validate and convert chrome_root to full-path form."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800648 if parser.values.chrome_rev is None:
649 parser.values.chrome_rev = constants.CHROME_REV_LOCAL
650
Ryan Cui5ba7e152012-05-10 14:36:52 -0700651 parser.values.chrome_root = value
Brian Harring3fec5a82012-03-01 05:57:03 -0800652
653
654def _CheckChromeRevOption(_option, _opt_str, value, parser):
655 """Validate the chrome_rev option."""
656 value = value.strip()
657 if value not in constants.VALID_CHROME_REVISIONS:
658 raise optparse.OptionValueError('Invalid chrome rev specified')
659
660 parser.values.chrome_rev = value
661
662
Ryan Cui5ba7e152012-05-10 14:36:52 -0700663class CustomParser(optparse.OptionParser):
664 def add_remote_option(self, *args, **kwargs):
665 """For arguments that are passed-through to remote trybot."""
666 return optparse.OptionParser.add_option(self, *args,
667 remote_pass_through=True,
668 **kwargs)
669
670
671class CustomGroup(optparse.OptionGroup):
672 def add_remote_option(self, *args, **kwargs):
673 """For arguments that are passed-through to remote trybot."""
674 return optparse.OptionGroup.add_option(self, *args,
675 remote_pass_through=True,
676 **kwargs)
677
678
Ryan Cuif7f24692012-05-18 16:35:33 -0700679# pylint: disable=W0613
Ryan Cui5ba7e152012-05-10 14:36:52 -0700680def check_path(option, opt, value):
681 """Expand paths and make them absolute."""
682 expanded = osutils.ExpandPath(value)
683 if expanded == '/':
684 raise optparse.OptionValueError('Invalid path %s specified for %s'
685 % (expanded, opt))
686
687 return expanded
688
689
690class CustomOption(optparse.Option):
691 """Subclass Option class to implement pass-through and path evaluation."""
692 TYPES = optparse.Option.TYPES + ('path',)
693 TYPE_CHECKER = optparse.Option.TYPE_CHECKER.copy()
694 TYPE_CHECKER['path'] = check_path
695
696 def __init__(self, *args, **kwargs):
697 # The remote_pass_through argument specifies whether we should directly
698 # pass the argument (with its value) onto the remote trybot.
699 self.pass_through = kwargs.pop('remote_pass_through', False)
700 optparse.Option.__init__(self, *args, **kwargs)
701
702 def take_action(self, action, dest, opt, value, values, parser):
703 optparse.Option.take_action(self, action, dest, opt, value, values,
704 parser)
705 if self.pass_through:
706 parser.values.pass_through_args.append(opt)
707 if self.nargs and self.nargs > 1:
708 # value is a tuple if nargs > 1
709 string_list = [str(val) for val in list(value)]
710 parser.values.pass_through_args.extend(string_list)
711 elif value:
712 parser.values.pass_through_args.append(str(value))
713
714
Brian Harring3fec5a82012-03-01 05:57:03 -0800715def _CreateParser():
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700716 """Generate and return the parser with all the options."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800717 # Parse options
718 usage = "usage: %prog [options] buildbot_config"
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700719 parser = CustomParser(usage=usage, option_class=CustomOption)
Brian Harring3fec5a82012-03-01 05:57:03 -0800720
721 # Main options
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700722 # The remote_pass_through parameter to add_option is implemented by the
723 # CustomOption class. See CustomOption for more information.
Brian Harring3fec5a82012-03-01 05:57:03 -0800724 parser.add_option('-a', '--all', action='store_true', dest='print_all',
725 default=False,
726 help=('List all of the buildbot configs available. Use '
727 'with the --list option'))
Ryan Cui5ba7e152012-05-10 14:36:52 -0700728 parser.add_option('-r', '--buildroot', dest='buildroot', type='path',
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700729 help='Root directory where source is checked out to, and '
730 'where the build occurs. For external build configs, '
731 "defaults to 'trybot' directory at top level of your "
732 'repo-managed checkout.')
733 parser.add_remote_option('--chrome_rev', default=None, type='string',
734 action='callback', dest='chrome_rev',
735 callback=_CheckChromeRevOption,
736 help=('Revision of Chrome to use, of type [%s]'
737 % '|'.join(constants.VALID_CHROME_REVISIONS)))
738 parser.add_remote_option('-g', '--gerrit-patches', action='append',
739 default=[], type='string',
740 metavar="'Id1 *int_Id2...IdN'",
741 help=("Space-separated list of short-form Gerrit "
742 "Change-Id's or change numbers to patch. "
743 "Please prepend '*' to internal Change-Id's"))
Brian Harring3fec5a82012-03-01 05:57:03 -0800744 parser.add_option('-l', '--list', action='store_true', dest='list',
745 default=False,
746 help=('List the suggested trybot configs to use. Use '
747 '--all to list all of the available configs.'))
Ryan Cui54da0702012-04-19 18:38:08 -0700748 parser.add_option('--local', default=False, action='store_true',
749 help=('Specifies that this tryjob should be run locally.'))
Ryan Cuicedd8a52012-03-22 02:28:35 -0700750 parser.add_option('-p', '--local-patches', action='append', default=[],
Brian Harring3fec5a82012-03-01 05:57:03 -0800751 metavar="'<project1>[:<branch1>]...<projectN>[:<branchN>]'",
752 help=('Space-separated list of project branches with '
753 'patches to apply. Projects are specified by name. '
754 'If no branch is specified the current branch of the '
755 'project will be used.'))
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700756 parser.add_remote_option('--profile', default=None, type='string',
757 action='store', dest='profile',
758 help='Name of profile to sub-specify board variant.')
Brian Harring3fec5a82012-03-01 05:57:03 -0800759 parser.add_option('--remote', default=False, action='store_true',
Brian Harring3fec5a82012-03-01 05:57:03 -0800760 help=('Specifies that this tryjob should be run remotely.'))
761
762 # Advanced options
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700763 group = CustomGroup(
Brian Harring3fec5a82012-03-01 05:57:03 -0800764 parser,
765 'Advanced Options',
766 'Caution: use these options at your own risk.')
767
Ryan Cui42aeae32012-05-21 17:09:09 -0700768 # bootstrap-args are not verified by the bootstrap code. It gets passed
769 # direcly to the bootstrap re-execution.
770 group.add_remote_option('--bootstrap-args', action='append',
771 default=[], help=optparse.SUPPRESS_HELP)
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700772 group.add_remote_option('--buildbot', dest='buildbot', action='store_true',
773 default=False, help='This is running on a buildbot')
774 group.add_remote_option('--buildnumber', help='build number', type='int',
775 default=0)
Ryan Cui5ba7e152012-05-10 14:36:52 -0700776 group.add_option('--chrome_root', default=None, type='path',
777 action='callback', callback=_CheckChromeRootOption,
778 dest='chrome_root', help='Local checkout of Chrome to use.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700779 group.add_remote_option('--chrome_version', default=None, type='string',
780 action='callback', dest='chrome_version',
781 callback=_CheckChromeVersionOption,
782 help='Used with SPEC logic to force a particular SVN '
783 'revision of chrome rather than the latest.')
784 group.add_remote_option('--clobber', action='store_true', dest='clobber',
785 default=False,
786 help='Clears an old checkout before syncing')
787 group.add_remote_option('--lkgm', action='store_true', dest='lkgm',
788 default=False,
789 help='Sync to last known good manifest blessed by '
790 'PFQ')
Ryan Cui5ba7e152012-05-10 14:36:52 -0700791 parser.add_option('--log_dir', dest='log_dir', type='path',
Brian Harring3fec5a82012-03-01 05:57:03 -0800792 help=('Directory where logs are stored.'))
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700793 group.add_remote_option('--maxarchives', dest='max_archive_builds',
794 default=3, type='int',
795 help="Change the local saved build count limit.")
796 group.add_remote_option('--noarchive', action='store_false', dest='archive',
797 default=True, help="Don't run archive stage.")
Ryan Cuif7f24692012-05-18 16:35:33 -0700798 group.add_remote_option('--nobootstrap', action='store_false',
799 dest='bootstrap', default=True,
800 help="Don't checkout and run from a standalone "
801 "chromite repo.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700802 group.add_remote_option('--nobuild', action='store_false', dest='build',
803 default=True,
804 help="Don't actually build (for cbuildbot dev)")
805 group.add_remote_option('--noclean', action='store_false', dest='clean',
806 default=True, help="Don't clean the buildroot")
Ryan Cuif7f24692012-05-18 16:35:33 -0700807 group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
808 default=True,
809 help='Disable cbuildbots usage of cgroups.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700810 group.add_remote_option('--noprebuilts', action='store_false',
811 dest='prebuilts', default=True,
812 help="Don't upload prebuilts.")
813 group.add_remote_option('--nosync', action='store_false', dest='sync',
814 default=True, help="Don't sync before building.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700815 group.add_remote_option('--notests', action='store_false', dest='tests',
816 default=True,
817 help='Override values from buildconfig and run no '
818 'tests.')
819 group.add_remote_option('--nouprev', action='store_false', dest='uprev',
820 default=True,
821 help='Override values from buildconfig and never '
822 'uprev.')
823 group.add_option('--pass-through', dest='pass_through_args', action='append',
824 type='string', default=[], help=optparse.SUPPRESS_HELP)
Brian Harring3fec5a82012-03-01 05:57:03 -0800825 group.add_option('--reference-repo', action='store', default=None,
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700826 dest='reference_repo',
827 help='Reuse git data stored in an existing repo '
828 'checkout. This can drastically reduce the network '
829 'time spent setting up the trybot checkout. By '
830 "default, if this option isn't given but cbuildbot "
831 'is invoked from a repo checkout, cbuildbot will '
832 'use the repo root.')
833 # Indicates this is running on a remote trybot machine.
Ryan Cuiba41ad32012-03-08 17:15:29 -0800834 group.add_option('--remote-trybot', dest='remote_trybot', action='store_true',
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700835 default=False, help=optparse.SUPPRESS_HELP)
Ryan Cuicedd8a52012-03-22 02:28:35 -0700836 # Patches uploaded by trybot client when run using the -p option.
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700837 group.add_remote_option('--remote-patches', action='append', default=[],
838 help=optparse.SUPPRESS_HELP)
Ryan Cuicedd8a52012-03-22 02:28:35 -0700839 group.add_option('--resume', action='store_true', default=False,
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700840 help='Skip stages already successfully completed.')
841 group.add_remote_option('--timeout', action='store', type='int', default=0,
842 help='Specify the maximum amount of time this job '
843 'can run for, at which point the build will be '
844 'aborted. If set to zero, then there is no '
845 'timeout.')
Ryan Cui5ba7e152012-05-10 14:36:52 -0700846 group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT,
847 help=optparse.SUPPRESS_HELP)
Ryan Cuif7f24692012-05-18 16:35:33 -0700848 # Causes cbuildbot to bootstrap itself twice, in the sequence A->B->C.
849 # A(unpatched) patches and bootstraps B. B patches and bootstraps C.
850 group.add_remote_option('--test-bootstrap', action='store_true',
851 default=False, help=optparse.SUPPRESS_HELP)
Ryan Cui39bdbbf2012-02-29 16:15:39 -0800852 group.add_option('--test-tryjob', action='store_true',
853 default=False,
854 help='Submit a tryjob to the test repository. Will not '
855 'show up on the production trybot waterfall.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700856 group.add_remote_option('--validation_pool', default=None,
857 help='Path to a pickled validation pool. Intended '
858 'for use only with the commit queue.')
859 group.add_remote_option('--version', dest='force_version', default=None,
860 help='Used with manifest logic. Forces use of this '
861 'version rather than create or get latest.')
Brian Harring3fec5a82012-03-01 05:57:03 -0800862
863 parser.add_option_group(group)
864
865 # Debug options
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700866 group = CustomGroup(parser, "Debug Options")
Brian Harring3fec5a82012-03-01 05:57:03 -0800867
Ryan Cui85867972012-02-23 18:21:49 -0800868 group.add_option('--debug', action='store_true', default=None,
Brian Harring3fec5a82012-03-01 05:57:03 -0800869 help='Override some options to run as a developer.')
870 group.add_option('--dump_config', action='store_true', dest='dump_config',
871 default=False,
872 help='Dump out build config options, and exit.')
873 group.add_option('--notee', action='store_false', dest='tee', default=True,
874 help="Disable logging and internal tee process. Primarily "
875 "used for debugging cbuildbot itself.")
876 parser.add_option_group(group)
877 return parser
878
879
Ryan Cui85867972012-02-23 18:21:49 -0800880def _FinishParsing(options, args):
881 """Perform some parsing tasks that need to take place after optparse.
882
883 This function needs to be easily testable! Keep it free of
884 environment-dependent code. Put more detailed usage validation in
885 _PostParseCheck().
Brian Harring3fec5a82012-03-01 05:57:03 -0800886
887 Args:
Ryan Cui85867972012-02-23 18:21:49 -0800888 options, args: The options/args object returned by optparse
Brian Harring3fec5a82012-03-01 05:57:03 -0800889 """
Brian Harring07039b52012-05-13 17:56:47 -0700890 # Setup logging levels first so any parsing triggered log messages
891 # are appropriately filtered.
892 logging.getLogger().setLevel(
893 logging.DEBUG if options.debug else logging.INFO)
894
Brian Harring3fec5a82012-03-01 05:57:03 -0800895 if options.chrome_root:
896 if options.chrome_rev != constants.CHROME_REV_LOCAL:
897 cros_lib.Die('Chrome rev must be %s if chrome_root is set.' %
898 constants.CHROME_REV_LOCAL)
899 else:
900 if options.chrome_rev == constants.CHROME_REV_LOCAL:
901 cros_lib.Die('Chrome root must be set if chrome_rev is %s.' %
902 constants.CHROME_REV_LOCAL)
903
904 if options.chrome_version:
905 if options.chrome_rev != constants.CHROME_REV_SPEC:
906 cros_lib.Die('Chrome rev must be %s if chrome_version is set.' %
907 constants.CHROME_REV_SPEC)
908 else:
909 if options.chrome_rev == constants.CHROME_REV_SPEC:
910 cros_lib.Die('Chrome rev must not be %s if chrome_version is not set.' %
911 constants.CHROME_REV_SPEC)
912
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700913 patches = bool(options.gerrit_patches or options.local_patches)
914 if options.remote:
915 if options.local:
916 cros_lib.Die('Cannot specify both --remote and --local')
Ryan Cui54da0702012-04-19 18:38:08 -0700917
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700918 if not options.buildbot and not patches:
919 cros_lib.Die('Must provide patches when running with --remote.')
Brian Harring3fec5a82012-03-01 05:57:03 -0800920
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700921 # --debug needs to be explicitly passed through for remote invocations.
922 release_mode_with_patches = (options.buildbot and patches and
923 '--debug' not in options.pass_through_args)
924 else:
925 if len(args) > 1:
926 cros_lib.Die('Multiple configs not supported if not running with '
927 '--remote.')
928
929 release_mode_with_patches = (options.buildbot and patches and
930 not options.debug)
931
932 # When running in release mode, make sure we are running with checked-in code.
933 # We want checked-in cbuildbot/scripts to prevent errors, and we want to build
934 # a release image with checked-in code for CrOS packages.
935 if release_mode_with_patches:
936 cros_lib.Die('Cannot provide patches when running with --buildbot!')
Brian Harring3fec5a82012-03-01 05:57:03 -0800937
Ryan Cuiba41ad32012-03-08 17:15:29 -0800938 if options.buildbot and options.remote_trybot:
939 cros_lib.Die('--buildbot and --remote-trybot cannot be used together.')
940
Ryan Cui85867972012-02-23 18:21:49 -0800941 # Record whether --debug was set explicitly vs. it was inferred.
942 options.debug_forced = False
943 if options.debug:
944 options.debug_forced = True
945 else:
Ryan Cui16ca5812012-03-08 20:34:27 -0800946 # We don't set debug by default for
947 # 1. --buildbot invocations.
948 # 2. --remote invocations, because it needs to push changes to the tryjob
949 # repo.
950 options.debug = not options.buildbot and not options.remote
Brian Harring3fec5a82012-03-01 05:57:03 -0800951
Brian Harring3fec5a82012-03-01 05:57:03 -0800952
Ryan Cuicedd8a52012-03-22 02:28:35 -0700953def _SplitAndFlatten(appended_items):
954 """Given a list of space-separated items, split into flattened list.
955
956 Given ['abc def', 'hij'] return ['abc', 'def', 'hij'].
957 Arguments:
958 appended_items: List of delimiter-separated items.
959
960 Returns: Flattened list.
961 """
962 new_list = []
963 for item in appended_items:
Mike Frysinger4bd23892012-03-26 15:08:52 -0400964 new_list.extend(item.split())
Ryan Cuicedd8a52012-03-22 02:28:35 -0700965 return new_list
966
967
Brian Harring1d7ba942012-04-24 06:37:18 -0700968# pylint: disable=W0613
Ryan Cui85867972012-02-23 18:21:49 -0800969def _PostParseCheck(options, args):
970 """Perform some usage validation after we've parsed the arguments
Brian Harring3fec5a82012-03-01 05:57:03 -0800971
Ryan Cui85867972012-02-23 18:21:49 -0800972 Args:
973 options/args: The options/args object returned by optparse
974 """
Ryan Cui5ba7e152012-05-10 14:36:52 -0700975 if options.local_patches and not repository.IsARepoRoot(options.sourceroot):
976 raise Exception('Could not find repo checkout at %s!'
977 % options.sourceroot)
978
Brian Harring1d7ba942012-04-24 06:37:18 -0700979 options.gerrit_patches = _SplitAndFlatten(options.gerrit_patches)
980 options.remote_patches = _SplitAndFlatten(options.remote_patches)
981 try:
982 # TODO(rcui): Split this into two stages, one that parses, another that
983 # validates. Parsing step will be called by _FinishParsing().
984 options.local_patches = _CheckLocalPatches(
Ryan Cui5ba7e152012-05-10 14:36:52 -0700985 options.sourceroot,
Brian Harring1d7ba942012-04-24 06:37:18 -0700986 _SplitAndFlatten(options.local_patches))
987 except optparse.OptionValueError as e:
988 cros_lib.Die(str(e))
989
990 default = os.environ.get('CBUILDBOT_DEFAULT_MODE')
991 if (default and not any([options.local, options.buildbot,
992 options.remote, options.remote_trybot])):
993 cros_lib.Info("CBUILDBOT_DEFAULT_MODE=%s env var detected, using it."
994 % default)
995 default = default.lower()
996 if default == 'local':
997 options.local = True
998 elif default == 'remote':
999 options.remote = True
1000 elif default == 'buildbot':
1001 options.buildbot = True
1002 else:
1003 cros_lib.Die("CBUILDBOT_DEFAULT_MODE value %s isn't supported. "
1004 % default)
Ryan Cui85867972012-02-23 18:21:49 -08001005
1006
1007def _ParseCommandLine(parser, argv):
1008 """Completely parse the commandline arguments"""
Brian Harring3fec5a82012-03-01 05:57:03 -08001009 (options, args) = parser.parse_args(argv)
Ryan Cui54da0702012-04-19 18:38:08 -07001010 if options.list:
1011 _PrintValidConfigs(options.print_all)
1012 sys.exit(0)
1013
Ryan Cui8be16062012-04-24 12:05:26 -07001014 # Strip out null arguments.
1015 # TODO(rcui): Remove when buildbot is fixed
1016 args = [arg for arg in args if arg]
1017 if not args:
1018 parser.error('Invalid usage. Use -h to see usage. Use -l to list '
1019 'supported configs.')
1020
Ryan Cui85867972012-02-23 18:21:49 -08001021 _FinishParsing(options, args)
1022 return options, args
1023
1024
1025def main(argv):
1026 # Set umask to 022 so files created by buildbot are readable.
1027 os.umask(022)
1028
1029 if cros_lib.IsInsideChroot():
1030 cros_lib.Die('Please run cbuildbot from outside the chroot.')
1031
1032 parser = _CreateParser()
1033 (options, args) = _ParseCommandLine(parser, argv)
Brian Harring3fec5a82012-03-01 05:57:03 -08001034
Brian Harring3fec5a82012-03-01 05:57:03 -08001035 _PostParseCheck(options, args)
1036
1037 if options.remote:
Chris Sosa4f6ffaf2012-05-01 17:05:44 -07001038 cros_lib.logger.setLevel(logging.WARNING)
Ryan Cui16ca5812012-03-08 20:34:27 -08001039
Brian Harring3fec5a82012-03-01 05:57:03 -08001040 # Verify configs are valid.
1041 for bot in args:
1042 _GetConfig(bot)
1043
1044 # Verify gerrit patches are valid.
Ryan Cui16ca5812012-03-08 20:34:27 -08001045 print 'Verifying patches...'
Ryan Cui16d9e1f2012-05-11 10:50:18 -07001046 patch_pool = AcquirePoolFromOptions(options, _GetChromiteTrackingBranch())
1047
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001048 # --debug need to be explicitly passed through for remote invocations.
1049 if options.buildbot and '--debug' not in options.pass_through_args:
1050 _ConfirmRemoteBuildbotRun()
1051
Ryan Cui16ca5812012-03-08 20:34:27 -08001052 print 'Submitting tryjob...'
Ryan Cui16d9e1f2012-05-11 10:50:18 -07001053 tryjob = remote_try.RemoteTryJob(options, args, patch_pool.local_patches)
Ryan Cui39bdbbf2012-02-29 16:15:39 -08001054 tryjob.Submit(testjob=options.test_tryjob, dryrun=options.debug)
Ryan Cui16ca5812012-03-08 20:34:27 -08001055 print 'Tryjob submitted!'
1056 print ('Go to %s to view the status of your job.'
Ryan Cui4906e1c2012-04-03 20:09:34 -07001057 % tryjob.GetTrybotWaterfallLink())
Brian Harring3fec5a82012-03-01 05:57:03 -08001058 sys.exit(0)
Ryan Cui54da0702012-04-19 18:38:08 -07001059 elif (not options.buildbot and not options.remote_trybot
1060 and not options.resume and not options.local):
1061 cros_lib.Warning('Running in LOCAL TRYBOT mode! Use --remote to submit '
1062 'REMOTE tryjobs. Use --local to suppress this message.')
1063 cros_lib.Warning('Starting April 30th, --local will be required to run the '
1064 'local trybot.')
1065 time.sleep(5)
Brian Harring3fec5a82012-03-01 05:57:03 -08001066
Ryan Cui8be16062012-04-24 12:05:26 -07001067 # Only expecting one config
1068 bot_id = args[-1]
1069 build_config = _GetConfig(bot_id)
Brian Harring3fec5a82012-03-01 05:57:03 -08001070
1071 if options.reference_repo is None:
Ryan Cui5ba7e152012-05-10 14:36:52 -07001072 repo_path = os.path.join(options.sourceroot, '.repo')
Brian Harring3fec5a82012-03-01 05:57:03 -08001073 # If we're being run from a repo checkout, reuse the repo's git pool to
1074 # cut down on sync time.
1075 if os.path.exists(repo_path):
Ryan Cui5ba7e152012-05-10 14:36:52 -07001076 options.reference_repo = options.sourceroot
Brian Harring3fec5a82012-03-01 05:57:03 -08001077 elif options.reference_repo:
1078 if not os.path.exists(options.reference_repo):
1079 parser.error('Reference path %s does not exist'
1080 % (options.reference_repo,))
1081 elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
1082 parser.error('Reference path %s does not look to be the base of a '
1083 'repo checkout; no .repo exists in the root.'
1084 % (options.reference_repo,))
Ryan Cuid4a24212012-04-04 18:08:12 -07001085
Brian Harringf11bf682012-05-14 15:53:43 -07001086 if (options.buildbot or options.remote_trybot) and not options.resume:
Brian Harring470f6112012-03-02 11:47:10 -08001087 if not options.cgroups:
Ryan Cuid4a24212012-04-04 18:08:12 -07001088 parser.error('Options --buildbot/--remote-trybot and --nocgroups cannot '
1089 'be used together. Cgroup support is required for '
1090 'buildbot/remote-trybot mode.')
Brian Harring470f6112012-03-02 11:47:10 -08001091 if not cgroups.Cgroup.CgroupsSupported():
Ryan Cuid4a24212012-04-04 18:08:12 -07001092 parser.error('Option --buildbot/--remote-trybot was given, but this '
1093 'system does not support cgroups. Failing.')
Brian Harring3fec5a82012-03-01 05:57:03 -08001094
Brian Harring351ce442012-03-09 16:38:14 -08001095 missing = []
1096 for program in _BUILDBOT_REQUIRED_BINARIES:
1097 ret = cros_lib.RunCommand('which %s' % program, shell=True,
1098 redirect_stderr=True, redirect_stdout=True,
1099 error_code_ok=True, print_cmd=False)
1100 if ret.returncode != 0:
1101 missing.append(program)
1102
1103 if missing:
Ryan Cuid4a24212012-04-04 18:08:12 -07001104 parser.error("Option --buildbot/--remote-trybot requires the following "
1105 "binaries which couldn't be found in $PATH: %s"
Brian Harring351ce442012-03-09 16:38:14 -08001106 % (', '.join(missing)))
1107
Brian Harring3fec5a82012-03-01 05:57:03 -08001108 if options.reference_repo:
1109 options.reference_repo = os.path.abspath(options.reference_repo)
1110
1111 if options.dump_config:
1112 # This works, but option ordering is bad...
1113 print 'Configuration %s:' % bot_id
1114 pretty_printer = pprint.PrettyPrinter(indent=2)
1115 pretty_printer.pprint(build_config)
1116 sys.exit(0)
1117
1118 if not options.buildroot:
1119 if options.buildbot:
1120 parser.error('Please specify a buildroot with the --buildroot option.')
Matt Tennantd55b1f42012-04-13 14:15:01 -07001121
Ryan Cui5ba7e152012-05-10 14:36:52 -07001122 options.buildroot = _DetermineDefaultBuildRoot(options.sourceroot,
1123 build_config['internal'])
Brian Harring470f6112012-03-02 11:47:10 -08001124 # We use a marker file in the buildroot to indicate the user has
1125 # consented to using this directory.
1126 if not os.path.exists(repository.GetTrybotMarkerPath(options.buildroot)):
1127 _ConfirmBuildRoot(options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -08001128
1129 # Sanity check of buildroot- specifically that it's not pointing into the
1130 # midst of an existing repo since git-repo doesn't support nesting.
Brian Harring3fec5a82012-03-01 05:57:03 -08001131 if (not repository.IsARepoRoot(options.buildroot) and
David James6b80dc62012-02-29 15:34:40 -08001132 repository.InARepoRepository(options.buildroot)):
Brian Harring3fec5a82012-03-01 05:57:03 -08001133 parser.error('Configured buildroot %s points into a repository checkout, '
1134 'rather than the root of it. This is not supported.'
1135 % options.buildroot)
1136
Brian Harringd166aaf2012-05-14 18:31:53 -07001137 log_file = None
1138 if options.tee:
1139 default_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
1140 dirname = options.log_dir or default_dir
1141 log_file = os.path.join(dirname, _BUILDBOT_LOG_FILE)
1142
1143 osutils.SafeMakedirs(dirname)
1144 _BackupPreviousLog(log_file)
1145
Brian Harringc2d09d92012-05-13 22:03:15 -07001146 with cros_lib.ContextManagerStack() as stack:
1147 critical_section = stack.Add(cleanup.EnforcedCleanupSection)
1148 stack.Add(sudo.SudoKeepAlive)
Brian Harringd166aaf2012-05-14 18:31:53 -07001149
Brian Harringc2d09d92012-05-13 22:03:15 -07001150 if not options.resume:
Brian Harring2bf55e12012-05-13 21:31:55 -07001151 # If we're in resume mode, use our parents tempdir rather than
1152 # nesting another layer.
Brian Harringc2d09d92012-05-13 22:03:15 -07001153 stack.Add(osutils.TempDirContextManager, 'cbuildbot-tmp')
1154 logging.debug("Cbuildbot tempdir is %r.", os.environ.get('TMP'))
Brian Harringd166aaf2012-05-14 18:31:53 -07001155
1156 if log_file is not None:
1157 stack.Add(tee.Tee, log_file)
1158 options.preserve_paths = set([_DEFAULT_LOG_DIR])
1159
Brian Harringc2d09d92012-05-13 22:03:15 -07001160 if options.cgroups:
1161 stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
Brian Harringa184efa2012-03-04 11:51:25 -08001162
Brian Harringc2d09d92012-05-13 22:03:15 -07001163 # Mark everything between EnforcedCleanupSection and here as having to
1164 # be rolled back via the contextmanager cleanup handlers. This
1165 # ensures that sudo bits cannot outlive cbuildbot, that anything
1166 # cgroups would kill gets killed, etc.
1167 critical_section.ForkWatchdog()
Brian Harringd166aaf2012-05-14 18:31:53 -07001168
Brian Harringc2d09d92012-05-13 22:03:15 -07001169 if options.timeout > 0:
1170 stack.Add(cros_lib.Timeout, options.timeout)
Brian Harringa184efa2012-03-04 11:51:25 -08001171
Brian Harringc2d09d92012-05-13 22:03:15 -07001172 if not options.buildbot:
1173 build_config = cbuildbot_config.OverrideConfigForTrybot(
1174 build_config,
1175 options.remote_trybot)
1176
1177 _RunBuildStagesWrapper(options, build_config)