blob: 8c598a2caf35a95f13515c780a7b13a847c19a3b [file] [log] [blame]
Brian Harring3fec5a82012-03-01 05:57:03 -08001#!/usr/bin/python
2
Mike Frysingerd6925b52012-07-16 16:11:00 -04003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Brian Harring3fec5a82012-03-01 05:57:03 -08004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Main builder code for Chromium OS.
8
9Used by Chromium OS buildbot configuration for all Chromium OS builds including
10full and pre-flight-queue builds.
11"""
12
David James1fddb8f2013-04-25 15:23:07 -070013import collections
Brian Harring3fec5a82012-03-01 05:57:03 -080014import distutils.version
Mike Frysinger6903c762012-12-04 01:57:16 -050015import errno
Brian Harring3fec5a82012-03-01 05:57:03 -080016import glob
Chris Sosa4f6ffaf2012-05-01 17:05:44 -070017import logging
Brian Harring3fec5a82012-03-01 05:57:03 -080018import optparse
19import os
Brian Harring3fec5a82012-03-01 05:57:03 -080020import sys
Ryan Cui54da0702012-04-19 18:38:08 -070021import time
David James3541a132013-03-18 13:21:58 -070022import traceback
Brian Harring3fec5a82012-03-01 05:57:03 -080023
Brian Harring3fec5a82012-03-01 05:57:03 -080024from chromite.buildbot import cbuildbot_config
25from chromite.buildbot import cbuildbot_stages as stages
26from chromite.buildbot import cbuildbot_results as results_lib
Matt Tennantefb59522013-10-18 13:49:28 -070027from chromite.buildbot import cbuildbot_run
Brian Harring3fec5a82012-03-01 05:57:03 -080028from chromite.buildbot import constants
Brian Harring3fec5a82012-03-01 05:57:03 -080029from chromite.buildbot import remote_try
30from chromite.buildbot import repository
31from chromite.buildbot import tee
Ryan Cui16d9e1f2012-05-11 10:50:18 -070032from chromite.buildbot import trybot_patch_pool
Brian Harring3fec5a82012-03-01 05:57:03 -080033
Brian Harringc92a7012012-02-29 10:11:34 -080034from chromite.lib import cgroups
Brian Harringa184efa2012-03-04 11:51:25 -080035from chromite.lib import cleanup
Brian Harringb6cf9142012-09-01 20:43:17 -070036from chromite.lib import commandline
Brian Harring1b8c4c82012-05-29 23:03:04 -070037from chromite.lib import cros_build_lib
David Jamesa0a664e2013-02-13 09:52:01 -080038from chromite.lib import gclient
Brian Harring511055e2012-10-10 02:58:59 -070039from chromite.lib import gerrit
David James97d95872012-11-16 15:09:56 -080040from chromite.lib import git
Brian Harringaf019fb2012-05-10 15:06:13 -070041from chromite.lib import osutils
Brian Harring511055e2012-10-10 02:58:59 -070042from chromite.lib import patch as cros_patch
David James6450a0a2012-12-04 07:59:53 -080043from chromite.lib import parallel
Brian Harring3fec5a82012-03-01 05:57:03 -080044from chromite.lib import sudo
David James3432acd2013-11-27 10:02:18 -080045from chromite.lib import timeout_util
Brian Harring3fec5a82012-03-01 05:57:03 -080046
Aviv Kesheta0159be2013-12-12 13:56:28 -080047import mock
48
Ryan Cuiadd49122012-03-21 22:19:58 -070049
Brian Harring3fec5a82012-03-01 05:57:03 -080050_DEFAULT_LOG_DIR = 'cbuildbot_logs'
51_BUILDBOT_LOG_FILE = 'cbuildbot.log'
52_DEFAULT_EXT_BUILDROOT = 'trybot'
53_DEFAULT_INT_BUILDROOT = 'trybot-internal'
Brian Harring351ce442012-03-09 16:38:14 -080054_BUILDBOT_REQUIRED_BINARIES = ('pbzip2',)
Ryan Cui1c13a252012-10-16 15:00:16 -070055_API_VERSION_ATTR = 'api_version'
Brian Harring3fec5a82012-03-01 05:57:03 -080056
57
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070058def _PrintValidConfigs(display_all=False):
Brian Harring3fec5a82012-03-01 05:57:03 -080059 """Print a list of valid buildbot configs.
60
Mike Frysinger02e1e072013-11-10 22:11:34 -050061 Args:
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070062 display_all: Print all configs. Otherwise, prints only configs with
63 trybot_list=True.
Brian Harring3fec5a82012-03-01 05:57:03 -080064 """
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070065 def _GetSortKey(config_name):
66 config_dict = cbuildbot_config.config[config_name]
67 return (not config_dict['trybot_list'], config_dict['description'],
68 config_name)
69
Brian Harring3fec5a82012-03-01 05:57:03 -080070 COLUMN_WIDTH = 45
Matt Tennant763497d2014-01-17 16:45:54 -080071 print
Brian Harring3fec5a82012-03-01 05:57:03 -080072 print 'config'.ljust(COLUMN_WIDTH), 'description'
73 print '------'.ljust(COLUMN_WIDTH), '-----------'
74 config_names = cbuildbot_config.config.keys()
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070075 config_names.sort(key=_GetSortKey)
Brian Harring3fec5a82012-03-01 05:57:03 -080076 for name in config_names:
Ryan Cui4f6cf7e2012-04-18 16:12:27 -070077 if display_all or cbuildbot_config.config[name]['trybot_list']:
78 desc = cbuildbot_config.config[name].get('description')
79 desc = desc if desc else ''
Brian Harring3fec5a82012-03-01 05:57:03 -080080 print name.ljust(COLUMN_WIDTH), desc
81
Matt Tennant763497d2014-01-17 16:45:54 -080082 print
83
Brian Harring3fec5a82012-03-01 05:57:03 -080084
85def _GetConfig(config_name):
Matt Tennant763497d2014-01-17 16:45:54 -080086 """Gets the configuration for the build if it exists, None otherwise."""
87 if cbuildbot_config.config.has_key(config_name):
88 return cbuildbot_config.config[config_name]
Brian Harring3fec5a82012-03-01 05:57:03 -080089
90
Ryan Cuie1e4e662012-05-21 16:39:46 -070091def AcquirePoolFromOptions(options):
Ryan Cui16d9e1f2012-05-11 10:50:18 -070092 """Generate patch objects from passed in options.
Brian Harring3fec5a82012-03-01 05:57:03 -080093
94 Args:
Ryan Cui16d9e1f2012-05-11 10:50:18 -070095 options: The options object generated by optparse.
Brian Harring3fec5a82012-03-01 05:57:03 -080096
Ryan Cuif7f24692012-05-18 16:35:33 -070097 Returns:
98 trybot_patch_pool.TrybotPatchPool object.
99
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700100 Raises:
Brian Harring511055e2012-10-10 02:58:59 -0700101 gerrit.GerritException, cros_patch.PatchException
Brian Harring3fec5a82012-03-01 05:57:03 -0800102 """
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700103 gerrit_patches = []
104 local_patches = []
105 remote_patches = []
Brian Harring3fec5a82012-03-01 05:57:03 -0800106
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700107 if options.gerrit_patches:
Brian Harring511055e2012-10-10 02:58:59 -0700108 gerrit_patches = gerrit.GetGerritPatchInfo(
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700109 options.gerrit_patches)
110 for patch in gerrit_patches:
111 if patch.IsAlreadyMerged():
Brian Harring1b8c4c82012-05-29 23:03:04 -0700112 cros_build_lib.Warning('Patch %s has already been merged.' % str(patch))
Brian Harring3fec5a82012-03-01 05:57:03 -0800113
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700114 if options.local_patches:
David James97d95872012-11-16 15:09:56 -0800115 manifest = git.ManifestCheckout.Cached(options.sourceroot)
Brian Harring609dc4e2012-05-07 02:17:44 -0700116 local_patches = cros_patch.PrepareLocalPatches(manifest,
117 options.local_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800118
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700119 if options.remote_patches:
120 remote_patches = cros_patch.PrepareRemotePatches(
121 options.remote_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800122
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700123 return trybot_patch_pool.TrybotPatchPool(gerrit_patches, local_patches,
124 remote_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800125
126
Brian Harring3fec5a82012-03-01 05:57:03 -0800127class Builder(object):
128 """Parent class for all builder types.
129
Matt Tennant759e2352013-09-27 15:14:44 -0700130 This class functions as an abstract parent class for various build types.
131 Its intended use is builder_instance.Run().
Brian Harring3fec5a82012-03-01 05:57:03 -0800132
Matt Tennant3f351552014-01-03 14:28:00 -0800133 Attributes:
134 _run: The BuilderRun object for this run.
135 archive_stages: Dict of BuildConfig keys to ArchiveStage values.
136 patch_pool: TrybotPatchPool.
Brian Harring3fec5a82012-03-01 05:57:03 -0800137 """
138
Matt Tennantefb59522013-10-18 13:49:28 -0700139 def __init__(self, builder_run):
Brian Harring3fec5a82012-03-01 05:57:03 -0800140 """Initializes instance variables. Must be called by all subclasses."""
Matt Tennantefb59522013-10-18 13:49:28 -0700141 self._run = builder_run
Brian Harring3fec5a82012-03-01 05:57:03 -0800142
Matt Tennantefb59522013-10-18 13:49:28 -0700143 if self._run.config.chromeos_official:
Brian Harring3fec5a82012-03-01 05:57:03 -0800144 os.environ['CHROMEOS_OFFICIAL'] = '1'
145
David James58e0c092012-03-04 20:31:12 -0800146 self.archive_stages = {}
Brian Harring76d1bf62012-06-01 13:52:48 -0700147 self.patch_pool = trybot_patch_pool.TrybotPatchPool()
Brian Harring3fec5a82012-03-01 05:57:03 -0800148
149 def Initialize(self):
150 """Runs through the initialization steps of an actual build."""
Matt Tennantefb59522013-10-18 13:49:28 -0700151 if self._run.options.resume:
152 results_lib.LoadCheckpoint(self._run.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800153
Brian Harring3fec5a82012-03-01 05:57:03 -0800154 self._RunStage(stages.CleanUpStage)
155
156 def _GetStageInstance(self, stage, *args, **kwargs):
Matt Tennant759e2352013-09-27 15:14:44 -0700157 """Helper function to get a stage instance given the args.
Brian Harring3fec5a82012-03-01 05:57:03 -0800158
Matt Tennantefb59522013-10-18 13:49:28 -0700159 Useful as almost all stages just take in builder_run.
Brian Harring3fec5a82012-03-01 05:57:03 -0800160 """
Matt Tennantefb59522013-10-18 13:49:28 -0700161 # Normally the default BuilderRun (self._run) is used, but it can
162 # be overridden with "builder_run" kwargs (e.g. for child configs).
163 builder_run = kwargs.pop('builder_run', self._run)
164 return stage(builder_run, *args, **kwargs)
Brian Harring3fec5a82012-03-01 05:57:03 -0800165
166 def _SetReleaseTag(self):
Matt Tennantefb59522013-10-18 13:49:28 -0700167 """Sets the release tag from the manifest manager.
Brian Harring3fec5a82012-03-01 05:57:03 -0800168
Matt Tennantf4980ee2014-01-29 09:21:27 -0800169 Must be run after sync stage as syncing enables us to have a release tag,
170 and must be run before any usage of attrs.release_tag.
Matt Tennantefb59522013-10-18 13:49:28 -0700171
172 TODO(mtennant): Find a bottleneck place in syncing that can set this
Matt Tennantf4980ee2014-01-29 09:21:27 -0800173 directly. Be careful, as there are several kinds of syncing stages, and
174 sync stages have been known to abort with sys.exit calls.
Brian Harring3fec5a82012-03-01 05:57:03 -0800175 """
Matt Tennantefb59522013-10-18 13:49:28 -0700176 manifest_manager = getattr(self._run.attrs, 'manifest_manager', None)
Brian Harring3fec5a82012-03-01 05:57:03 -0800177 if manifest_manager:
Matt Tennantefb59522013-10-18 13:49:28 -0700178 self._run.attrs.release_tag = manifest_manager.current_version
Matt Tennant3f351552014-01-03 14:28:00 -0800179 else:
180 self._run.attrs.release_tag = None
Brian Harring3fec5a82012-03-01 05:57:03 -0800181
182 def _RunStage(self, stage, *args, **kwargs):
Matt Tennant759e2352013-09-27 15:14:44 -0700183 """Wrapper to run a stage.
184
185 Args:
186 stage: A BuilderStage class.
187 args: args to pass to stage constructor.
188 kwargs: kwargs to pass to stage constructor.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500189
Matt Tennant759e2352013-09-27 15:14:44 -0700190 Returns:
Matt Tennant759e2352013-09-27 15:14:44 -0700191 Whatever the stage's Run method returns.
192 """
Brian Harring3fec5a82012-03-01 05:57:03 -0800193 stage_instance = self._GetStageInstance(stage, *args, **kwargs)
194 return stage_instance.Run()
195
Matt Tennantf4980ee2014-01-29 09:21:27 -0800196 def _RunSyncStage(self, sync_instance):
197 """Run given |sync_instance| stage and be sure attrs.release_tag set."""
198 try:
199 sync_instance.Run()
200 finally:
201 self._SetReleaseTag()
202
203
Brian Harring3fec5a82012-03-01 05:57:03 -0800204 def GetSyncInstance(self):
205 """Returns an instance of a SyncStage that should be run.
206
207 Subclasses must override this method.
208 """
209 raise NotImplementedError()
210
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400211 def GetCompletionInstance(self):
Aviv Keshetf2746152013-11-26 10:42:46 -0800212 """Returns the MasterSlaveSyncCompletionStage for this build.
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400213
214 Subclasses may override this method.
215
Mike Frysinger02e1e072013-11-10 22:11:34 -0500216 Returns:
217 None
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400218 """
219 return None
220
Brian Harring3fec5a82012-03-01 05:57:03 -0800221 def RunStages(self):
222 """Subclasses must override this method. Runs the appropriate code."""
223 raise NotImplementedError()
224
Brian Harring3fec5a82012-03-01 05:57:03 -0800225 def _ReExecuteInBuildroot(self, sync_instance):
226 """Reexecutes self in buildroot and returns True if build succeeds.
227
228 This allows the buildbot code to test itself when changes are patched for
229 buildbot-related code. This is a no-op if the buildroot == buildroot
230 of the running chromite checkout.
231
232 Args:
Matt Tennant85348852014-01-10 15:16:02 -0800233 sync_instance: Instance of the sync stage that was run to sync.
Brian Harring3fec5a82012-03-01 05:57:03 -0800234
235 Returns:
236 True if the Build succeeded.
237 """
Matt Tennantefb59522013-10-18 13:49:28 -0700238 if not self._run.options.resume:
239 results_lib.WriteCheckpoint(self._run.options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800240
Ryan Cui1c13a252012-10-16 15:00:16 -0700241 args = stages.BootstrapStage.FilterArgsForTargetCbuildbot(
Matt Tennantefb59522013-10-18 13:49:28 -0700242 self._run.options.buildroot, constants.PATH_TO_CBUILDBOT,
243 self._run.options)
Brian Harring37e559b2012-05-22 20:47:32 -0700244
David James2333c182013-02-13 16:16:15 -0800245 # Specify a buildroot explicitly (just in case, for local trybot).
Brian Harring3fec5a82012-03-01 05:57:03 -0800246 # Suppress any timeout options given from the commandline in the
247 # invoked cbuildbot; our timeout will enforce it instead.
Ryan Cui1c13a252012-10-16 15:00:16 -0700248 args += ['--resume', '--timeout', '0', '--notee', '--nocgroups',
Matt Tennantefb59522013-10-18 13:49:28 -0700249 '--buildroot', os.path.abspath(self._run.options.buildroot)]
Brian Harringae0a5322012-09-15 01:46:51 -0700250
Matt Tennantefb59522013-10-18 13:49:28 -0700251 if hasattr(self._run.attrs, 'manifest_manager'):
252 # TODO(mtennant): Is this the same as self._run.attrs.release_tag?
253 ver = self._run.attrs.manifest_manager.current_version
Ryan Cui1c13a252012-10-16 15:00:16 -0700254 args += ['--version', ver]
Brian Harring3fec5a82012-03-01 05:57:03 -0800255
David Jamesf421c6d2013-04-11 15:37:57 -0700256 pool = getattr(sync_instance, 'pool', None)
257 if pool:
Matt Tennantefb59522013-10-18 13:49:28 -0700258 filename = os.path.join(self._run.options.buildroot,
259 'validation_pool.dump')
David Jamesf421c6d2013-04-11 15:37:57 -0700260 pool.Save(filename)
261 args += ['--validation_pool', filename]
Brian Harring3fec5a82012-03-01 05:57:03 -0800262
David Jamesac8c2a72013-02-13 18:44:33 -0800263 # Reset the cache dir so that the child will calculate it automatically.
Matt Tennantefb59522013-10-18 13:49:28 -0700264 if not self._run.options.cache_dir_specified:
David Jamesac8c2a72013-02-13 18:44:33 -0800265 commandline.BaseParser.ConfigureCacheDir(None)
266
Brian Harring3fec5a82012-03-01 05:57:03 -0800267 # Re-run the command in the buildroot.
268 # Finally, be generous and give the invoked cbuildbot 30s to shutdown
269 # when something occurs. It should exit quicker, but the sigterm may
270 # hit while the system is particularly busy.
Brian Harring1b8c4c82012-05-29 23:03:04 -0700271 return_obj = cros_build_lib.RunCommand(
Matt Tennantefb59522013-10-18 13:49:28 -0700272 args, cwd=self._run.options.buildroot, error_code_ok=True,
273 kill_timeout=30)
Brian Harring3fec5a82012-03-01 05:57:03 -0800274 return return_obj.returncode == 0
275
Ryan Cuif7f24692012-05-18 16:35:33 -0700276 def _InitializeTrybotPatchPool(self):
277 """Generate patch pool from patches specified on the command line.
278
279 Do this only if we need to patch changes later on.
280 """
281 changes_stage = stages.PatchChangesStage.StageNamePrefix()
282 check_func = results_lib.Results.PreviouslyCompletedRecord
Matt Tennantefb59522013-10-18 13:49:28 -0700283 if not check_func(changes_stage) or self._run.options.bootstrap:
284 self.patch_pool = AcquirePoolFromOptions(self._run.options)
Ryan Cuif7f24692012-05-18 16:35:33 -0700285
286 def _GetBootstrapStage(self):
287 """Constructs and returns the BootStrapStage object.
288
289 We return None when there are no chromite patches to test, and
290 --test-bootstrap wasn't passed in.
291 """
292 stage = None
293 chromite_pool = self.patch_pool.Filter(project=constants.CHROMITE_PROJECT)
Ryan Cui5616a512012-08-17 13:39:36 -0700294 manifest_pool = self.patch_pool.FilterManifest()
David James97d95872012-11-16 15:09:56 -0800295 chromite_branch = git.GetChromiteTrackingBranch()
Matt Tennantefb59522013-10-18 13:49:28 -0700296 if (chromite_pool or manifest_pool or
297 self._run.options.test_bootstrap or
298 chromite_branch != self._run.options.branch):
299 stage = stages.BootstrapStage(self._run, chromite_pool,
300 manifest_pool)
Ryan Cuif7f24692012-05-18 16:35:33 -0700301 return stage
302
Brian Harring3fec5a82012-03-01 05:57:03 -0800303 def Run(self):
Ryan Cuif7f24692012-05-18 16:35:33 -0700304 """Main runner for this builder class. Runs build and prints summary.
305
306 Returns:
307 Whether the build succeeded.
308 """
309 self._InitializeTrybotPatchPool()
310
Matt Tennantefb59522013-10-18 13:49:28 -0700311 if self._run.options.bootstrap:
Ryan Cuif7f24692012-05-18 16:35:33 -0700312 bootstrap_stage = self._GetBootstrapStage()
313 if bootstrap_stage:
314 # BootstrapStage blocks on re-execution of cbuildbot.
315 bootstrap_stage.Run()
316 return bootstrap_stage.returncode == 0
317
Brian Harring3fec5a82012-03-01 05:57:03 -0800318 print_report = True
David James3d4d3502012-04-09 15:12:06 -0700319 exception_thrown = False
Brian Harring3fec5a82012-03-01 05:57:03 -0800320 success = True
David James9ebb2a42013-08-13 20:29:57 -0700321 sync_instance = None
Brian Harring3fec5a82012-03-01 05:57:03 -0800322 try:
323 self.Initialize()
324 sync_instance = self.GetSyncInstance()
Matt Tennantf4980ee2014-01-29 09:21:27 -0800325 self._RunSyncStage(sync_instance)
Brian Harring3fec5a82012-03-01 05:57:03 -0800326
Matt Tennantefb59522013-10-18 13:49:28 -0700327 if self._run.ShouldPatchAfterSync():
Ryan Cui88b901c2013-06-21 11:35:30 -0700328 # Filter out patches to manifest, since PatchChangesStage can't handle
329 # them. Manifest patches are patched in the BootstrapStage.
330 non_manifest_patches = self.patch_pool.FilterManifest(negate=True)
331 if non_manifest_patches:
332 self._RunStage(stages.PatchChangesStage, non_manifest_patches)
Brian Harring3fec5a82012-03-01 05:57:03 -0800333
Matt Tennantefb59522013-10-18 13:49:28 -0700334 if self._run.ShouldReexecAfterSync():
Brian Harring3fec5a82012-03-01 05:57:03 -0800335 print_report = False
336 success = self._ReExecuteInBuildroot(sync_instance)
337 else:
338 self.RunStages()
Matt Tennant759e2352013-09-27 15:14:44 -0700339
David James3541a132013-03-18 13:21:58 -0700340 except Exception as ex:
341 # If the build is marked as successful, but threw exceptions, that's a
342 # problem.
David James7fbf2d42012-07-14 18:23:49 -0700343 exception_thrown = True
David James3541a132013-03-18 13:21:58 -0700344 if results_lib.Results.BuildSucceededSoFar():
345 traceback.print_exc(file=sys.stdout)
David James7fbf2d42012-07-14 18:23:49 -0700346 raise
Matt Tennant759e2352013-09-27 15:14:44 -0700347
David James3541a132013-03-18 13:21:58 -0700348 if not (print_report and isinstance(ex, results_lib.StepFailure)):
349 raise
Matt Tennant759e2352013-09-27 15:14:44 -0700350
Brian Harring3fec5a82012-03-01 05:57:03 -0800351 finally:
352 if print_report:
Matt Tennantefb59522013-10-18 13:49:28 -0700353 results_lib.WriteCheckpoint(self._run.options.buildroot)
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400354 completion_instance = self.GetCompletionInstance()
Matt Tennantefb59522013-10-18 13:49:28 -0700355 self._RunStage(stages.ReportStage, self.archive_stages, sync_instance,
356 completion_instance)
Brian Harring3fec5a82012-03-01 05:57:03 -0800357 success = results_lib.Results.BuildSucceededSoFar()
David James3d4d3502012-04-09 15:12:06 -0700358 if exception_thrown and success:
359 success = False
David Jamesbb20ac82012-07-18 10:59:16 -0700360 cros_build_lib.PrintBuildbotStepWarnings()
361 print """\
David James3d4d3502012-04-09 15:12:06 -0700362Exception thrown, but all stages marked successful. This is an internal error,
363because the stage that threw the exception should be marked as failing."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800364
365 return success
366
367
David James1fddb8f2013-04-25 15:23:07 -0700368BoardConfig = collections.namedtuple('BoardConfig', ['board', 'name'])
369
370
Brian Harring3fec5a82012-03-01 05:57:03 -0800371class SimpleBuilder(Builder):
372 """Builder that performs basic vetting operations."""
373
374 def GetSyncInstance(self):
375 """Sync to lkgm or TOT as necessary.
376
Mike Frysinger02e1e072013-11-10 22:11:34 -0500377 Returns:
378 The instance of the sync stage to run.
Brian Harring3fec5a82012-03-01 05:57:03 -0800379 """
Matt Tennantefb59522013-10-18 13:49:28 -0700380 if self._run.options.force_version:
Ryan Cui88b901c2013-06-21 11:35:30 -0700381 sync_stage = self._GetStageInstance(stages.ManifestVersionedSyncStage)
Matt Tennantefb59522013-10-18 13:49:28 -0700382 elif self._run.config.use_lkgm:
Brian Harring3fec5a82012-03-01 05:57:03 -0800383 sync_stage = self._GetStageInstance(stages.LKGMSyncStage)
Matt Tennantefb59522013-10-18 13:49:28 -0700384 elif self._run.config.use_chrome_lkgm:
Chris Sosa52a81b62012-11-14 06:12:54 -0800385 sync_stage = self._GetStageInstance(stages.ChromeLKGMSyncStage)
Brian Harring3fec5a82012-03-01 05:57:03 -0800386 else:
387 sync_stage = self._GetStageInstance(stages.SyncStage)
388
389 return sync_stage
390
David James4f2d8302013-03-19 06:35:58 -0700391 @staticmethod
392 def _RunParallelStages(stage_objs):
Matt Tennant759e2352013-09-27 15:14:44 -0700393 """Run the specified stages in parallel.
394
395 Args:
396 stage_objs: BuilderStage objects.
397 """
David James4f2d8302013-03-19 06:35:58 -0700398 steps = [stage.Run for stage in stage_objs]
399 try:
400 parallel.RunParallelSteps(steps)
401 except BaseException as ex:
402 # If a stage threw an exception, it might not have correctly reported
403 # results (e.g. because it was killed before it could report the
404 # results.) In this case, attribute the exception to any stages that
405 # didn't report back correctly (if any).
406 for stage in stage_objs:
David James6b704242013-10-18 14:51:37 -0700407 for name in stage.GetStageNames():
408 if not results_lib.Results.StageHasResults(name):
409 results_lib.Results.Record(name, ex, str(ex))
Matt Tennant759e2352013-09-27 15:14:44 -0700410
David James4f2d8302013-03-19 06:35:58 -0700411 raise
412
Matt Tennantefb59522013-10-18 13:49:28 -0700413 def _RunBackgroundStagesForBoard(self, builder_run, board, compilecheck):
Matt Tennant759e2352013-09-27 15:14:44 -0700414 """Run background board-specific stages for the specified board.
415
416 Args:
Matt Tennantefb59522013-10-18 13:49:28 -0700417 builder_run: BuilderRun object for these background stages.
Matt Tennant759e2352013-09-27 15:14:44 -0700418 board: Board name.
419 compilecheck: Boolean. If True, run only the compile steps.
420 """
Matt Tennantefb59522013-10-18 13:49:28 -0700421 config = builder_run.config
422 archive_stage = self.archive_stages[BoardConfig(board, config.name)]
423 if config.pgo_generate:
Ahmad Sharifa2db40a2013-07-22 13:51:52 -0400424 self._RunParallelStages([archive_stage])
David James1fddb8f2013-04-25 15:23:07 -0700425 return
Matt Tennant759e2352013-09-27 15:14:44 -0700426
David James565bc9a2013-04-08 14:54:45 -0700427 if compilecheck:
428 self._RunStage(stages.BuildPackagesStage, board, archive_stage,
Matt Tennantefb59522013-10-18 13:49:28 -0700429 builder_run=builder_run)
430 self._RunStage(stages.UnitTestStage, board,
431 builder_run=builder_run)
David James565bc9a2013-04-08 14:54:45 -0700432 return
Matt Tennant759e2352013-09-27 15:14:44 -0700433
Mike Frysinger31ac04f2013-11-10 21:07:48 -0500434 # While this stage list is run in parallel, the order here dictates the
435 # order that things will be shown in the log. So group things together
436 # that make sense when read in order. Also keep in mind that, since we
437 # gather output manually, early slow stages will prevent any output from
438 # later stages showing up until it finishes.
Ryan Cui3ea98e02013-08-07 16:01:48 -0700439 stage_list = []
Matt Tennantefb59522013-10-18 13:49:28 -0700440 if builder_run.options.chrome_sdk and config.chrome_sdk:
Ryan Cui3ea98e02013-08-07 16:01:48 -0700441 stage_list.append([stages.ChromeSDKStage, board, archive_stage])
David James6b704242013-10-18 14:51:37 -0700442 stage_list += [
443 [stages.RetryStage, 1, stages.VMTestStage, board, archive_stage],
444 [stages.SignerTestStage, board, archive_stage],
Don Garrett7604d6a2014-01-15 14:45:39 -0800445 [stages.SignerResultsStage, board, archive_stage],
David James6b704242013-10-18 14:51:37 -0700446 [stages.UnitTestStage, board],
447 [stages.UploadPrebuiltsStage, board, archive_stage],
Mike Frysinger31ac04f2013-11-10 21:07:48 -0500448 [stages.DevInstallerPrebuiltsStage, board, archive_stage],
449 [stages.DebugSymbolsStage, board, archive_stage],
David James6b704242013-10-18 14:51:37 -0700450 ]
Brian Harring3fec5a82012-03-01 05:57:03 -0800451
David James58e0c092012-03-04 20:31:12 -0800452 # We can not run hw tests without archiving the payloads.
Matt Tennantefb59522013-10-18 13:49:28 -0700453 if builder_run.options.archive:
454 for suite_config in config.hw_tests:
Chris Sosaba250522013-03-27 18:57:34 -0700455 if suite_config.async:
456 stage_list.append([stages.ASyncHWTestStage, board, archive_stage,
457 suite_config])
458 elif suite_config.suite == constants.HWTEST_AU_SUITE:
459 stage_list.append([stages.AUTestStage, board, archive_stage,
460 suite_config])
Aviv Keshet23efd042013-10-01 13:59:08 -0700461 elif suite_config.suite == constants.HWTEST_QAV_SUITE:
462 stage_list.append([stages.QATestStage, board,
463 archive_stage, suite_config])
Chris Sosa077df812013-01-25 17:05:28 -0800464 else:
Chris Sosaba250522013-03-27 18:57:34 -0700465 stage_list.append([stages.HWTestStage, board, archive_stage,
466 suite_config])
Chris Sosa817b1f92012-07-19 15:00:23 -0700467
Matt Tennantefb59522013-10-18 13:49:28 -0700468 stage_objs = [self._GetStageInstance(*x, builder_run=builder_run)
469 for x in stage_list]
David James4f2d8302013-03-19 06:35:58 -0700470 self._RunParallelStages(stage_objs + [archive_stage])
Brian Harring3fec5a82012-03-01 05:57:03 -0800471
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400472 def _RunChrootBuilderTypeBuild(self):
473 """Runs through stages of a CHROOT_BUILDER_TYPE build."""
474 self._RunStage(stages.UprevStage, boards=[], enter_chroot=False)
475 self._RunStage(stages.InitSDKStage)
476 self._RunStage(stages.SetupBoardStage, [constants.CHROOT_BUILDER_BOARD])
477 self._RunStage(stages.SyncChromeStage)
478 self._RunStage(stages.PatchChromeStage)
479 self._RunStage(stages.SDKPackageStage)
480 self._RunStage(stages.SDKTestStage)
481 self._RunStage(stages.UploadPrebuiltsStage,
482 constants.CHROOT_BUILDER_BOARD, None)
483
484 def _RunRefreshPackagesTypeBuild(self):
485 """Runs through the stages of a REFRESH_PACKAGES_TYPE build."""
486 self._RunStage(stages.InitSDKStage)
487 self._RunStage(stages.SetupBoardStage)
488 self._RunStage(stages.RefreshPackageStatusStage)
489
Matt Tennante8179042013-10-01 15:47:32 -0700490 def _RunMasterPaladinBuild(self):
491 """Runs through the stages of the paladin (commit queue) master build."""
492 self._RunStage(stages.InitSDKStage)
493 self._RunStage(stages.UprevStage)
494 self._RunStage(stages.MasterUploadPrebuiltsStage)
495
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400496 def _RunDefaultTypeBuild(self):
497 """Runs through the stages of a non-special-type build."""
498 self._RunStage(stages.InitSDKStage)
499 self._RunStage(stages.UprevStage)
500 self._RunStage(stages.SetupBoardStage)
Matt Tennant85348852014-01-10 15:16:02 -0800501 self._RunStage(stages.SyncChromeStage)
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400502 self._RunStage(stages.PatchChromeStage)
503
Matt Tennantefb59522013-10-18 13:49:28 -0700504 # Prepare stages to run in background. If child_configs exist then
505 # run each of those here, otherwise use default config.
Matt Tennant950c6cb2014-01-10 16:45:03 -0800506 builder_runs = self._run.GetUngroupedBuilderRuns()
Matt Tennantefb59522013-10-18 13:49:28 -0700507
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400508 tasks = []
Matt Tennantefb59522013-10-18 13:49:28 -0700509 for builder_run in builder_runs:
510 for board in builder_run.config.boards:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400511 archive_stage = self._GetStageInstance(
Matt Tennantefb59522013-10-18 13:49:28 -0700512 stages.ArchiveStage, board, builder_run=builder_run,
Matt Tennant85348852014-01-10 15:16:02 -0800513 chrome_version=self._run.attrs.chrome_version)
Matt Tennantefb59522013-10-18 13:49:28 -0700514 board_config = BoardConfig(board, builder_run.config.name)
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400515 self.archive_stages[board_config] = archive_stage
Matt Tennantefb59522013-10-18 13:49:28 -0700516 tasks.append((builder_run, board, archive_stage))
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400517
518 # Set up a process pool to run test/archive stages in the background.
519 # This process runs task(board) for each board added to the queue.
520 task_runner = self._RunBackgroundStagesForBoard
521 with parallel.BackgroundTaskRunner(task_runner) as queue:
Matt Tennantefb59522013-10-18 13:49:28 -0700522 for builder_run, board, archive_stage in tasks:
523 compilecheck = (builder_run.config.compilecheck or
524 builder_run.options.compilecheck)
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400525 if not compilecheck:
526 # Run BuildPackages and BuildImage in the foreground, generating
527 # or using PGO data if requested.
Matt Tennantefb59522013-10-18 13:49:28 -0700528 kwargs = {'archive_stage': archive_stage, 'builder_run': builder_run}
529 if builder_run.config.pgo_generate:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400530 kwargs['pgo_generate'] = True
Matt Tennantefb59522013-10-18 13:49:28 -0700531 elif builder_run.config.pgo_use:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400532 kwargs['pgo_use'] = True
533
534 self._RunStage(stages.BuildPackagesStage, board, **kwargs)
535 self._RunStage(stages.BuildImageStage, board, **kwargs)
536
Matt Tennantefb59522013-10-18 13:49:28 -0700537 if builder_run.config.pgo_generate:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400538 suite = cbuildbot_config.PGORecordTest()
539 self._RunStage(stages.HWTestStage, board, archive_stage, suite,
Matt Tennantefb59522013-10-18 13:49:28 -0700540 builder_run=builder_run)
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400541
542 # Kick off our background stages.
Matt Tennantefb59522013-10-18 13:49:28 -0700543 queue.put([builder_run, board, compilecheck])
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400544
Brian Harring3fec5a82012-03-01 05:57:03 -0800545 def RunStages(self):
546 """Runs through build process."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800547 # TODO(sosa): Split these out into classes.
Matt Tennantefb59522013-10-18 13:49:28 -0700548 if self._run.config.build_type == constants.PRE_CQ_LAUNCHER_TYPE:
David James816a5942013-04-11 22:59:32 -0700549 self._RunStage(stages.PreCQLauncherStage)
Matt Tennantefb59522013-10-18 13:49:28 -0700550 elif self._run.config.build_type == constants.CREATE_BRANCH_TYPE:
Ryan Cui88b901c2013-06-21 11:35:30 -0700551 self._RunStage(stages.BranchUtilStage)
Matt Tennantefb59522013-10-18 13:49:28 -0700552 elif self._run.config.build_type == constants.CHROOT_BUILDER_TYPE:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400553 self._RunChrootBuilderTypeBuild()
Matt Tennantefb59522013-10-18 13:49:28 -0700554 elif self._run.config.build_type == constants.REFRESH_PACKAGES_TYPE:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400555 self._RunRefreshPackagesTypeBuild()
Matt Tennante8179042013-10-01 15:47:32 -0700556 elif (self._run.config.build_type == constants.PALADIN_TYPE and
557 self._run.config.master):
558 self._RunMasterPaladinBuild()
Brian Harring3fec5a82012-03-01 05:57:03 -0800559 else:
Aviv Keshet7dc300f2013-10-11 12:57:49 -0400560 self._RunDefaultTypeBuild()
David James58e0c092012-03-04 20:31:12 -0800561
Brian Harring3fec5a82012-03-01 05:57:03 -0800562
563class DistributedBuilder(SimpleBuilder):
564 """Build class that has special logic to handle distributed builds.
565
566 These builds sync using git/manifest logic in manifest_versions. In general
567 they use a non-distributed builder code for the bulk of the work.
568 """
Ryan Cuif7f24692012-05-18 16:35:33 -0700569 def __init__(self, *args, **kwargs):
Brian Harring3fec5a82012-03-01 05:57:03 -0800570 """Initializes a buildbot builder.
571
572 Extra variables:
573 completion_stage_class: Stage used to complete a build. Set in the Sync
574 stage.
575 """
Ryan Cuif7f24692012-05-18 16:35:33 -0700576 super(DistributedBuilder, self).__init__(*args, **kwargs)
Brian Harring3fec5a82012-03-01 05:57:03 -0800577 self.completion_stage_class = None
David Jamesf421c6d2013-04-11 15:37:57 -0700578 self.sync_stage = None
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400579 self._completion_stage = None
Brian Harring3fec5a82012-03-01 05:57:03 -0800580
581 def GetSyncInstance(self):
582 """Syncs the tree using one of the distributed sync logic paths.
583
Mike Frysinger02e1e072013-11-10 22:11:34 -0500584 Returns:
585 The instance of the sync stage to run.
Brian Harring3fec5a82012-03-01 05:57:03 -0800586 """
587 # Determine sync class to use. CQ overrides PFQ bits so should check it
588 # first.
Matt Tennantefb59522013-10-18 13:49:28 -0700589 if self._run.config.pre_cq or self._run.options.pre_cq:
David Jamesf421c6d2013-04-11 15:37:57 -0700590 sync_stage = self._GetStageInstance(stages.PreCQSyncStage,
591 self.patch_pool.gerrit_patches)
592 self.completion_stage_class = stages.PreCQCompletionStage
593 self.patch_pool.gerrit_patches = []
Matt Tennantefb59522013-10-18 13:49:28 -0700594 elif cbuildbot_config.IsCQType(self._run.config.build_type):
595 if self._run.config.do_not_apply_cq_patches:
Aviv Keshet00bc9512013-11-25 14:58:47 -0800596 sync_stage = self._GetStageInstance(stages.MasterSlaveSyncStage)
597 else:
598 sync_stage = self._GetStageInstance(stages.CommitQueueSyncStage)
Brian Harring3fec5a82012-03-01 05:57:03 -0800599 self.completion_stage_class = stages.CommitQueueCompletionStage
Matt Tennantefb59522013-10-18 13:49:28 -0700600 elif cbuildbot_config.IsPFQType(self._run.config.build_type):
Aviv Keshetf2746152013-11-26 10:42:46 -0800601 sync_stage = self._GetStageInstance(stages.MasterSlaveSyncStage)
602 self.completion_stage_class = stages.MasterSlaveSyncCompletionStage
Brian Harring3fec5a82012-03-01 05:57:03 -0800603 else:
604 sync_stage = self._GetStageInstance(stages.ManifestVersionedSyncStage)
605 self.completion_stage_class = stages.ManifestVersionedSyncCompletionStage
606
David Jamesf421c6d2013-04-11 15:37:57 -0700607 self.sync_stage = sync_stage
608 return self.sync_stage
Brian Harring3fec5a82012-03-01 05:57:03 -0800609
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400610 def GetCompletionInstance(self):
611 """Returns the completion_stage_class instance that was used for this build.
612
Mike Frysinger1a736a82013-12-12 01:50:59 -0500613 Returns:
614 None if the completion_stage instance was not yet created (this
615 occurs during Publish).
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400616 """
617 return self._completion_stage
618
Brian Harring3fec5a82012-03-01 05:57:03 -0800619 def Publish(self, was_build_successful):
620 """Completes build by publishing any required information."""
621 completion_stage = self._GetStageInstance(self.completion_stage_class,
David Jamesf421c6d2013-04-11 15:37:57 -0700622 self.sync_stage,
Brian Harring3fec5a82012-03-01 05:57:03 -0800623 was_build_successful)
Aviv Keshet5c39b6a2013-10-07 15:44:54 -0400624 self._completion_stage = completion_stage
David James11fd7e82013-10-23 20:02:16 -0700625 completion_successful = False
626 try:
627 completion_stage.Run()
628 completion_successful = True
629 finally:
630 if not completion_successful:
631 was_build_successful = False
Matt Tennantefb59522013-10-18 13:49:28 -0700632 if self._run.config.push_overlays:
David James11fd7e82013-10-23 20:02:16 -0700633 self._RunStage(stages.PublishUprevChangesStage, was_build_successful)
Brian Harring3fec5a82012-03-01 05:57:03 -0800634
635 def RunStages(self):
636 """Runs simple builder logic and publishes information to overlays."""
637 was_build_successful = False
David James843ed252013-09-11 16:18:10 -0700638 try:
639 super(DistributedBuilder, self).RunStages()
640 was_build_successful = results_lib.Results.BuildSucceededSoFar()
641 except SystemExit as ex:
642 # If a stage calls sys.exit(0), it's exiting with success, so that means
643 # we should mark ourselves as successful.
644 if ex.code == 0:
645 was_build_successful = True
646 raise
647 finally:
648 self.Publish(was_build_successful)
Brian Harring3fec5a82012-03-01 05:57:03 -0800649
Brian Harring3fec5a82012-03-01 05:57:03 -0800650
651def _ConfirmBuildRoot(buildroot):
652 """Confirm with user the inferred buildroot, and mark it as confirmed."""
Brian Harring521e7242012-11-01 16:57:42 -0700653 cros_build_lib.Warning('Using default directory %s as buildroot', buildroot)
654 if not cros_build_lib.BooleanPrompt(default=False):
655 print('Please specify a different buildroot via the --buildroot option.')
Brian Harring3fec5a82012-03-01 05:57:03 -0800656 sys.exit(0)
657
658 if not os.path.exists(buildroot):
659 os.mkdir(buildroot)
660
661 repository.CreateTrybotMarker(buildroot)
662
663
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700664def _ConfirmRemoteBuildbotRun():
665 """Confirm user wants to run with --buildbot --remote."""
Brian Harring521e7242012-11-01 16:57:42 -0700666 cros_build_lib.Warning(
667 'You are about to launch a PRODUCTION job! This is *NOT* a '
668 'trybot run! Are you sure?')
669 if not cros_build_lib.BooleanPrompt(default=False):
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700670 print('Please specify --pass-through="--debug".')
671 sys.exit(0)
672
673
Ryan Cui5ba7e152012-05-10 14:36:52 -0700674def _DetermineDefaultBuildRoot(sourceroot, internal_build):
Brian Harring3fec5a82012-03-01 05:57:03 -0800675 """Default buildroot to be under the directory that contains current checkout.
676
Mike Frysinger02e1e072013-11-10 22:11:34 -0500677 Args:
Brian Harring3fec5a82012-03-01 05:57:03 -0800678 internal_build: Whether the build is an internal build
Ryan Cui5ba7e152012-05-10 14:36:52 -0700679 sourceroot: Use specified sourceroot.
Brian Harring3fec5a82012-03-01 05:57:03 -0800680 """
Ryan Cui5ba7e152012-05-10 14:36:52 -0700681 if not repository.IsARepoRoot(sourceroot):
Brian Harring1b8c4c82012-05-29 23:03:04 -0700682 cros_build_lib.Die(
683 'Could not find root of local checkout at %s. Please specify '
684 'using the --sourceroot option.' % sourceroot)
Brian Harring3fec5a82012-03-01 05:57:03 -0800685
686 # Place trybot buildroot under the directory containing current checkout.
Ryan Cui5ba7e152012-05-10 14:36:52 -0700687 top_level = os.path.dirname(os.path.realpath(sourceroot))
Brian Harring3fec5a82012-03-01 05:57:03 -0800688 if internal_build:
689 buildroot = os.path.join(top_level, _DEFAULT_INT_BUILDROOT)
690 else:
691 buildroot = os.path.join(top_level, _DEFAULT_EXT_BUILDROOT)
692
693 return buildroot
694
695
Mike Frysinger6903c762012-12-04 01:57:16 -0500696def _DisableYamaHardLinkChecks():
697 """Disable Yama kernel hardlink security checks.
698
699 The security module disables hardlinking to files you do not have
700 write access to which causes some of our build scripts problems.
701 Disable it so we don't have to worry about it.
702 """
703 PROC_PATH = '/proc/sys/kernel/yama/protected_nonaccess_hardlinks'
704 SYSCTL_PATH = PROC_PATH[len('/proc/sys/'):].replace('/', '.')
705
706 # Yama not available in this system -- nothing to do.
707 if not os.path.exists(PROC_PATH):
708 return
709
710 # Already disabled -- nothing to do.
711 if osutils.ReadFile(PROC_PATH).strip() == '0':
712 return
713
714 # Create a hardlink in a tempdir and see if we get back EPERM.
David James4bc13702013-03-26 08:08:04 -0700715 with osutils.TempDir() as tempdir:
Mike Frysinger6903c762012-12-04 01:57:16 -0500716 try:
717 os.link('/bin/sh', os.path.join(tempdir, 'sh'))
718 except OSError as e:
719 if e.errno == errno.EPERM:
720 cros_build_lib.Warning('Disabling Yama hardlink security')
721 cros_build_lib.SudoRunCommand(['sysctl', '%s=0' % SYSCTL_PATH])
722
723
Brian Harring3fec5a82012-03-01 05:57:03 -0800724def _BackupPreviousLog(log_file, backup_limit=25):
725 """Rename previous log.
726
727 Args:
728 log_file: The absolute path to the previous log.
Aviv Keshet9e4236b2013-12-13 13:07:50 -0800729 backup_limit: Maximum number of old logs to keep.
Brian Harring3fec5a82012-03-01 05:57:03 -0800730 """
731 if os.path.exists(log_file):
732 old_logs = sorted(glob.glob(log_file + '.*'),
733 key=distutils.version.LooseVersion)
734
735 if len(old_logs) >= backup_limit:
736 os.remove(old_logs[0])
737
738 last = 0
739 if old_logs:
740 last = int(old_logs.pop().rpartition('.')[2])
741
742 os.rename(log_file, log_file + '.' + str(last + 1))
743
Ryan Cui5616a512012-08-17 13:39:36 -0700744
Gaurav Shah298aa372014-01-31 09:27:24 -0800745def _IsDistributedBuilder(options, chrome_rev, build_config):
746 """Determines whether the builder should be a DistributedBuilder.
747
748 Args:
749 options: options passed on the commandline.
750 chrome_rev: Chrome revision to build.
751 build_config: Builder configuration dictionary.
752
753 Returns:
754 True if the builder should be a distributed_builder
755 """
756 if build_config['pre_cq'] or options.pre_cq:
757 return True
758 elif not options.buildbot:
759 return False
760 elif chrome_rev in (constants.CHROME_REV_TOT,
761 constants.CHROME_REV_LOCAL,
762 constants.CHROME_REV_SPEC):
763 # We don't do distributed logic to TOT Chrome PFQ's, nor local
764 # chrome roots (e.g. chrome try bots)
765 # TODO(davidjames): Update any builders that rely on this logic to use
766 # manifest_version=False instead.
767 return False
768 elif build_config['manifest_version']:
769 return True
770
771 return False
772
773
David James944a48e2012-03-07 12:19:03 -0800774def _RunBuildStagesWrapper(options, build_config):
Brian Harring3fec5a82012-03-01 05:57:03 -0800775 """Helper function that wraps RunBuildStages()."""
Matt Tennant7feda352013-12-20 14:03:40 -0800776 cros_build_lib.Info('cbuildbot was executed with args %s' %
777 cros_build_lib.CmdToStr(sys.argv))
Brian Harring3fec5a82012-03-01 05:57:03 -0800778
David Jamesa0a664e2013-02-13 09:52:01 -0800779 chrome_rev = build_config['chrome_rev']
780 if options.chrome_rev:
781 chrome_rev = options.chrome_rev
782 if chrome_rev == constants.CHROME_REV_TOT:
783 # Build the TOT Chrome revision.
784 svn_url = gclient.GetBaseURLs()[0]
785 options.chrome_version = gclient.GetTipOfTrunkSvnRevision(svn_url)
786 options.chrome_rev = constants.CHROME_REV_SPEC
787
David James4a404a52013-02-19 13:07:59 -0800788 # If it's likely we'll need to build Chrome, fetch the source.
Mike Frysingerf4d9ab82013-03-05 19:38:05 -0500789 if build_config['sync_chrome'] is None:
790 options.managed_chrome = (chrome_rev != constants.CHROME_REV_LOCAL and
791 (not build_config['usepkg_build_packages'] or chrome_rev or
David James3cce4642013-05-10 17:50:23 -0700792 build_config['profile'] or options.rietveld_patches))
Mike Frysingerf4d9ab82013-03-05 19:38:05 -0500793 else:
794 options.managed_chrome = build_config['sync_chrome']
David James2333c182013-02-13 16:16:15 -0800795
796 if options.managed_chrome:
797 # Tell Chrome to fetch the source locally.
Matt Tennant628ffdd2013-11-27 14:44:39 -0800798 internal = constants.USE_CHROME_INTERNAL in build_config['useflags']
David James2333c182013-02-13 16:16:15 -0800799 chrome_src = 'chrome-src-internal' if internal else 'chrome-src'
800 options.chrome_root = os.path.join(options.cache_dir, 'distfiles', 'target',
801 chrome_src)
David James9e27e662013-02-14 13:42:43 -0800802 elif options.rietveld_patches:
David James4a404a52013-02-19 13:07:59 -0800803 cros_build_lib.Die('This builder does not support Rietveld patches.')
David James2333c182013-02-13 16:16:15 -0800804
Matt Tennant95a42ad2013-12-27 15:38:36 -0800805 # We are done munging options values, so freeze options object now to avoid
806 # further abuse of it.
807 # TODO(mtennant): one by one identify each options value override and see if
808 # it can be handled another way. Try to push this freeze closer and closer
809 # to the start of the script (e.g. in or after _PostParseCheck).
810 options.Freeze()
811
Matt Tennantefb59522013-10-18 13:49:28 -0700812 builder_run = cbuildbot_run.BuilderRun(options, build_config)
Gaurav Shah298aa372014-01-31 09:27:24 -0800813 if _IsDistributedBuilder(options, chrome_rev, build_config):
814 builder_cls = DistributedBuilder
815 else:
816 builder_cls = SimpleBuilder
Matt Tennantefb59522013-10-18 13:49:28 -0700817 builder = builder_cls(builder_run)
818 if not builder.Run():
Brian Harringd166aaf2012-05-14 18:31:53 -0700819 sys.exit(1)
Brian Harring3fec5a82012-03-01 05:57:03 -0800820
821
822# Parser related functions
Ryan Cui5ba7e152012-05-10 14:36:52 -0700823def _CheckLocalPatches(sourceroot, local_patches):
Brian Harring3fec5a82012-03-01 05:57:03 -0800824 """Do an early quick check of the passed-in patches.
825
826 If the branch of a project is not specified we append the current branch the
827 project is on.
Ryan Cui5ba7e152012-05-10 14:36:52 -0700828
David Jamese3b06062013-11-09 18:52:02 -0800829 TODO(davidjames): The project:branch format isn't unique, so this means that
830 we can't differentiate what directory the user intended to apply patches to.
831 We should references by directory instead.
832
Ryan Cui5ba7e152012-05-10 14:36:52 -0700833 Args:
834 sourceroot: The checkout where patches are coming from.
Aviv Keshet9e4236b2013-12-13 13:07:50 -0800835 local_patches: List of patches to check in project:branch format.
David Jamese3b06062013-11-09 18:52:02 -0800836
837 Returns:
838 A list of patches that have been verified, in project:branch format.
Brian Harring3fec5a82012-03-01 05:57:03 -0800839 """
Ryan Cuicedd8a52012-03-22 02:28:35 -0700840 verified_patches = []
David James97d95872012-11-16 15:09:56 -0800841 manifest = git.ManifestCheckout.Cached(sourceroot)
Ryan Cuicedd8a52012-03-22 02:28:35 -0700842 for patch in local_patches:
David Jamese3b06062013-11-09 18:52:02 -0800843 project, _, branch = patch.partition(':')
844
Gaurav Shah7afb0562013-12-26 15:05:39 -0800845 checkouts = manifest.FindCheckouts(project, only_patchable=True)
David Jamese3b06062013-11-09 18:52:02 -0800846 if not checkouts:
847 cros_build_lib.Die('Project %s does not exist.' % (project,))
848 if len(checkouts) > 1:
Brian Harring1b8c4c82012-05-29 23:03:04 -0700849 cros_build_lib.Die(
David Jamese3b06062013-11-09 18:52:02 -0800850 'We do not yet support local patching for projects that are checked '
851 'out to multiple directories. Try uploading your patch to gerrit '
852 'and referencing it via the -g option instead.'
853 )
Brian Harring3fec5a82012-03-01 05:57:03 -0800854
David Jamese3b06062013-11-09 18:52:02 -0800855 ok = False
856 for checkout in checkouts:
857 project_dir = checkout.GetPath(absolute=True)
Brian Harring3fec5a82012-03-01 05:57:03 -0800858
David Jamese3b06062013-11-09 18:52:02 -0800859 # If no branch was specified, we use the project's current branch.
Brian Harring3fec5a82012-03-01 05:57:03 -0800860 if not branch:
David Jamese3b06062013-11-09 18:52:02 -0800861 local_branch = git.GetCurrentBranch(project_dir)
862 else:
863 local_branch = branch
Brian Harring3fec5a82012-03-01 05:57:03 -0800864
David Jamese3b06062013-11-09 18:52:02 -0800865 if local_branch and git.DoesLocalBranchExist(project_dir, local_branch):
866 verified_patches.append('%s:%s' % (project, local_branch))
867 ok = True
868
869 if not ok:
870 if branch:
871 cros_build_lib.Die('Project %s does not have branch %s'
872 % (project, branch))
873 else:
874 cros_build_lib.Die('Project %s is not on a branch!' % (project,))
Brian Harring3fec5a82012-03-01 05:57:03 -0800875
Ryan Cuicedd8a52012-03-22 02:28:35 -0700876 return verified_patches
Brian Harring3fec5a82012-03-01 05:57:03 -0800877
878
Brian Harring3fec5a82012-03-01 05:57:03 -0800879def _CheckChromeVersionOption(_option, _opt_str, value, parser):
880 """Upgrade other options based on chrome_version being passed."""
881 value = value.strip()
882
883 if parser.values.chrome_rev is None and value:
884 parser.values.chrome_rev = constants.CHROME_REV_SPEC
885
886 parser.values.chrome_version = value
887
888
889def _CheckChromeRootOption(_option, _opt_str, value, parser):
890 """Validate and convert chrome_root to full-path form."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800891 if parser.values.chrome_rev is None:
892 parser.values.chrome_rev = constants.CHROME_REV_LOCAL
893
Ryan Cui5ba7e152012-05-10 14:36:52 -0700894 parser.values.chrome_root = value
Brian Harring3fec5a82012-03-01 05:57:03 -0800895
896
897def _CheckChromeRevOption(_option, _opt_str, value, parser):
898 """Validate the chrome_rev option."""
899 value = value.strip()
900 if value not in constants.VALID_CHROME_REVISIONS:
901 raise optparse.OptionValueError('Invalid chrome rev specified')
902
903 parser.values.chrome_rev = value
904
905
David Jamesac8c2a72013-02-13 18:44:33 -0800906def FindCacheDir(_parser, _options):
Brian Harringae0a5322012-09-15 01:46:51 -0700907 return None
908
909
Ryan Cui5ba7e152012-05-10 14:36:52 -0700910class CustomGroup(optparse.OptionGroup):
Aviv Keshet9e4236b2013-12-13 13:07:50 -0800911 """Custom option group which supports arguments passed-through to trybot."""
Ryan Cui5ba7e152012-05-10 14:36:52 -0700912 def add_remote_option(self, *args, **kwargs):
913 """For arguments that are passed-through to remote trybot."""
914 return optparse.OptionGroup.add_option(self, *args,
915 remote_pass_through=True,
916 **kwargs)
917
918
Ryan Cui1c13a252012-10-16 15:00:16 -0700919class CustomOption(commandline.FilteringOption):
920 """Subclass FilteringOption class to implement pass-through and api."""
Ryan Cui5ba7e152012-05-10 14:36:52 -0700921
Ryan Cui1c13a252012-10-16 15:00:16 -0700922 ACTIONS = commandline.FilteringOption.ACTIONS + ('extend',)
923 STORE_ACTIONS = commandline.FilteringOption.STORE_ACTIONS + ('extend',)
924 TYPED_ACTIONS = commandline.FilteringOption.TYPED_ACTIONS + ('extend',)
925 ALWAYS_TYPED_ACTIONS = (commandline.FilteringOption.ALWAYS_TYPED_ACTIONS +
926 ('extend',))
Ryan Cui79319ab2012-05-21 12:59:18 -0700927
Ryan Cui5ba7e152012-05-10 14:36:52 -0700928 def __init__(self, *args, **kwargs):
929 # The remote_pass_through argument specifies whether we should directly
930 # pass the argument (with its value) onto the remote trybot.
931 self.pass_through = kwargs.pop('remote_pass_through', False)
Ryan Cui1c13a252012-10-16 15:00:16 -0700932 self.api_version = int(kwargs.pop('api', '0'))
933 commandline.FilteringOption.__init__(self, *args, **kwargs)
Ryan Cui5ba7e152012-05-10 14:36:52 -0700934
935 def take_action(self, action, dest, opt, value, values, parser):
Ryan Cui79319ab2012-05-21 12:59:18 -0700936 if action == 'extend':
Mike Frysingerd6925b52012-07-16 16:11:00 -0400937 # If there is extra spaces between each argument, we get '' which later
938 # code barfs on, so skip those. e.g. We see this with the forms:
939 # cbuildbot -p 'proj:branch ' ...
940 # cbuildbot -p ' proj:branch' ...
941 # cbuildbot -p 'proj:branch proj2:branch' ...
942 lvalue = value.split()
Ryan Cui79319ab2012-05-21 12:59:18 -0700943 values.ensure_value(dest, []).extend(lvalue)
Ryan Cui79319ab2012-05-21 12:59:18 -0700944
Ryan Cui1c13a252012-10-16 15:00:16 -0700945 commandline.FilteringOption.take_action(
946 self, action, dest, opt, value, values, parser)
Ryan Cui5ba7e152012-05-10 14:36:52 -0700947
948
Ryan Cui1c13a252012-10-16 15:00:16 -0700949class CustomParser(commandline.FilteringParser):
Aviv Keshet9e4236b2013-12-13 13:07:50 -0800950 """Custom option parser which supports arguments passed-trhough to trybot"""
Matt Tennante8179042013-10-01 15:47:32 -0700951
Brian Harringb6cf9142012-09-01 20:43:17 -0700952 DEFAULT_OPTION_CLASS = CustomOption
953
954 def add_remote_option(self, *args, **kwargs):
955 """For arguments that are passed-through to remote trybot."""
Ryan Cui1c13a252012-10-16 15:00:16 -0700956 return self.add_option(*args, remote_pass_through=True, **kwargs)
Brian Harringb6cf9142012-09-01 20:43:17 -0700957
958
Brian Harring3fec5a82012-03-01 05:57:03 -0800959def _CreateParser():
Ryan Cui16d9e1f2012-05-11 10:50:18 -0700960 """Generate and return the parser with all the options."""
Brian Harring3fec5a82012-03-01 05:57:03 -0800961 # Parse options
Matt Tennant763497d2014-01-17 16:45:54 -0800962 usage = "usage: %prog [options] buildbot_config [buildbot_config ...]"
Brian Harringae0a5322012-09-15 01:46:51 -0700963 parser = CustomParser(usage=usage, caching=FindCacheDir)
Brian Harring3fec5a82012-03-01 05:57:03 -0800964
965 # Main options
Mike Frysinger81af6ef2013-04-03 02:24:09 -0400966 parser.add_option('-l', '--list', action='store_true', dest='list',
967 default=False,
968 help='List the suggested trybot configs to use (see --all)')
Brian Harring3fec5a82012-03-01 05:57:03 -0800969 parser.add_option('-a', '--all', action='store_true', dest='print_all',
970 default=False,
Mike Frysinger81af6ef2013-04-03 02:24:09 -0400971 help='List all of the buildbot configs available w/--list')
972
973 parser.add_option('--local', default=False, action='store_true',
Ryan Cui88b901c2013-06-21 11:35:30 -0700974 help='Specifies that this tryjob should be run locally. '
975 'Implies --debug.')
Mike Frysinger81af6ef2013-04-03 02:24:09 -0400976 parser.add_option('--remote', default=False, action='store_true',
Ryan Cui88b901c2013-06-21 11:35:30 -0700977 help='Specifies that this tryjob should be run remotely.')
Mike Frysinger81af6ef2013-04-03 02:24:09 -0400978
Ryan Cuie1e4e662012-05-21 16:39:46 -0700979 parser.add_remote_option('-b', '--branch',
980 help='The manifest branch to test. The branch to '
981 'check the buildroot out to.')
Ryan Cui5ba7e152012-05-10 14:36:52 -0700982 parser.add_option('-r', '--buildroot', dest='buildroot', type='path',
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700983 help='Root directory where source is checked out to, and '
984 'where the build occurs. For external build configs, '
985 "defaults to 'trybot' directory at top level of your "
986 'repo-managed checkout.')
987 parser.add_remote_option('--chrome_rev', default=None, type='string',
988 action='callback', dest='chrome_rev',
989 callback=_CheckChromeRevOption,
990 help=('Revision of Chrome to use, of type [%s]'
991 % '|'.join(constants.VALID_CHROME_REVISIONS)))
Ryan Cuieaa9efd2012-04-25 17:56:45 -0700992 parser.add_remote_option('--profile', default=None, type='string',
993 action='store', dest='profile',
994 help='Name of profile to sub-specify board variant.')
Brian Harring3fec5a82012-03-01 05:57:03 -0800995
Ryan Cuif4f84be2012-07-09 18:50:41 -0700996 #
Mike Frysinger81af6ef2013-04-03 02:24:09 -0400997 # Patch selection options.
998 #
999
1000 group = CustomGroup(
1001 parser,
1002 'Patch Options')
1003
1004 group.add_remote_option('-g', '--gerrit-patches', action='extend',
1005 default=[], type='string',
1006 metavar="'Id1 *int_Id2...IdN'",
1007 help="Space-separated list of short-form Gerrit "
1008 "Change-Id's or change numbers to patch. "
1009 "Please prepend '*' to internal Change-Id's")
1010 group.add_remote_option('-G', '--rietveld-patches', action='extend',
1011 default=[], type='string',
1012 metavar="'id1[:subdir1]...idN[:subdirN]'",
1013 help='Space-separated list of short-form Rietveld '
1014 'issue numbers to patch. If no subdir is '
1015 'specified, the src directory is used.')
1016 group.add_option('-p', '--local-patches', action='extend', default=[],
1017 metavar="'<project1>[:<branch1>]...<projectN>[:<branchN>]'",
1018 help='Space-separated list of project branches with '
1019 'patches to apply. Projects are specified by name. '
1020 'If no branch is specified the current branch of the '
1021 'project will be used.')
1022
1023 parser.add_option_group(group)
1024
1025 #
1026 # Remote trybot options.
1027 #
1028
1029 group = CustomGroup(
1030 parser,
1031 'Remote Trybot Options (--remote)')
1032
1033 group.add_remote_option('--hwtest', dest='hwtest', action='store_true',
1034 default=False,
1035 help='Run the HWTest stage (tests on real hardware)')
1036 group.add_option('--remote-description', default=None,
1037 help='Attach an optional description to a --remote run '
1038 'to make it easier to identify the results when it '
1039 'finishes')
1040 group.add_option('--slaves', action='extend', default=[],
1041 help='Specify specific remote tryslaves to run on (e.g. '
1042 'build149-m2); if the bot is busy, it will be queued')
1043 group.add_option('--test-tryjob', action='store_true',
1044 default=False,
1045 help='Submit a tryjob to the test repository. Will not '
1046 'show up on the production trybot waterfall.')
1047
1048 parser.add_option_group(group)
1049
1050 #
Ryan Cui88b901c2013-06-21 11:35:30 -07001051 # Branch creation options.
1052 #
1053
1054 group = CustomGroup(
1055 parser,
1056 'Branch Creation Options (used with branch-util)')
1057
1058 group.add_remote_option('--branch-name',
1059 help='The branch to create or delete.')
1060 group.add_remote_option('--delete-branch', default=False, action='store_true',
1061 help='Delete the branch specified in --branch-name.')
1062 group.add_remote_option('--rename-to', type='string',
1063 help='Rename a branch to the specified name.')
1064 group.add_remote_option('--force-create', default=False, action='store_true',
1065 help='Overwrites an existing branch.')
1066
1067 parser.add_option_group(group)
1068
1069 #
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001070 # Advanced options.
Ryan Cuif4f84be2012-07-09 18:50:41 -07001071 #
1072
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001073 group = CustomGroup(
Brian Harring3fec5a82012-03-01 05:57:03 -08001074 parser,
1075 'Advanced Options',
1076 'Caution: use these options at your own risk.')
1077
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001078 group.add_remote_option('--bootstrap-args', action='append', default=[],
1079 help='Args passed directly to the bootstrap re-exec '
1080 'to skip verification by the bootstrap code')
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001081 group.add_remote_option('--buildbot', dest='buildbot', action='store_true',
1082 default=False, help='This is running on a buildbot')
1083 group.add_remote_option('--buildnumber', help='build number', type='int',
1084 default=0)
Ryan Cui5ba7e152012-05-10 14:36:52 -07001085 group.add_option('--chrome_root', default=None, type='path',
1086 action='callback', callback=_CheckChromeRootOption,
1087 dest='chrome_root', help='Local checkout of Chrome to use.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001088 group.add_remote_option('--chrome_version', default=None, type='string',
1089 action='callback', dest='chrome_version',
1090 callback=_CheckChromeVersionOption,
1091 help='Used with SPEC logic to force a particular SVN '
1092 'revision of chrome rather than the latest.')
1093 group.add_remote_option('--clobber', action='store_true', dest='clobber',
1094 default=False,
1095 help='Clears an old checkout before syncing')
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001096 group.add_remote_option('--latest-toolchain', action='store_true',
1097 default=False,
1098 help='Use the latest toolchain.')
Ryan Cui5ba7e152012-05-10 14:36:52 -07001099 parser.add_option('--log_dir', dest='log_dir', type='path',
Brian Harring3fec5a82012-03-01 05:57:03 -08001100 help=('Directory where logs are stored.'))
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001101 group.add_remote_option('--maxarchives', dest='max_archive_builds',
1102 default=3, type='int',
1103 help="Change the local saved build count limit.")
Ryan Cuibbd3d4b2012-08-17 12:20:37 -07001104 parser.add_remote_option('--manifest-repo-url',
1105 help=('Overrides the default manifest repo url.'))
David James565bc9a2013-04-08 14:54:45 -07001106 group.add_remote_option('--compilecheck', action='store_true', default=False,
1107 help='Only verify compilation and unit tests.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001108 group.add_remote_option('--noarchive', action='store_false', dest='archive',
1109 default=True, help="Don't run archive stage.")
Ryan Cuif7f24692012-05-18 16:35:33 -07001110 group.add_remote_option('--nobootstrap', action='store_false',
1111 dest='bootstrap', default=True,
1112 help="Don't checkout and run from a standalone "
1113 "chromite repo.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001114 group.add_remote_option('--nobuild', action='store_false', dest='build',
1115 default=True,
1116 help="Don't actually build (for cbuildbot dev)")
1117 group.add_remote_option('--noclean', action='store_false', dest='clean',
1118 default=True, help="Don't clean the buildroot")
Ryan Cuif7f24692012-05-18 16:35:33 -07001119 group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
1120 default=True,
1121 help='Disable cbuildbots usage of cgroups.')
Ryan Cui3ea98e02013-08-07 16:01:48 -07001122 group.add_remote_option('--nochromesdk', action='store_false',
1123 dest='chrome_sdk', default=True,
1124 help="Don't run the ChromeSDK stage which builds "
1125 "Chrome outside of the chroot.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001126 group.add_remote_option('--noprebuilts', action='store_false',
1127 dest='prebuilts', default=True,
1128 help="Don't upload prebuilts.")
Ryan Cui88b901c2013-06-21 11:35:30 -07001129 group.add_remote_option('--nopatch', action='store_false',
1130 dest='postsync_patch', default=True,
1131 help=("Don't run PatchChanges stage. This does not "
1132 "disable patching in of chromite patches "
1133 "during BootstrapStage."))
1134 group.add_remote_option('--noreexec', action='store_false',
1135 dest='postsync_reexec', default=True,
1136 help="Don't reexec into the buildroot after syncing.")
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001137 group.add_remote_option('--nosdk', action='store_true',
1138 default=False,
1139 help='Re-create the SDK from scratch.')
Don Garrett7604d6a2014-01-15 14:45:39 -08001140 group.add_remote_option('--nosigner-results', action='store_false',
1141 dest='signer_results', default=True,
1142 help="Don't display signing results.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001143 group.add_remote_option('--nosync', action='store_false', dest='sync',
1144 default=True, help="Don't sync before building.")
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001145 group.add_remote_option('--notests', action='store_false', dest='tests',
1146 default=True,
1147 help='Override values from buildconfig and run no '
1148 'tests.')
1149 group.add_remote_option('--nouprev', action='store_false', dest='uprev',
1150 default=True,
1151 help='Override values from buildconfig and never '
1152 'uprev.')
David Jamesdac7a912013-11-18 11:14:44 -08001153 group.add_option('--reference-repo', action='store', default=None,
1154 dest='reference_repo',
1155 help='Reuse git data stored in an existing repo '
1156 'checkout. This can drastically reduce the network '
1157 'time spent setting up the trybot checkout. By '
1158 "default, if this option isn't given but cbuildbot "
1159 'is invoked from a repo checkout, cbuildbot will '
1160 'use the repo root.')
Ryan Cuicedd8a52012-03-22 02:28:35 -07001161 group.add_option('--resume', action='store_true', default=False,
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001162 help='Skip stages already successfully completed.')
1163 group.add_remote_option('--timeout', action='store', type='int', default=0,
1164 help='Specify the maximum amount of time this job '
1165 'can run for, at which point the build will be '
1166 'aborted. If set to zero, then there is no '
1167 'timeout.')
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001168 group.add_remote_option('--version', dest='force_version', default=None,
1169 help='Used with manifest logic. Forces use of this '
Aviv Keshetc2c123d2013-10-14 18:06:12 -04001170 'version rather than create or get latest. '
1171 'Examples: 4815.0.0-rc1, 4815.1.2')
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001172
1173 parser.add_option_group(group)
1174
1175 #
1176 # Internal options.
1177 #
1178
1179 group = CustomGroup(
1180 parser,
Mike Frysinger34db8692013-11-11 14:54:08 -05001181 'Internal Chromium OS Build Team Options',
1182 'Caution: these are for meant for the Chromium OS build team only')
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001183
1184 group.add_remote_option('--archive-base', type='gs_path',
1185 help='Base GS URL (gs://<bucket_name>/<path>) to '
1186 'upload archive artifacts to')
Brian Harringab8fb5c2012-07-18 11:28:22 -07001187 group.add_remote_option('--cq-gerrit-query', dest='cq_gerrit_override',
1188 default=None,
1189 help=
1190 "If given, this gerrit query will be used to find what patches to test, "
Aviv Keshetefd6bbf2013-12-12 13:36:57 -08001191 "rather than the normal 'CommitQueue>=1 AND Verified=1 AND CodeReview=2' "
Brian Harringab8fb5c2012-07-18 11:28:22 -07001192 "query it defaults to. Use with care- note additionally this setting "
1193 "only has an effect if the buildbot target is a cq target, and we're "
1194 "in buildbot mode.")
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001195 group.add_option('--pass-through', dest='pass_through_args', action='append',
1196 type='string', default=[])
David Jamesf421c6d2013-04-11 15:37:57 -07001197 group.add_remote_option('--pre-cq', action='store_true', default=False,
1198 help='Mark CLs as tested by the PreCQ on success.')
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001199 group.add_option('--reexec-api-version', dest='output_api_version',
Ryan Cuif4f84be2012-07-09 18:50:41 -07001200 action='store_true', default=False,
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001201 help='Used for handling forwards/backwards compatibility '
1202 'with --resume and --bootstrap')
1203 group.add_option('--remote-trybot', dest='remote_trybot',
1204 action='store_true', default=False,
1205 help='Indicates this is running on a remote trybot machine')
1206 group.add_remote_option('--remote-patches', action='extend', default=[],
1207 help='Patches uploaded by the trybot client when run '
1208 'using the -p option')
Brian Harringf611e6e2012-07-17 18:47:44 -07001209 # Note the default here needs to be hardcoded to 3; that is the last version
1210 # that lacked this functionality.
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001211 group.add_option('--remote-version', default=3, type=int, action='store',
1212 help='Used for compatibility checks w/tryjobs running in '
1213 'older chromite instances')
1214 group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT)
1215 group.add_remote_option('--test-bootstrap', action='store_true',
1216 default=False,
1217 help='Causes cbuildbot to bootstrap itself twice, in '
1218 'the sequence A->B->C: A(unpatched) patches and '
1219 'bootstraps B; B patches and bootstraps C')
1220 group.add_remote_option('--validation_pool', default=None,
1221 help='Path to a pickled validation pool. Intended '
1222 'for use only with the commit queue.')
Aviv Kesheta0159be2013-12-12 13:56:28 -08001223 group.add_remote_option('--mock-tree-status', dest='mock_tree_status',
1224 default=None, action='store',
1225 help='Override the tree status value that would be '
1226 'returned from the the actual tree. Example '
1227 'values: open, closed, throttled. When used '
1228 'in conjunction with --debug, the tree status '
1229 'will not be ignored as it usually is in a '
1230 '--debug run.')
Mike Frysinger81af6ef2013-04-03 02:24:09 -04001231
1232 parser.add_option_group(group)
Ryan Cuif4f84be2012-07-09 18:50:41 -07001233
1234 #
Brian Harring3fec5a82012-03-01 05:57:03 -08001235 # Debug options
Ryan Cuif4f84be2012-07-09 18:50:41 -07001236 #
Brian Harringfec89fe2012-09-23 07:30:54 -07001237 # Temporary hack; in place till --dry-run replaces --debug.
1238 # pylint: disable=W0212
Brian Harring009db502012-10-10 02:21:37 -07001239 group = parser.debug_group
Brian Harringfec89fe2012-09-23 07:30:54 -07001240 debug = [x for x in group.option_list if x._long_opts == ['--debug']][0]
1241 debug.help += " Currently functions as --dry-run in addition."
1242 debug.pass_through = True
Brian Harring3fec5a82012-03-01 05:57:03 -08001243 group.add_option('--notee', action='store_false', dest='tee', default=True,
1244 help="Disable logging and internal tee process. Primarily "
1245 "used for debugging cbuildbot itself.")
Brian Harring3fec5a82012-03-01 05:57:03 -08001246 return parser
1247
1248
Ryan Cui85867972012-02-23 18:21:49 -08001249def _FinishParsing(options, args):
1250 """Perform some parsing tasks that need to take place after optparse.
1251
1252 This function needs to be easily testable! Keep it free of
1253 environment-dependent code. Put more detailed usage validation in
1254 _PostParseCheck().
Brian Harring3fec5a82012-03-01 05:57:03 -08001255
1256 Args:
Matt Tennant759e2352013-09-27 15:14:44 -07001257 options: The options object returned by optparse
1258 args: The args object returned by optparse
Brian Harring3fec5a82012-03-01 05:57:03 -08001259 """
Ryan Cui41023d92012-11-13 19:59:50 -08001260 # Populate options.pass_through_args.
1261 accepted, _ = commandline.FilteringParser.FilterArgs(
1262 options.parsed_args, lambda x: x.opt_inst.pass_through)
1263 options.pass_through_args.extend(accepted)
Brian Harring07039b52012-05-13 17:56:47 -07001264
Brian Harring3fec5a82012-03-01 05:57:03 -08001265 if options.chrome_root:
1266 if options.chrome_rev != constants.CHROME_REV_LOCAL:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001267 cros_build_lib.Die('Chrome rev must be %s if chrome_root is set.' %
1268 constants.CHROME_REV_LOCAL)
David Jamesa0a664e2013-02-13 09:52:01 -08001269 elif options.chrome_rev == constants.CHROME_REV_LOCAL:
1270 cros_build_lib.Die('Chrome root must be set if chrome_rev is %s.' %
1271 constants.CHROME_REV_LOCAL)
Brian Harring3fec5a82012-03-01 05:57:03 -08001272
1273 if options.chrome_version:
1274 if options.chrome_rev != constants.CHROME_REV_SPEC:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001275 cros_build_lib.Die('Chrome rev must be %s if chrome_version is set.' %
1276 constants.CHROME_REV_SPEC)
David Jamesa0a664e2013-02-13 09:52:01 -08001277 elif options.chrome_rev == constants.CHROME_REV_SPEC:
1278 cros_build_lib.Die(
1279 'Chrome rev must not be %s if chrome_version is not set.'
1280 % constants.CHROME_REV_SPEC)
Brian Harring3fec5a82012-03-01 05:57:03 -08001281
David James9e27e662013-02-14 13:42:43 -08001282 patches = bool(options.gerrit_patches or options.local_patches or
1283 options.rietveld_patches)
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001284 if options.remote:
1285 if options.local:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001286 cros_build_lib.Die('Cannot specify both --remote and --local')
Ryan Cui54da0702012-04-19 18:38:08 -07001287
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001288 if not options.buildbot and not patches:
Gaurav Shahed0399e2014-02-03 13:36:22 -08001289 prompt = ('No patches were provided; are you sure you want to just '
1290 'run a remote build of %s?' % (
1291 options.branch if options.branch else 'ToT'))
1292 if not cros_build_lib.BooleanPrompt(prompt=prompt, default=False):
Brian Harring7cc4b932012-11-01 16:58:55 -07001293 cros_build_lib.Die('Must provide patches when running with --remote.')
Brian Harring3fec5a82012-03-01 05:57:03 -08001294
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001295 # --debug needs to be explicitly passed through for remote invocations.
1296 release_mode_with_patches = (options.buildbot and patches and
1297 '--debug' not in options.pass_through_args)
1298 else:
1299 if len(args) > 1:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001300 cros_build_lib.Die('Multiple configs not supported if not running with '
Brian Harringf1aad832012-07-18 10:46:39 -07001301 '--remote. Got %r', args)
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001302
Ryan Cui79319ab2012-05-21 12:59:18 -07001303 if options.slaves:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001304 cros_build_lib.Die('Cannot use --slaves if not running with --remote.')
Ryan Cui79319ab2012-05-21 12:59:18 -07001305
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001306 release_mode_with_patches = (options.buildbot and patches and
1307 not options.debug)
1308
David James5734ea32012-08-15 20:23:49 -07001309 # When running in release mode, make sure we are running with checked-in code.
1310 # We want checked-in cbuildbot/scripts to prevent errors, and we want to build
1311 # a release image with checked-in code for CrOS packages.
1312 if release_mode_with_patches:
1313 cros_build_lib.Die(
1314 'Cannot provide patches when running with --buildbot!')
1315
Ryan Cuiba41ad32012-03-08 17:15:29 -08001316 if options.buildbot and options.remote_trybot:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001317 cros_build_lib.Die(
1318 '--buildbot and --remote-trybot cannot be used together.')
Ryan Cuiba41ad32012-03-08 17:15:29 -08001319
Ryan Cui85867972012-02-23 18:21:49 -08001320 # Record whether --debug was set explicitly vs. it was inferred.
1321 options.debug_forced = False
1322 if options.debug:
1323 options.debug_forced = True
Ryan Cui88b901c2013-06-21 11:35:30 -07001324 if not options.debug:
Ryan Cui16ca5812012-03-08 20:34:27 -08001325 # We don't set debug by default for
1326 # 1. --buildbot invocations.
1327 # 2. --remote invocations, because it needs to push changes to the tryjob
1328 # repo.
1329 options.debug = not options.buildbot and not options.remote
Brian Harring3fec5a82012-03-01 05:57:03 -08001330
Ryan Cui1c13a252012-10-16 15:00:16 -07001331 # Record the configs targeted.
1332 options.build_targets = args[:]
1333
Ryan Cui88b901c2013-06-21 11:35:30 -07001334 if constants.BRANCH_UTIL_CONFIG in options.build_targets:
Ryan Cui7bd8db62013-08-08 16:29:51 -07001335 if options.remote:
1336 cros_build_lib.Die(
Matt Tennanta130ea32013-12-19 09:38:39 -08001337 'Running %s as a remote tryjob is not yet supported.',
1338 constants.BRANCH_UTIL_CONFIG)
Ryan Cui88b901c2013-06-21 11:35:30 -07001339 if len(options.build_targets) > 1:
1340 cros_build_lib.Die(
Matt Tennanta130ea32013-12-19 09:38:39 -08001341 'Cannot run %s with any other configs.',
1342 constants.BRANCH_UTIL_CONFIG)
Ryan Cui88b901c2013-06-21 11:35:30 -07001343 if not options.branch_name:
1344 cros_build_lib.Die(
Matt Tennanta130ea32013-12-19 09:38:39 -08001345 'Must specify --branch-name with the %s config.',
1346 constants.BRANCH_UTIL_CONFIG)
1347 if options.branch and options.branch != options.branch_name:
Ryan Cui88b901c2013-06-21 11:35:30 -07001348 cros_build_lib.Die(
Matt Tennanta130ea32013-12-19 09:38:39 -08001349 'If --branch is specified with the %s config, it must'
1350 ' have the same value as --branch-name.',
1351 constants.BRANCH_UTIL_CONFIG)
1352
1353 exclusive_opts = {'--version': options.force_version,
1354 '--delete-branch': options.delete_branch,
1355 '--rename-to': options.rename_to,
1356 }
1357 if 1 != sum(1 for x in exclusive_opts.values() if x):
1358 cros_build_lib.Die('When using the %s config, you must'
1359 ' specifiy one and only one of the following'
1360 ' options: %s.', constants.BRANCH_UTIL_CONFIG,
1361 ', '.join(exclusive_opts.keys()))
1362
1363 # When deleting or renaming a branch, the --branch and --nobootstrap
1364 # options are implied.
1365 if options.delete_branch or options.rename_to:
1366 if not options.branch:
1367 cros_build_lib.Info('Automatically enabling sync to branch %s'
1368 ' for this %s flow.', options.branch_name,
1369 constants.BRANCH_UTIL_CONFIG)
1370 options.branch = options.branch_name
1371 if options.bootstrap:
1372 cros_build_lib.Info('Automatically disabling bootstrap step for'
1373 ' this %s flow.', constants.BRANCH_UTIL_CONFIG)
1374 options.bootstrap = False
1375
Ryan Cui88b901c2013-06-21 11:35:30 -07001376 elif any([options.delete_branch, options.rename_to, options.branch_name]):
1377 cros_build_lib.Die(
1378 'Cannot specify --delete-branch, --rename-to or --branch-name when not '
1379 'running the %s config', constants.BRANCH_UTIL_CONFIG)
1380
Brian Harring3fec5a82012-03-01 05:57:03 -08001381
Brian Harring1d7ba942012-04-24 06:37:18 -07001382# pylint: disable=W0613
Brian Harringae0a5322012-09-15 01:46:51 -07001383def _PostParseCheck(parser, options, args):
Ryan Cui85867972012-02-23 18:21:49 -08001384 """Perform some usage validation after we've parsed the arguments
Brian Harring3fec5a82012-03-01 05:57:03 -08001385
Ryan Cui85867972012-02-23 18:21:49 -08001386 Args:
Aviv Keshet9e4236b2013-12-13 13:07:50 -08001387 parser: Option parser that was used to parse arguments.
1388 options: The options returned by optparse.
1389 args: The args returned by optparse.
Ryan Cui85867972012-02-23 18:21:49 -08001390 """
Ryan Cuie1e4e662012-05-21 16:39:46 -07001391 if not options.branch:
David James97d95872012-11-16 15:09:56 -08001392 options.branch = git.GetChromiteTrackingBranch()
Ryan Cuie1e4e662012-05-21 16:39:46 -07001393
Brian Harringae0a5322012-09-15 01:46:51 -07001394 if not repository.IsARepoRoot(options.sourceroot):
1395 if options.local_patches:
1396 raise Exception('Could not find repo checkout at %s!'
1397 % options.sourceroot)
1398
David Jamesac8c2a72013-02-13 18:44:33 -08001399 # Because the default cache dir depends on other options, FindCacheDir
1400 # always returns None, and we setup the default here.
Brian Harringae0a5322012-09-15 01:46:51 -07001401 if options.cache_dir is None:
1402 # Note, options.sourceroot is set regardless of the path
1403 # actually existing.
David Jamesac8c2a72013-02-13 18:44:33 -08001404 if options.buildroot is not None:
Brian Harringae0a5322012-09-15 01:46:51 -07001405 options.cache_dir = os.path.join(options.buildroot, '.cache')
David Jamesac8c2a72013-02-13 18:44:33 -08001406 elif os.path.exists(options.sourceroot):
1407 options.cache_dir = os.path.join(options.sourceroot, '.cache')
Brian Harringae0a5322012-09-15 01:46:51 -07001408 else:
1409 options.cache_dir = parser.FindCacheDir(parser, options)
1410 options.cache_dir = os.path.abspath(options.cache_dir)
Brian Harring8c1d7b12012-10-04 17:36:32 -07001411 parser.ConfigureCacheDir(options.cache_dir)
Brian Harringae0a5322012-09-15 01:46:51 -07001412
Yu-Ju Hong2c066762013-10-28 14:05:08 -07001413 osutils.SafeMakedirsNonRoot(options.cache_dir)
Ryan Cui5ba7e152012-05-10 14:36:52 -07001414
Brian Harring609dc4e2012-05-07 02:17:44 -07001415 if options.local_patches:
Brian Harring1d7ba942012-04-24 06:37:18 -07001416 options.local_patches = _CheckLocalPatches(
Brian Harring609dc4e2012-05-07 02:17:44 -07001417 options.sourceroot, options.local_patches)
Brian Harring1d7ba942012-04-24 06:37:18 -07001418
1419 default = os.environ.get('CBUILDBOT_DEFAULT_MODE')
1420 if (default and not any([options.local, options.buildbot,
1421 options.remote, options.remote_trybot])):
Brian Harring1b8c4c82012-05-29 23:03:04 -07001422 cros_build_lib.Info("CBUILDBOT_DEFAULT_MODE=%s env var detected, using it."
1423 % default)
Brian Harring1d7ba942012-04-24 06:37:18 -07001424 default = default.lower()
1425 if default == 'local':
1426 options.local = True
1427 elif default == 'remote':
1428 options.remote = True
1429 elif default == 'buildbot':
1430 options.buildbot = True
1431 else:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001432 cros_build_lib.Die("CBUILDBOT_DEFAULT_MODE value %s isn't supported. "
1433 % default)
Ryan Cui85867972012-02-23 18:21:49 -08001434
Matt Tennant763497d2014-01-17 16:45:54 -08001435 # Ensure that all args are legitimate config targets.
1436 invalid_target = False
1437 for arg in args:
Matt Tennant2c192032014-01-16 13:49:28 -08001438 build_config = _GetConfig(arg)
1439 if not build_config:
Matt Tennant763497d2014-01-17 16:45:54 -08001440 cros_build_lib.Error('No such configuraton target: "%s".', arg)
1441 invalid_target = True
1442
Matt Tennant2c192032014-01-16 13:49:28 -08001443 # The --version option is not compatible with an external target unless the
1444 # --buildbot option is specified. More correctly, only "paladin versions"
1445 # will work with external targets, and those are only used with --buildbot.
1446 # If --buildbot is specified, then user should know what they are doing and
1447 # only specify a version that will work. See crbug.com/311648.
1448 elif options.force_version and not options.buildbot:
1449 if not build_config.internal:
1450 cros_build_lib.Die('Cannot specify --version without --buildbot for an'
1451 ' external target (%s).' % arg)
1452
Matt Tennant763497d2014-01-17 16:45:54 -08001453 if invalid_target:
1454 print 'Please specify one of:'
1455 _PrintValidConfigs()
1456 cros_build_lib.Die('One or more invalid configuration targets specified.')
1457
Ryan Cui85867972012-02-23 18:21:49 -08001458
1459def _ParseCommandLine(parser, argv):
1460 """Completely parse the commandline arguments"""
Brian Harring3fec5a82012-03-01 05:57:03 -08001461 (options, args) = parser.parse_args(argv)
Brian Harring37e559b2012-05-22 20:47:32 -07001462
Matt Tennant763497d2014-01-17 16:45:54 -08001463 # Strip out null arguments.
1464 # TODO(rcui): Remove when buildbot is fixed
1465 args = [arg for arg in args if arg]
1466
1467 # A couple options, like --list, trigger a quick exit.
Brian Harring37e559b2012-05-22 20:47:32 -07001468 if options.output_api_version:
Ryan Cui724f1d32012-10-15 18:10:22 -07001469 print constants.REEXEC_API_VERSION
Brian Harring37e559b2012-05-22 20:47:32 -07001470 sys.exit(0)
1471
Ryan Cui54da0702012-04-19 18:38:08 -07001472 if options.list:
Matt Tennant763497d2014-01-17 16:45:54 -08001473 if args:
1474 cros_build_lib.Die('No arguments expected with the --list options.')
Ryan Cui54da0702012-04-19 18:38:08 -07001475 _PrintValidConfigs(options.print_all)
1476 sys.exit(0)
1477
Ryan Cui8be16062012-04-24 12:05:26 -07001478 if not args:
Matt Tennant763497d2014-01-17 16:45:54 -08001479 parser.error('Invalid usage: no configuration targets provided.'
1480 'Use -h to see usage. Use -l to list supported configs.')
Ryan Cui8be16062012-04-24 12:05:26 -07001481
Ryan Cui85867972012-02-23 18:21:49 -08001482 _FinishParsing(options, args)
1483 return options, args
1484
1485
Matt Tennant759e2352013-09-27 15:14:44 -07001486# TODO(build): This function is too damn long.
Ryan Cui85867972012-02-23 18:21:49 -08001487def main(argv):
David James59a0a2b2013-03-22 14:04:44 -07001488 # Turn on strict sudo checks.
1489 cros_build_lib.STRICT_SUDO = True
1490
Ryan Cui85867972012-02-23 18:21:49 -08001491 # Set umask to 022 so files created by buildbot are readable.
Mike Frysinger60ec1012013-10-21 00:11:10 -04001492 os.umask(0o22)
Ryan Cui85867972012-02-23 18:21:49 -08001493
Ryan Cui85867972012-02-23 18:21:49 -08001494 parser = _CreateParser()
1495 (options, args) = _ParseCommandLine(parser, argv)
Brian Harring3fec5a82012-03-01 05:57:03 -08001496
Brian Harringae0a5322012-09-15 01:46:51 -07001497 _PostParseCheck(parser, options, args)
Brian Harring3fec5a82012-03-01 05:57:03 -08001498
Mike Frysinger8fd67dc2012-12-03 23:51:18 -05001499 cros_build_lib.AssertOutsideChroot()
Zdenek Behan98ec2fb2012-08-31 17:12:18 +02001500
Brian Harring3fec5a82012-03-01 05:57:03 -08001501 if options.remote:
Brian Harring1b8c4c82012-05-29 23:03:04 -07001502 cros_build_lib.logger.setLevel(logging.WARNING)
Ryan Cui16ca5812012-03-08 20:34:27 -08001503
Brian Harring3fec5a82012-03-01 05:57:03 -08001504 # Verify configs are valid.
Dan Shi0bdb7132013-07-30 16:22:12 -07001505 # If hwtest flag is enabled, post a warning that HWTest step may fail if the
1506 # specified board is not a released platform or it is a generic overlay.
Brian Harring3fec5a82012-03-01 05:57:03 -08001507 for bot in args:
Aviv Kesheta96a2a92013-02-05 17:21:51 -08001508 build_config = _GetConfig(bot)
1509 if options.hwtest:
Dan Shi0bdb7132013-07-30 16:22:12 -07001510 cros_build_lib.Warning(
1511 'If %s is not a released platform or it is a generic overlay, '
1512 'the HWTest step will most likely not run; please ask the lab '
1513 'team for help if this is unexpected.' % build_config['boards'])
Brian Harring3fec5a82012-03-01 05:57:03 -08001514
1515 # Verify gerrit patches are valid.
Ryan Cui16ca5812012-03-08 20:34:27 -08001516 print 'Verifying patches...'
Ryan Cuie1e4e662012-05-21 16:39:46 -07001517 patch_pool = AcquirePoolFromOptions(options)
Ryan Cui16d9e1f2012-05-11 10:50:18 -07001518
Ryan Cuieaa9efd2012-04-25 17:56:45 -07001519 # --debug need to be explicitly passed through for remote invocations.
1520 if options.buildbot and '--debug' not in options.pass_through_args:
1521 _ConfirmRemoteBuildbotRun()
1522
Ryan Cui16ca5812012-03-08 20:34:27 -08001523 print 'Submitting tryjob...'
Ryan Cui16d9e1f2012-05-11 10:50:18 -07001524 tryjob = remote_try.RemoteTryJob(options, args, patch_pool.local_patches)
Ryan Cuia25d8eb2012-07-11 14:54:27 -07001525 tryjob.Submit(testjob=options.test_tryjob, dryrun=False)
Ryan Cui16ca5812012-03-08 20:34:27 -08001526 print 'Tryjob submitted!'
1527 print ('Go to %s to view the status of your job.'
Ryan Cui4906e1c2012-04-03 20:09:34 -07001528 % tryjob.GetTrybotWaterfallLink())
Brian Harring3fec5a82012-03-01 05:57:03 -08001529 sys.exit(0)
Matt Tennant759e2352013-09-27 15:14:44 -07001530
Ryan Cui54da0702012-04-19 18:38:08 -07001531 elif (not options.buildbot and not options.remote_trybot
1532 and not options.resume and not options.local):
Ryan Cui88b901c2013-06-21 11:35:30 -07001533 options.local = True
Brian Harring1b8c4c82012-05-29 23:03:04 -07001534 cros_build_lib.Warning(
1535 'Running in LOCAL TRYBOT mode! Use --remote to submit REMOTE '
1536 'tryjobs. Use --local to suppress this message.')
1537 cros_build_lib.Warning(
Ryan Cui51591352012-07-09 15:15:53 -07001538 'In the future, --local will be required to run the local '
Brian Harring1b8c4c82012-05-29 23:03:04 -07001539 'trybot.')
Ryan Cui54da0702012-04-19 18:38:08 -07001540 time.sleep(5)
Brian Harring3fec5a82012-03-01 05:57:03 -08001541
Matt Tennant759e2352013-09-27 15:14:44 -07001542 # Only one config arg is allowed in this mode, which was confirmed earlier.
Ryan Cui8be16062012-04-24 12:05:26 -07001543 bot_id = args[-1]
1544 build_config = _GetConfig(bot_id)
Brian Harring3fec5a82012-03-01 05:57:03 -08001545
David Jamesdac7a912013-11-18 11:14:44 -08001546 if options.reference_repo is None:
1547 repo_path = os.path.join(options.sourceroot, '.repo')
1548 # If we're being run from a repo checkout, reuse the repo's git pool to
1549 # cut down on sync time.
1550 if os.path.exists(repo_path):
1551 options.reference_repo = options.sourceroot
1552 elif options.reference_repo:
1553 if not os.path.exists(options.reference_repo):
1554 parser.error('Reference path %s does not exist'
1555 % (options.reference_repo,))
1556 elif not os.path.exists(os.path.join(options.reference_repo, '.repo')):
1557 parser.error('Reference path %s does not look to be the base of a '
1558 'repo checkout; no .repo exists in the root.'
1559 % (options.reference_repo,))
1560
Brian Harringf11bf682012-05-14 15:53:43 -07001561 if (options.buildbot or options.remote_trybot) and not options.resume:
Brian Harring470f6112012-03-02 11:47:10 -08001562 if not options.cgroups:
Ryan Cuid4a24212012-04-04 18:08:12 -07001563 parser.error('Options --buildbot/--remote-trybot and --nocgroups cannot '
1564 'be used together. Cgroup support is required for '
1565 'buildbot/remote-trybot mode.')
Mike Frysingera78a56e2012-11-20 06:02:30 -05001566 if not cgroups.Cgroup.IsSupported():
Ryan Cuid4a24212012-04-04 18:08:12 -07001567 parser.error('Option --buildbot/--remote-trybot was given, but this '
1568 'system does not support cgroups. Failing.')
Brian Harring3fec5a82012-03-01 05:57:03 -08001569
David Jamesaad5cc72012-10-26 15:03:13 -07001570 missing = osutils.FindMissingBinaries(_BUILDBOT_REQUIRED_BINARIES)
Brian Harring351ce442012-03-09 16:38:14 -08001571 if missing:
Ryan Cuid4a24212012-04-04 18:08:12 -07001572 parser.error("Option --buildbot/--remote-trybot requires the following "
1573 "binaries which couldn't be found in $PATH: %s"
Brian Harring351ce442012-03-09 16:38:14 -08001574 % (', '.join(missing)))
1575
David Jamesdac7a912013-11-18 11:14:44 -08001576 if options.reference_repo:
1577 options.reference_repo = os.path.abspath(options.reference_repo)
1578
Brian Harring3fec5a82012-03-01 05:57:03 -08001579 if not options.buildroot:
1580 if options.buildbot:
Mike Frysinger6903c762012-12-04 01:57:16 -05001581 parser.error('Please specify a buildroot with the --buildbot option.')
Matt Tennantd55b1f42012-04-13 14:15:01 -07001582
Ryan Cui5ba7e152012-05-10 14:36:52 -07001583 options.buildroot = _DetermineDefaultBuildRoot(options.sourceroot,
1584 build_config['internal'])
Brian Harring470f6112012-03-02 11:47:10 -08001585 # We use a marker file in the buildroot to indicate the user has
1586 # consented to using this directory.
1587 if not os.path.exists(repository.GetTrybotMarkerPath(options.buildroot)):
1588 _ConfirmBuildRoot(options.buildroot)
Brian Harring3fec5a82012-03-01 05:57:03 -08001589
1590 # Sanity check of buildroot- specifically that it's not pointing into the
1591 # midst of an existing repo since git-repo doesn't support nesting.
Brian Harring3fec5a82012-03-01 05:57:03 -08001592 if (not repository.IsARepoRoot(options.buildroot) and
David James13a69c92013-05-09 10:37:42 -07001593 git.FindRepoDir(options.buildroot)):
Brian Harring3fec5a82012-03-01 05:57:03 -08001594 parser.error('Configured buildroot %s points into a repository checkout, '
1595 'rather than the root of it. This is not supported.'
1596 % options.buildroot)
1597
Chris Sosab5ea3b42012-10-25 15:25:20 -07001598 if not options.log_dir:
1599 options.log_dir = os.path.join(options.buildroot, _DEFAULT_LOG_DIR)
1600
Brian Harringd166aaf2012-05-14 18:31:53 -07001601 log_file = None
1602 if options.tee:
Chris Sosab5ea3b42012-10-25 15:25:20 -07001603 log_file = os.path.join(options.log_dir, _BUILDBOT_LOG_FILE)
1604 osutils.SafeMakedirs(options.log_dir)
Brian Harringd166aaf2012-05-14 18:31:53 -07001605 _BackupPreviousLog(log_file)
1606
Brian Harring1b8c4c82012-05-29 23:03:04 -07001607 with cros_build_lib.ContextManagerStack() as stack:
Aaron Gable881fccb2013-09-27 18:35:24 -07001608 # TODO(ferringb): update this once
1609 # https://chromium-review.googlesource.com/25359
David Jamescebc7272013-07-17 16:45:05 -07001610 # is landed- it's sensitive to the manifest-versions cache path.
1611 options.preserve_paths = set(['manifest-versions', '.cache',
1612 'manifest-versions-internal'])
1613 if log_file is not None:
1614 # We don't want the critical section to try to clean up the tee process,
1615 # so we run Tee (forked off) outside of it. This prevents a deadlock
1616 # because the Tee process only exits when its pipe is closed, and the
1617 # critical section accidentally holds on to that file handle.
1618 stack.Add(tee.Tee, log_file)
1619 options.preserve_paths.add(_DEFAULT_LOG_DIR)
1620
Brian Harringc2d09d92012-05-13 22:03:15 -07001621 critical_section = stack.Add(cleanup.EnforcedCleanupSection)
1622 stack.Add(sudo.SudoKeepAlive)
Brian Harringd166aaf2012-05-14 18:31:53 -07001623
Brian Harringc2d09d92012-05-13 22:03:15 -07001624 if not options.resume:
Brian Harring2bf55e12012-05-13 21:31:55 -07001625 # If we're in resume mode, use our parents tempdir rather than
1626 # nesting another layer.
David James4bc13702013-03-26 08:08:04 -07001627 stack.Add(osutils.TempDir, prefix='cbuildbot-tmp', set_global=True)
Brian Harringc2d09d92012-05-13 22:03:15 -07001628 logging.debug("Cbuildbot tempdir is %r.", os.environ.get('TMP'))
Brian Harringd166aaf2012-05-14 18:31:53 -07001629
Brian Harringc2d09d92012-05-13 22:03:15 -07001630 if options.cgroups:
1631 stack.Add(cgroups.SimpleContainChildren, 'cbuildbot')
Brian Harringa184efa2012-03-04 11:51:25 -08001632
Brian Harringc2d09d92012-05-13 22:03:15 -07001633 # Mark everything between EnforcedCleanupSection and here as having to
1634 # be rolled back via the contextmanager cleanup handlers. This
1635 # ensures that sudo bits cannot outlive cbuildbot, that anything
1636 # cgroups would kill gets killed, etc.
David Jamesfb3aac92013-10-16 13:26:52 -07001637 stack.Add(critical_section.ForkWatchdog)
Brian Harringd166aaf2012-05-14 18:31:53 -07001638
Brian Harringc2d09d92012-05-13 22:03:15 -07001639 if options.timeout > 0:
David James3432acd2013-11-27 10:02:18 -08001640 stack.Add(timeout_util.FatalTimeout, options.timeout)
Brian Harringa184efa2012-03-04 11:51:25 -08001641
Brian Harringc2d09d92012-05-13 22:03:15 -07001642 if not options.buildbot:
1643 build_config = cbuildbot_config.OverrideConfigForTrybot(
1644 build_config,
David Jamese8dcfc42013-03-21 19:49:49 -07001645 options)
Brian Harringc2d09d92012-05-13 22:03:15 -07001646
Mike Frysinger6903c762012-12-04 01:57:16 -05001647 if options.buildbot or options.remote_trybot:
1648 _DisableYamaHardLinkChecks()
1649
Aviv Kesheta0159be2013-12-12 13:56:28 -08001650 if options.mock_tree_status is not None:
1651 stack.Add(mock.patch.object, timeout_util, '_GetStatus',
1652 return_value=options.mock_tree_status)
1653
Brian Harringc2d09d92012-05-13 22:03:15 -07001654 _RunBuildStagesWrapper(options, build_config)