Re-execute in standalone patched chromite repo.
To more thoroughly test chromite patches, including clean, sync, and
patch stages.
BUG=chromium-os:30710
TEST=ongoing
Change-Id: Ie81701379ad562ae6109e9f5fc51ca021ce0b4dd
Reviewed-on: https://gerrit.chromium.org/gerrit/22508
Reviewed-by: David James <davidjames@chromium.org>
Commit-Ready: Ryan Cui <rcui@chromium.org>
Tested-by: Ryan Cui <rcui@chromium.org>
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index 23bad0e..b19bfe1 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -46,7 +46,6 @@
_BUILDBOT_LOG_FILE = 'cbuildbot.log'
_DEFAULT_EXT_BUILDROOT = 'trybot'
_DEFAULT_INT_BUILDROOT = 'trybot-internal'
-_PATH_TO_CBUILDBOT = os.path.join(constants.CHROMITE_BIN_SUBDIR, 'cbuildbot')
_DISTRIBUTED_TYPES = [constants.COMMIT_QUEUE_TYPE, constants.PFQ_TYPE,
constants.CANARY_TYPE, constants.CHROME_PFQ_TYPE,
constants.PALADIN_TYPE]
@@ -121,6 +120,9 @@
Args:
options: The options object generated by optparse.
+ Returns:
+ trybot_patch_pool.TrybotPatchPool object.
+
Raises:
gerrit_helper.GerritException, cros_patch.PatchException
"""
@@ -164,16 +166,12 @@
Vars:
build_config: The configuration dictionary from cbuildbot_config.
options: The options provided from optparse in main().
- completed_stages_file: Where we store resume state.
archive_url: Where our artifacts for this builder will be archived.
tracking_branch: The tracking branch for this build.
release_tag: The associated "chrome os version" of this build.
- gerrit_patches: Gerrit patches to be included in build.
- local_patches: Local patches to be included in build.
- remote_patches: Uploaded local patches to be included in build.
"""
- def __init__(self, options, build_config):
+ def __init__(self, options, build_config, target_manifest_branch):
"""Initializes instance variables. Must be called by all subclasses."""
self.build_config = build_config
self.options = options
@@ -182,27 +180,18 @@
if self.build_config['chromeos_official']:
os.environ['CHROMEOS_OFFICIAL'] = '1'
- self.completed_stages_file = os.path.join(options.buildroot,
- '.completed_stages')
self.archive_stages = {}
self.archive_urls = {}
self.release_tag = None
- self.target_manifest_branch = _GetChromiteTrackingBranch()
+ self.target_manifest_branch = target_manifest_branch
self.patch_pool = trybot_patch_pool.GetEmptyPool()
+ bs.BuilderStage.SetManifestBranch(target_manifest_branch)
+
def Initialize(self):
"""Runs through the initialization steps of an actual build."""
- if self.options.resume and os.path.exists(self.completed_stages_file):
- with open(self.completed_stages_file, 'r') as load_file:
- results_lib.Results.RestoreCompletedStages(load_file)
-
- # We only want to do this if we need to patch changes.
- if not results_lib.Results.GetPrevious().get(
- stages.PatchChangesStage.StageNamePrefix()):
- self.patch_pool = AcquirePoolFromOptions(self.options,
- self.target_manifest_branch)
-
- bs.BuilderStage.SetManifestBranch(self.target_manifest_branch)
+ if self.options.resume:
+ results_lib.LoadCheckpoint(self.options.buildroot)
# Check branch matching early.
if _IsIncrementalBuild(self.options.buildroot, self.options.clobber):
@@ -244,11 +233,6 @@
"""Subclasses must override this method. Runs the appropriate code."""
raise NotImplementedError()
- def _WriteCheckpoint(self):
- """Drops a completed stages file with current state."""
- with open(self.completed_stages_file, 'w+') as save_file:
- results_lib.Results.SaveCompletedStages(save_file)
-
def _ShouldReExecuteInBuildRoot(self):
"""Returns True if this build should be re-executed in the buildroot."""
abs_buildroot = os.path.abspath(self.options.buildroot)
@@ -267,9 +251,8 @@
Returns:
True if the Build succeeded.
"""
- # If we are resuming, use last checkpoint.
if not self.options.resume:
- self._WriteCheckpoint()
+ results_lib.WriteCheckpoint(self.options.buildroot)
# Re-write paths to use absolute paths.
# Suppress any timeout options given from the commandline in the
@@ -294,12 +277,49 @@
# when something occurs. It should exit quicker, but the sigterm may
# hit while the system is particularly busy.
return_obj = cros_lib.RunCommand(
- [_PATH_TO_CBUILDBOT] + sys.argv[1:] + args_to_append,
+ [constants.PATH_TO_CBUILDBOT] + sys.argv[1:] + args_to_append,
cwd=self.options.buildroot, error_code_ok=True, kill_timeout=30)
return return_obj.returncode == 0
+ def _InitializeTrybotPatchPool(self):
+ """Generate patch pool from patches specified on the command line.
+
+ Do this only if we need to patch changes later on.
+ """
+ changes_stage = stages.PatchChangesStage.StageNamePrefix()
+ check_func = results_lib.Results.PreviouslyCompletedRecord
+ if not check_func(changes_stage) or self.options.bootstrap:
+ self.patch_pool = AcquirePoolFromOptions(self.options,
+ self.target_manifest_branch)
+
+ def _GetBootstrapStage(self):
+ """Constructs and returns the BootStrapStage object.
+
+ We return None when there are no chromite patches to test, and
+ --test-bootstrap wasn't passed in.
+ """
+ stage = None
+ chromite_pool = self.patch_pool.Filter(project=constants.CHROMITE_PROJECT)
+ if chromite_pool or self.options.test_bootstrap:
+ stage = stages.BootstrapStage(self.options, self.build_config,
+ chromite_pool)
+ return stage
+
def Run(self):
- """Main runner for this builder class. Runs build and prints summary."""
+ """Main runner for this builder class. Runs build and prints summary.
+
+ Returns:
+ Whether the build succeeded.
+ """
+ self._InitializeTrybotPatchPool()
+
+ if self.options.bootstrap:
+ bootstrap_stage = self._GetBootstrapStage()
+ if bootstrap_stage:
+ # BootstrapStage blocks on re-execution of cbuildbot.
+ bootstrap_stage.Run()
+ return bootstrap_stage.returncode == 0
+
print_report = True
exception_thrown = False
success = True
@@ -322,7 +342,7 @@
raise
finally:
if print_report:
- self._WriteCheckpoint()
+ results_lib.WriteCheckpoint(self.options.buildroot)
print '\n\n\n@@@BUILD_STEP Report@@@\n'
results_lib.Results.Report(sys.stdout, self.archive_urls,
self.release_tag)
@@ -414,14 +434,14 @@
These builds sync using git/manifest logic in manifest_versions. In general
they use a non-distributed builder code for the bulk of the work.
"""
- def __init__(self, options, build_config):
+ def __init__(self, *args, **kwargs):
"""Initializes a buildbot builder.
Extra variables:
completion_stage_class: Stage used to complete a build. Set in the Sync
stage.
"""
- super(DistributedBuilder, self).__init__(options, build_config)
+ super(DistributedBuilder, self).__init__(*args, **kwargs)
self.completion_stage_class = None
def GetSyncInstance(self):
@@ -541,7 +561,6 @@
os.rename(log_file, log_file + '.' + str(last + 1))
-
def _RunBuildStagesWrapper(options, build_config):
"""Helper function that wraps RunBuildStages()."""
def IsDistributedBuilder():
@@ -562,11 +581,11 @@
cros_lib.Info("cbuildbot executed with args %s"
% ' '.join(map(repr, sys.argv)))
- if IsDistributedBuilder():
- buildbot = DistributedBuilder(options, build_config)
- else:
- buildbot = SimpleBuilder(options, build_config)
+ target_manifest_branch = _GetChromiteTrackingBranch()
+
+ target = DistributedBuilder if IsDistributedBuilder() else SimpleBuilder
+ buildbot = target(options, build_config, target_manifest_branch)
if not buildbot.Run():
sys.exit(1)
@@ -657,6 +676,7 @@
**kwargs)
+# pylint: disable=W0613
def check_path(option, opt, value):
"""Expand paths and make them absolute."""
expanded = osutils.ExpandPath(value)
@@ -771,19 +791,23 @@
help="Change the local saved build count limit.")
group.add_remote_option('--noarchive', action='store_false', dest='archive',
default=True, help="Don't run archive stage.")
+ group.add_remote_option('--nobootstrap', action='store_false',
+ dest='bootstrap', default=True,
+ help="Don't checkout and run from a standalone "
+ "chromite repo.")
group.add_remote_option('--nobuild', action='store_false', dest='build',
default=True,
help="Don't actually build (for cbuildbot dev)")
group.add_remote_option('--noclean', action='store_false', dest='clean',
default=True, help="Don't clean the buildroot")
+ group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
+ default=True,
+ help='Disable cbuildbots usage of cgroups.')
group.add_remote_option('--noprebuilts', action='store_false',
dest='prebuilts', default=True,
help="Don't upload prebuilts.")
group.add_remote_option('--nosync', action='store_false', dest='sync',
default=True, help="Don't sync before building.")
- group.add_remote_option('--nocgroups', action='store_false', dest='cgroups',
- default=True,
- help='Disable cbuildbots usage of cgroups.')
group.add_remote_option('--notests', action='store_false', dest='tests',
default=True,
help='Override values from buildconfig and run no '
@@ -817,6 +841,10 @@
'timeout.')
group.add_option('--sourceroot', type='path', default=constants.SOURCE_ROOT,
help=optparse.SUPPRESS_HELP)
+ # Causes cbuildbot to bootstrap itself twice, in the sequence A->B->C.
+ # A(unpatched) patches and bootstraps B. B patches and bootstraps C.
+ group.add_remote_option('--test-bootstrap', action='store_true',
+ default=False, help=optparse.SUPPRESS_HELP)
group.add_option('--test-tryjob', action='store_true',
default=False,
help='Submit a tryjob to the test repository. Will not '
@@ -940,9 +968,6 @@
Args:
options/args: The options/args object returned by optparse
"""
- if options.resume:
- return
-
if options.local_patches and not repository.IsARepoRoot(options.sourceroot):
raise Exception('Could not find repo checkout at %s!'
% options.sourceroot)