Run BuildPackages in parallel for grouped builders.
This CL re-lands CL:189999, but only runs BuildPackages in parallel.
We add a lock around BuildImage to ensure that only one image is built
at any given time.
This CL updates our grouped builders to run BuildPackages in parallel,
as much as possible.
Here's how it works:
- The first builder runs BuildPackages in the foreground, to do
a complete build from source.
- Subsequent builders run BuildPackages in the background, using
the binary packages generated by the first board.
To control which builders run in the foreground and the background,
I have added the "build_packages_in_background" config setting. This
is set to False for the first builder in a release-group and to True
for the remaining builders. On the other hand, for the pre-cq, it is
always set to True because the builders don't reuse each other's
binary packages.
BUG=chromium:352148, chromium:348226
TEST=Run sandybridge-release-group, confirm runtime is reduced.
Change-Id: I83c70ceaa5b96522e0b6d9a7a02f3a1f47394a83
Previous-Change-Id: I3f58bd1a75aded35ecc43fd7cbb31fe9687e5050
Previously-Reviewed-on: https://chromium-review.googlesource.com/189999
Reviewed-on: https://chromium-review.googlesource.com/190360
Reviewed-by: Don Garrett <dgarrett@chromium.org>
Commit-Queue: David James <davidjames@chromium.org>
Tested-by: David James <davidjames@chromium.org>
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index 341766b..6338ea4 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -17,6 +17,7 @@
import logging
import optparse
import os
+import multiprocessing
import pickle
import sys
import time
@@ -147,6 +148,7 @@
self.archive_stages = {}
self.patch_pool = trybot_patch_pool.TrybotPatchPool()
+ self._build_image_lock = multiprocessing.Lock()
def Initialize(self):
"""Runs through the initialization steps of an actual build."""
@@ -414,13 +416,12 @@
raise
- def _RunBackgroundStagesForBoard(self, builder_run, board, compilecheck):
+ def _RunBackgroundStagesForBoard(self, builder_run, board):
"""Run background board-specific stages for the specified board.
Args:
builder_run: BuilderRun object for these background stages.
board: Board name.
- compilecheck: Boolean. If True, run only the compile steps.
"""
config = builder_run.config
@@ -432,18 +433,26 @@
self._RunParallelStages([archive_stage])
return
- if compilecheck:
+ # signer_results can't complete without push_image.
+ assert not config.signer_results or config.push_image
+
+ # paygen can't complete without signer_results.
+ assert not config.paygen or config.signer_results
+
+ if config.build_packages_in_background:
self._RunStage(stages.BuildPackagesStage, board, archive_stage,
builder_run=builder_run)
+
+ if builder_run.config.compilecheck or builder_run.options.compilecheck:
self._RunStage(stages.UnitTestStage, board,
builder_run=builder_run)
return
- # signer_results can't complete without push_image.
- assert not config['signer_results'] or config['push_image']
-
- # paygen can't complete without signer_results.
- assert not config['paygen'] or config['signer_results']
+ # Build the image first before doing anything else.
+ # TODO(davidjames): Remove this lock once http://crbug.com/352994 is fixed.
+ with self._build_image_lock:
+ self._RunStage(stages.BuildImageStage, board, archive_stage=archive_stage,
+ builder_run=builder_run, pgo_use=config.pgo_use)
# While this stage list is run in parallel, the order here dictates the
# order that things will be shown in the log. So group things together
@@ -547,11 +556,9 @@
task_runner = self._RunBackgroundStagesForBoard
with parallel.BackgroundTaskRunner(task_runner) as queue:
for builder_run, board, archive_stage in tasks:
- compilecheck = (builder_run.config.compilecheck or
- builder_run.options.compilecheck)
- if not compilecheck:
- # Run BuildPackages and BuildImage in the foreground, generating
- # or using PGO data if requested.
+ if not builder_run.config.build_packages_in_background:
+ # Run BuildPackages in the foreground, generating or using PGO data
+ # if requested.
kwargs = {'archive_stage': archive_stage, 'builder_run': builder_run}
if builder_run.config.pgo_generate:
kwargs['pgo_generate'] = True
@@ -559,15 +566,16 @@
kwargs['pgo_use'] = True
self._RunStage(stages.BuildPackagesStage, board, **kwargs)
- self._RunStage(stages.BuildImageStage, board, **kwargs)
if builder_run.config.pgo_generate:
+ # Generate the PGO data before allowing any other tasks to run.
+ self._RunStage(stages.BuildImageStage, board, **kwargs)
suite = cbuildbot_config.PGORecordTest()
self._RunStage(stages.HWTestStage, board, archive_stage, suite,
builder_run=builder_run)
# Kick off our background stages.
- queue.put([builder_run, board, compilecheck])
+ queue.put([builder_run, board])
def RunStages(self):
"""Runs through build process."""