Chromite changes to support AFDO.
- Changes to support AFDO in the lumpy PFQ and the master PFQ (x86-generic).
- Enabled the support in the PFQ builder to generate the AFDO profile.
- Added alert in case of failures during the AFDO profile generation.
- Add support for passing USE=afdo_use to the canary builders. This is
NOT enabled yet.
- Changes to keep the support for a single builder that does profile
collection and optimization in a single builder.
At this point only the profile collection is enable. Optimized builds
are not enabled yet.
BUG=None
TEST=Used a modified builder that does 2 builds:
1) Does a regular build. Collects the 'perf' data using the
AFDO_Record autotest and calls GenerateOrFindAFDOData.
This is the same that will go into the PFQ builder for deployment.
2) Does an optimized build using the AFDO file specified in the
ebuild. This is the same that will go in to the canary builder
for deployment.
Having a single builder greatly simplifies testing.
Also did "white box" testing by calling directly the routines
that were added.
Tested PFQ builder Master and slave (lumpy-chromium-pfq and
x86-generic-chromium-pfq).
Tested that these builders were unaffected: master-paladin,
x86-generic-paladin, falco-chrome-pfq.
And ran run_tests.
Change-Id: I095dfaa874f3ffbe898fcb23f0139a226d7d6a21
Reviewed-on: https://chromium-review.googlesource.com/199250
Reviewed-by: David James <davidjames@chromium.org>
Commit-Queue: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
diff --git a/scripts/cbuildbot.py b/scripts/cbuildbot.py
index 2a91ef0..91fc900 100644
--- a/scripts/cbuildbot.py
+++ b/scripts/cbuildbot.py
@@ -15,24 +15,26 @@
import glob
import json
import logging
+import multiprocessing
import optparse
import os
-import multiprocessing
import pickle
import sys
import tempfile
import traceback
+from chromite.cbuildbot import afdo
from chromite.cbuildbot import cbuildbot_config
-from chromite.cbuildbot import failures_lib
-from chromite.cbuildbot import results_lib
from chromite.cbuildbot import cbuildbot_run
from chromite.cbuildbot import constants
+from chromite.cbuildbot import failures_lib
from chromite.cbuildbot import manifest_version
from chromite.cbuildbot import remote_try
from chromite.cbuildbot import repository
+from chromite.cbuildbot import results_lib
from chromite.cbuildbot import tee
from chromite.cbuildbot import trybot_patch_pool
+from chromite.cbuildbot.stages import afdo_stages
from chromite.cbuildbot.stages import artifact_stages
from chromite.cbuildbot.stages import branch_stages
from chromite.cbuildbot.stages import build_stages
@@ -486,7 +488,7 @@
# kill it once we migrate its uses to BuilderRun so that none of the
# stages below need it as an argument.
archive_stage = self.archive_stages[BoardConfig(board, config.name)]
- if config.pgo_generate:
+ if config.afdo_generate_min:
self._RunParallelStages([archive_stage])
return
@@ -506,7 +508,7 @@
# TODO(davidjames): Remove this lock once http://crbug.com/352994 is fixed.
with self._build_image_lock:
self._RunStage(build_stages.BuildImageStage, board,
- builder_run=builder_run, pgo_use=config.pgo_use)
+ builder_run=builder_run, afdo_use=config.afdo_use)
# While this stage list is run in parallel, the order here dictates the
# order that things will be shown in the log. So group things together
@@ -527,6 +529,9 @@
stage_list += [[generic_stages.RetryStage, 1, test_stages.VMTestStage,
board]]
+ if config.afdo_generate:
+ stage_list += [[afdo_stages.AFDODataGenerateStage, board]]
+
stage_list += [
[release_stages.SignerTestStage, board, archive_stage],
[generic_stages.RetryStage, 1,
@@ -621,24 +626,33 @@
with parallel.BackgroundTaskRunner(task_runner) as queue:
for builder_run, board in tasks:
if not builder_run.config.build_packages_in_background:
- # Run BuildPackages in the foreground, generating or using PGO data
+ # Run BuildPackages in the foreground, generating or using AFDO data
# if requested.
kwargs = {'builder_run': builder_run}
- if builder_run.config.pgo_generate:
- kwargs['pgo_generate'] = True
- elif builder_run.config.pgo_use:
- kwargs['pgo_use'] = True
+ if builder_run.config.afdo_generate_min:
+ kwargs['afdo_generate_min'] = True
+ elif builder_run.config.afdo_use:
+ kwargs['afdo_use'] = True
self._RunStage(build_stages.BuildPackagesStage, board, **kwargs)
- if builder_run.config.pgo_generate:
- # Generate the PGO data before allowing any other tasks to run.
+ if (builder_run.config.afdo_generate_min and
+ afdo.CanGenerateAFDOData(board)):
+ # Generate the AFDO data before allowing any other tasks to run.
self._RunStage(build_stages.BuildImageStage, board, **kwargs)
self._RunStage(artifact_stages.UploadTestArtifactsStage, board,
- builder_run=builder_run, suffix='[pgo_generate]')
- suite = cbuildbot_config.PGORecordTest()
+ builder_run=builder_run,
+ suffix='[afdo_generate_min]')
+ suite = cbuildbot_config.AFDORecordTest()
self._RunStage(test_stages.HWTestStage, board, suite,
builder_run=builder_run)
+ self._RunStage(afdo_stages.AFDODataGenerateStage, board,
+ builder_run=builder_run)
+
+ if (builder_run.config.afdo_generate_min and
+ builder_run.config.afdo_update_ebuild):
+ self._RunStage(afdo_stages.AFDOUpdateEbuildStage,
+ builder_run=builder_run)
# Kick off our background stages.
queue.put([builder_run, board])
@@ -733,6 +747,9 @@
try:
completion_stage.Run()
completion_successful = True
+ if (self._run.config.afdo_update_ebuild and
+ not self._run.config.afdo_generate_min):
+ self._RunStage(afdo_stages.AFDOUpdateEbuildStage)
finally:
if not completion_successful:
was_build_successful = False