toolchain: prepare_build support.
Add toolchain_util support for new Build API Toolchain Service
endpoints.
Call PrepareForBuild from the API controller, and begin adding calls to
BundleArtifacts.
Sufficient support for orderfile-generate and orderfile-verify is here.
The other toolchain artifact builders remain unsupported.
BUG=chromium:1019868
TEST=unit tests pass
Change-Id: I487a5b2a04e2a6e2094aa4b967de8ef5e0224e6a
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1981840
Reviewed-by: LaMont Jones <lamontjones@chromium.org>
Commit-Queue: LaMont Jones <lamontjones@chromium.org>
Tested-by: LaMont Jones <lamontjones@chromium.org>
diff --git a/api/controller/toolchain.py b/api/controller/toolchain.py
index 9730aa2..64af9e9 100644
--- a/api/controller/toolchain.py
+++ b/api/controller/toolchain.py
@@ -9,64 +9,98 @@
import collections
+from chromite.api import controller
from chromite.api import faux
from chromite.api import validate
+from chromite.api.controller import controller_util
from chromite.api.gen.chromite.api import toolchain_pb2
-from chromite.lib import toolchain_util
from chromite.api.gen.chromiumos.builder_config_pb2 import BuilderConfig
+from chromite.lib import cros_logging as logging
+from chromite.lib import toolchain_util
# TODO(crbug/1019868): Add handlers as needed.
_Handlers = collections.namedtuple('_Handlers', ['name', 'prepare', 'bundle'])
_TOOLCHAIN_ARTIFACT_HANDLERS = {
BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE:
- _Handlers('UnverifiedOrderingFile', None, None),
+ _Handlers('UnverifiedOrderingFile',
+ toolchain_util.PrepareForBuild,
+ toolchain_util.BundleArtifacts),
BuilderConfig.Artifacts.VERIFIED_ORDERING_FILE:
- _Handlers('VerifiedOrderingFile', None, None),
+ _Handlers('VerifiedOrderingFile',
+ toolchain_util.PrepareForBuild,
+ toolchain_util.BundleArtifacts),
BuilderConfig.Artifacts.CHROME_CLANG_WARNINGS_FILE:
- _Handlers('ChromeClangWarningsFile', None, None),
+ _Handlers('ChromeClangWarningsFile',
+ toolchain_util.PrepareForBuild, None),
BuilderConfig.Artifacts.UNVERIFIED_LLVM_PGO_FILE:
- _Handlers('UnverifiedLlvmPgoFile', None, None),
+ _Handlers('UnverifiedLlvmPgoFile',
+ toolchain_util.PrepareForBuild, None),
BuilderConfig.Artifacts.UNVERIFIED_CHROME_AFDO_FILE:
- _Handlers('UnverifiedChromeAfdoFile', None, None),
+ _Handlers('UnverifiedChromeAfdoFile',
+ toolchain_util.PrepareForBuild, None),
BuilderConfig.Artifacts.VERIFIED_CHROME_AFDO_FILE:
- _Handlers('VerifiedChromeAfdoFile', None, None),
+ _Handlers('VerifiedChromeAfdoFile',
+ toolchain_util.PrepareForBuild, None),
BuilderConfig.Artifacts.VERIFIED_KERNEL_AFDO_FILE:
- _Handlers('VerifiedKernelAfdoFile', None, None),
+ _Handlers('VerifiedKernelAfdoFile',
+ toolchain_util.PrepareForBuild, None),
}
# TODO(crbug/1031213): When @faux is expanded to have more than success/failure,
# this should be changed.
@faux.all_empty
-@validate.require('chroot.path', 'sysroot.path', 'sysroot.build_target.name',
- 'artifact_types')
+@validate.require('artifact_types')
+# Note: chroot and sysroot are unspecified the first time that the build_target
+# recipe calls PrepareForBuild. The second time, they are specified. No
+# validation check because "all" values are valid.
@validate.validation_complete
def PrepareForBuild(input_proto, output_proto, _config):
"""Prepare to build toolchain artifacts.
The handlers (from _TOOLCHAIN_ARTIFACT_HANDLERS above) are called with:
artifact_name (str): name of the artifact type.
- chroot_path (str): chroot path (absolute path)
- sysroot_path (str): sysroot path inside the chroot (e.g., /build/atlas)
- build_target_name (str): name of the build target (e.g., atlas)
+ chroot (chroot_lib.Chroot): chroot. Will be None if the chroot has not
+ yet been created.
+ sysroot_path (str): sysroot path inside the chroot (e.g., /build/atlas).
+ Will be an empty string if the sysroot has not yet been created.
+ build_target_name (str): name of the build target (e.g., atlas). Will be
+ an empty string if the sysroot has not yet been created.
+ input_artifacts ({(str) name:[str gs_locations]}): locations for possible
+ input artifacts. The handler is expected to know which keys it should
+ be using, and ignore any keys that it does not understand.
They locate and modify any ebuilds and/or source required for the artifact
being created, then return a value from toolchain_util.PrepareForBuildReturn.
+ This function sets output_proto.build_relevance to the result.
+
Args:
input_proto (PrepareForToolchainBuildRequest): The input proto
output_proto (PrepareForToolchainBuildResponse): The output proto
_config (api_config.ApiConfig): The API call config.
"""
- results = set()
+ if input_proto.chroot.path:
+ chroot = controller_util.ParseChroot(input_proto.chroot)
+ else:
+ chroot = None
+ input_artifacts = collections.defaultdict(list)
+ for art in input_proto.input_artifacts:
+ item = _TOOLCHAIN_ARTIFACT_HANDLERS.get(art.input_artifact_type)
+ if item:
+ input_artifacts[item.name].extend(
+ ['gs://%s' % str(x) for x in art.input_artifact_gs_locations])
+
+ results = set()
+ sysroot_path = input_proto.sysroot.path
+ build_target = input_proto.sysroot.build_target.name
for artifact_type in input_proto.artifact_types:
- # Ignore any artifact_types not handled.
- handler = _TOOLCHAIN_ARTIFACT_HANDLERS.get(artifact_type)
- if handler and handler.prepare:
+ # Unknown artifact_types are an error.
+ handler = _TOOLCHAIN_ARTIFACT_HANDLERS[artifact_type]
+ if handler.prepare:
results.add(handler.prepare(
- handler.name, input_proto.chroot.path, input_proto.sysroot.path,
- input_proto.sysroot.build_target.name))
+ handler.name, chroot, sysroot_path, build_target, input_artifacts))
# Translate the returns from the handlers we called.
# If any NEEDED => NEEDED
@@ -82,6 +116,7 @@
output_proto.build_relevance = proto_resp.POINTLESS
else:
output_proto.build_relevance = proto_resp.UNKNOWN
+ return controller.RETURN_CODE_SUCCESS
# TODO(crbug/1031213): When @faux is expanded to have more than success/failure,
@@ -90,13 +125,15 @@
@validate.require('chroot.path', 'sysroot.path', 'sysroot.build_target.name',
'output_dir', 'artifact_types')
@validate.exists('output_dir')
+@validate.validation_complete
def BundleArtifacts(input_proto, output_proto, _config):
"""Bundle toolchain artifacts.
The handlers (from _TOOLCHAIN_ARTIFACT_HANDLERS above) are called with:
artifact_name (str): name of the artifact type
- chroot_path (str): chroot path (absolute path)
+ chroot (chroot_lib.Chroot): chroot
sysroot_path (str): sysroot path inside the chroot (e.g., /build/atlas)
+ chrome_root (str): path to chrome root. (e.g., /b/s/w/ir/k/chrome)
build_target_name (str): name of the build target (e.g., atlas)
output_dir (str): absolute path where artifacts are being bundled.
(e.g., /b/s/w/ir/k/recipe_cleanup/artifactssptfMU)
@@ -108,18 +145,22 @@
output_proto (BundleToolchainResponse): The output proto
_config (api_config.ApiConfig): The API call config.
"""
- resp_artifact = toolchain_pb2.ArtifactInfo
+ chroot = controller_util.ParseChroot(input_proto.chroot)
for artifact_type in input_proto.artifact_types:
- # Ignore any artifact_types not handled.
- handler = _TOOLCHAIN_ARTIFACT_HANDLERS.get(artifact_type)
+ if artifact_type not in _TOOLCHAIN_ARTIFACT_HANDLERS:
+ logging.error('%s not understood', artifact_type)
+ return controller.RETURN_CODE_UNRECOVERABLE
+ handler = _TOOLCHAIN_ARTIFACT_HANDLERS[artifact_type]
if handler and handler.bundle:
artifacts = handler.bundle(
- handler.name, input_proto.chroot.path, input_proto.sysroot.path,
+ handler.name, chroot, input_proto.sysroot.path,
input_proto.sysroot.build_target.name, input_proto.output_dir)
if artifacts:
- output_proto.artifacts_info.append(
- resp_artifact(artifact_type=artifact_type, artifacts=artifacts))
+ art_info = output_proto.artifacts_info.add()
+ art_info.artifact_type = artifact_type
+ for artifact in artifacts:
+ art_info.artifacts.add().path = artifact
# TODO(crbug/1019868): Remove legacy code when cbuildbot builders are gone.
diff --git a/api/controller/toolchain_unittest.py b/api/controller/toolchain_unittest.py
index 675b646..6e51625 100644
--- a/api/controller/toolchain_unittest.py
+++ b/api/controller/toolchain_unittest.py
@@ -8,7 +8,9 @@
from __future__ import print_function
from chromite.api import api_config
+from chromite.api import controller
from chromite.api.controller import toolchain
+from chromite.api.gen.chromite.api import artifacts_pb2
from chromite.api.gen.chromite.api import sysroot_pb2
from chromite.api.gen.chromite.api import toolchain_pb2
from chromite.api.gen.chromiumos.builder_config_pb2 import BuilderConfig
@@ -18,6 +20,7 @@
from chromite.lib import cros_test_lib
from chromite.lib import toolchain_util
+# pylint: disable=protected-access
class UpdateEbuildWithAFDOArtifactsTest(cros_test_lib.MockTestCase,
api_config.ApiConfigMixin):
@@ -177,6 +180,16 @@
def setUp(self):
self.response = toolchain_pb2.PrepareForToolchainBuildResponse()
+ self.prep = self.PatchObject(
+ toolchain_util, 'PrepareForBuild',
+ return_value=toolchain_util.PrepareForBuildReturn.NEEDED)
+ self.bundle = self.PatchObject(
+ toolchain_util, 'BundleArtifacts', return_value=[])
+ self.PatchObject(toolchain, '_TOOLCHAIN_ARTIFACT_HANDLERS', {
+ BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE:
+ toolchain._Handlers('UnverifiedOrderingFile',
+ self.prep, self.bundle),
+ })
def _GetRequest(self, artifact_types=None):
if artifact_types is None:
@@ -189,11 +202,50 @@
chroot=chroot, sysroot=sysroot,
)
- def testReturnsUnknownForUnknown(self):
+ def testRaisesForUnknown(self):
request = self._GetRequest([BuilderConfig.Artifacts.IMAGE_ARCHIVES])
+ self.assertRaises(
+ KeyError,
+ toolchain.PrepareForBuild, request, self.response, self.api_config)
+
+ def testAcceptsNone(self):
+ request = toolchain_pb2.PrepareForToolchainBuildRequest(
+ artifact_types=[BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE],
+ chroot=None, sysroot=None)
toolchain.PrepareForBuild(request, self.response, self.api_config)
- self.assertEqual(toolchain_pb2.PrepareForToolchainBuildResponse.UNKNOWN,
- self.response.build_relevance)
+ self.prep.assert_called_once_with(
+ 'UnverifiedOrderingFile', None, '', '', {})
+
+ def testHandlesUnknownInputArtifacts(self):
+ request = toolchain_pb2.PrepareForToolchainBuildRequest(
+ artifact_types=[BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE],
+ chroot=None, sysroot=None, input_artifacts=[
+ BuilderConfig.Artifacts.InputArtifactInfo(
+ input_artifact_type=BuilderConfig.Artifacts.IMAGE_ZIP,
+ input_artifact_gs_locations=['path1']),
+ ])
+ toolchain.PrepareForBuild(request, self.response, self.api_config)
+ self.prep.assert_called_once_with(
+ 'UnverifiedOrderingFile', None, '', '', {})
+
+ def testHandlesDuplicateInputArtifacts(self):
+ request = toolchain_pb2.PrepareForToolchainBuildRequest(
+ artifact_types=[BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE],
+ chroot=None, sysroot=None, input_artifacts=[
+ BuilderConfig.Artifacts.InputArtifactInfo(
+ input_artifact_type=\
+ BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE,
+ input_artifact_gs_locations=['path1', 'path2']),
+ BuilderConfig.Artifacts.InputArtifactInfo(
+ input_artifact_type=\
+ BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE,
+ input_artifact_gs_locations=['path3']),
+ ])
+ toolchain.PrepareForBuild(request, self.response, self.api_config)
+ self.prep.assert_called_once_with(
+ 'UnverifiedOrderingFile', None, '', '', {
+ 'UnverifiedOrderingFile': [
+ 'gs://path1', 'gs://path2', 'gs://path3']})
class BundleToolchainTest(cros_test_lib.MockTempDirTestCase,
@@ -202,6 +254,16 @@
def setUp(self):
self.response = toolchain_pb2.BundleToolchainResponse()
+ self.prep = self.PatchObject(
+ toolchain_util, 'PrepareForBuild',
+ return_value=toolchain_util.PrepareForBuildReturn.NEEDED)
+ self.bundle = self.PatchObject(
+ toolchain_util, 'BundleArtifacts', return_value=[])
+ self.PatchObject(toolchain, '_TOOLCHAIN_ARTIFACT_HANDLERS', {
+ BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE:
+ toolchain._Handlers('UnverifiedOrderingFile',
+ self.prep, self.bundle),
+ })
def _GetRequest(self, artifact_types=None):
chroot = common_pb2.Chroot(path=self.tempdir)
@@ -213,15 +275,29 @@
artifact_types=artifact_types,
)
- def testReturnsUnknownForUnknown(self):
+ def testRaisesForUnknown(self):
request = self._GetRequest([BuilderConfig.Artifacts.IMAGE_ARCHIVES])
- toolchain.BundleArtifacts(request, self.response, self.api_config)
- self.assertEqual([], list(self.response.artifacts_info))
+ self.assertEqual(
+ controller.RETURN_CODE_UNRECOVERABLE,
+ toolchain.BundleArtifacts(request, self.response, self.api_config))
def testValidateOnly(self):
"""Sanity check that a validate only call does not execute any logic."""
- patch = self.PatchObject(toolchain_util, 'BundleArtifacts')
- request = self._GetRequest([BuilderConfig.Artifacts.IMAGE_ARCHIVES])
+ request = self._GetRequest(
+ [BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE])
toolchain.BundleArtifacts(request, self.response,
self.validate_only_config)
- patch.assert_not_called()
+ self.bundle.assert_not_called()
+
+ def testSetsArtifactsInfo(self):
+ request = self._GetRequest(
+ [BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE])
+ self.bundle.return_value = ['artifact.xz']
+ toolchain.BundleArtifacts(request, self.response, self.api_config)
+ self.assertEqual(1, len(self.response.artifacts_info))
+ self.assertEqual(
+ self.response.artifacts_info[0],
+ toolchain_pb2.ArtifactInfo(
+ artifact_type=BuilderConfig.Artifacts.UNVERIFIED_ORDERING_FILE,
+ artifacts=[
+ artifacts_pb2.Artifact(path=self.bundle.return_value[0])]))