api: rewrite breakpad and add debug symbols
Breakpad symbols were named wrong, referring to DEBUG symbols. We
translate the existing DEBUG to BREAKPAD_DEBUG and we correct them.
In addition we intro a new model for the global Get function to call
the individual handlers. Finally, DEBUG is also implemented.
BUG=b:185593007
TEST=call_scripts && units
Change-Id: I9f080c1261387e74ceb177a9994186883b57a347
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2959731
Tested-by: George Engelbrecht <engeg@google.com>
Reviewed-by: Alex Klein <saklein@chromium.org>
Commit-Queue: George Engelbrecht <engeg@google.com>
diff --git a/api/controller/artifacts.py b/api/controller/artifacts.py
index a7e6bf5..1a62605 100644
--- a/api/controller/artifacts.py
+++ b/api/controller/artifacts.py
@@ -5,11 +5,14 @@
"""Implements ArtifactService."""
import os
+from typing import Any, NamedTuple
from chromite.api import controller
from chromite.api import faux
from chromite.api import validate
from chromite.api.controller import controller_util
+from chromite.api.controller import image as image_controller
+from chromite.api.controller import sysroot as sysroot_controller
from chromite.api.gen.chromite.api import artifacts_pb2
from chromite.api.gen.chromite.api import toolchain_pb2
from chromite.api.gen.chromiumos import common_pb2
@@ -19,17 +22,26 @@
from chromite.lib import cros_logging as logging
from chromite.lib import sysroot_lib
from chromite.service import artifacts
-from chromite.service import image as image_service
-def _GetResponse(_input_proto, _output_proto, _config):
- """Currently bundles nothing."""
- # TODO(crbug/1034529): As methods migrate, begin populating them based on what
- # input_proto has defined.
+class RegisteredGet(NamedTuple):
+ """An registered function for calling Get on an artifact type."""
+ output_proto: artifacts_pb2.GetResponse
+ artifact_dict: Any
-@faux.success(_GetResponse)
+def ExampleGetResponse(_input_proto, _output_proto, _config):
+ """Give an example GetResponse with a minimal coverage set."""
+ _output_proto = artifacts_pb2.GetResponse(
+ artifacts=common_pb2.UploadedArtifactsByService(
+ image=image_controller.ExampleGetResponse(),
+ sysroot=sysroot_controller.ExampleGetResponse(),
+ ))
+ return controller.RETURN_CODE_SUCCESS
+
+
@faux.empty_error
+@faux.success(ExampleGetResponse)
@validate.exists('result_path.path.path')
@validate.validation_complete
def Get(input_proto, output_proto, _config):
@@ -37,33 +49,44 @@
Get all artifacts for the build.
- Note: crbug/1034529 introduces this method as a noop. As the individual
- artifact_type bundlers are added here, they *must* stop uploading it via the
- individual bundler function.
+ Note: As the individual artifact_type bundlers are added here, they *must*
+ stop uploading it via the individual bundler function.
Args:
input_proto (GetRequest): The input proto.
output_proto (GetResponse): The output proto.
_config (api_config.ApiConfig): The API call config.
"""
-
- image_proto = input_proto.artifact_info.image
- base_path = os.path.join(input_proto.chroot.path,
- input_proto.sysroot.path[1:])
output_dir = input_proto.result_path.path.path
- images_list = image_service.Get(image_proto, base_path, output_dir)
+ sysroot = controller_util.ParseSysroot(input_proto.sysroot)
+ chroot = controller_util.ParseChroot(input_proto.chroot)
+ build_target = controller_util.ParseBuildTarget(
+ input_proto.sysroot.build_target)
- for artifact_dict in images_list:
- output_proto.artifacts.image.artifacts.add(
- artifact_type=artifact_dict['type'],
- paths=[
- common_pb2.Path(
- path=x,
- location=common_pb2.Path.Location.OUTSIDE)
- for x in artifact_dict['paths']
- ])
+ # A list of RegisteredGet tuples (input proto, output proto, get results).
+ get_res_list = [
+ RegisteredGet(
+ output_proto.artifacts.image,
+ image_controller.GetArtifacts(
+ input_proto.artifact_info.image, chroot, sysroot, build_target,
+ output_dir)),
+ RegisteredGet(
+ output_proto.artifacts.sysroot,
+ sysroot_controller.GetArtifacts(
+ input_proto.artifact_info.sysroot, chroot, sysroot, build_target,
+ output_dir))
+ ]
+ for get_res in get_res_list:
+ for artifact_dict in get_res.artifact_dict:
+ get_res.output_proto.artifacts.add(
+ artifact_type=artifact_dict['type'],
+ paths=[
+ common_pb2.Path(
+ path=x, location=common_pb2.Path.Location.OUTSIDE)
+ for x in artifact_dict['paths']
+ ])
return controller.RETURN_CODE_SUCCESS
@@ -674,59 +697,3 @@
tarball = artifacts.BundleGceTarball(output_dir, image_dir)
output_proto.artifacts.add().path = tarball
-
-
-def _BundleDebugSymbolsResponse(input_proto, output_proto, _config):
- """Add artifact tarball to a successful response."""
- output_proto.artifacts.add().path = os.path.join(input_proto.output_dir,
- constants.DEBUG_SYMBOLS_TAR)
-
-
-@faux.success(_BundleDebugSymbolsResponse)
-@faux.empty_error
-@validate.require('build_target.name', 'output_dir')
-@validate.exists('output_dir')
-@validate.validation_complete
-def BundleDebugSymbols(input_proto, output_proto, _config):
- """Bundle the debug symbols into a tarball suitable for importing into GCE.
-
- Args:
- input_proto (BundleRequest): The input proto.
- output_proto (BundleResponse): The output proto.
- _config (api_config.ApiConfig): The API call config.
- """
- output_dir = input_proto.output_dir
-
- chroot = controller_util.ParseChroot(input_proto.chroot)
- build_target = controller_util.ParseBuildTarget(input_proto.build_target)
- result = artifacts.GenerateBreakpadSymbols(chroot,
- build_target,
- debug=True)
-
- # Verify breakpad symbol generation before gathering the sym files.
- if result.returncode != 0:
- return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
-
- with chroot.tempdir() as symbol_tmpdir, chroot.tempdir() as dest_tmpdir:
- breakpad_dir = os.path.join(chroot.path, 'build', build_target.name,
- 'usr/lib/debug/breakpad')
- # Call list on the atifacts.GatherSymbolFiles generator function to
- # materialize and consume all entries so that all are copied to
- # dest dir and complete list of all symbol files is returned.
- sym_file_list = list(artifacts.GatherSymbolFiles(tempdir=symbol_tmpdir,
- destdir=dest_tmpdir,
- paths=[breakpad_dir]))
- if not sym_file_list:
- logging.warning('No sym files found in %s.', breakpad_dir)
- # Create tarball from destination_tmp, then copy it...
- tarball_path = os.path.join(output_dir, constants.DEBUG_SYMBOLS_TAR)
- result = cros_build_lib.CreateTarball(tarball_path, dest_tmpdir)
- if result.returncode != 0:
- logging.error('Error (%d) when creating tarball %s from %s',
- result.returncode,
- tarball_path,
- dest_tmpdir)
- return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
- output_proto.artifacts.add().path = tarball_path
-
- return controller.RETURN_CODE_SUCCESS
diff --git a/api/controller/artifacts_unittest.py b/api/controller/artifacts_unittest.py
index 6fb0740..7481e55 100644
--- a/api/controller/artifacts_unittest.py
+++ b/api/controller/artifacts_unittest.py
@@ -12,7 +12,6 @@
from chromite.api.gen.chromite.api import artifacts_pb2
from chromite.api.gen.chromite.api import toolchain_pb2
from chromite.cbuildbot import commands
-from chromite.lib import build_target_lib
from chromite.lib import chroot_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
@@ -1127,73 +1126,3 @@
with self.assertRaises(cros_build_lib.DieSystemExit):
artifacts.BundleGceTarball(self.target_request, self.response,
self.api_config)
-
-
-class BundleDebugSymbolsTest(BundleTestCase):
- """Unittests for BundleDebugSymbols."""
-
- def setUp(self):
- # Create a chroot_path that also includes a chroot tmp dir.
- self.chroot_path = os.path.join(self.tempdir, 'chroot_dir')
- osutils.SafeMakedirs(self.chroot_path)
- osutils.SafeMakedirs(os.path.join(self.chroot_path, 'tmp'))
- # Create output dir.
- output_dir = os.path.join(self.tempdir, 'output_dir')
- osutils.SafeMakedirs(output_dir)
- # Build target request.
- self.target_request = self.BuildTargetRequest(
- build_target='target',
- output_dir=self.output_dir,
- chroot=self.chroot_path)
-
- def testValidateOnly(self):
- """Check that a validate only call does not execute any logic."""
- patch = self.PatchObject(artifacts_svc, 'GenerateBreakpadSymbols')
- artifacts.BundleDebugSymbols(self.target_request, self.response,
- self.validate_only_config)
- patch.assert_not_called()
-
- def testMockCall(self):
- """Test that a mock call does not execute logic, returns mocked value."""
- patch = self.PatchObject(artifacts_svc, 'GenerateBreakpadSymbols')
- artifacts.BundleDebugSymbols(self.target_request, self.response,
- self.mock_call_config)
- patch.assert_not_called()
- self.assertEqual(len(self.response.artifacts), 1)
- self.assertEqual(self.response.artifacts[0].path,
- os.path.join(self.output_dir,
- constants.DEBUG_SYMBOLS_TAR))
-
- def testBundleDebugSymbols(self):
- """BundleDebugSymbols calls cbuildbot/commands with correct args."""
- # Patch service layer functions.
- generate_breakpad_symbols_patch = self.PatchObject(
- artifacts_svc, 'GenerateBreakpadSymbols',
- return_value=cros_build_lib.CommandResult(returncode=0, output=''))
- gather_symbol_files_patch = self.PatchObject(
- artifacts_svc, 'GatherSymbolFiles',
- return_value=[artifacts_svc.SymbolFileTuple(
- source_file_name='path/to/source/file1.sym',
- relative_path='file1.sym')])
-
- artifacts.BundleDebugSymbols(self.target_request, self.response,
- self.api_config)
- # Verify mock objects were called.
- build_target = build_target_lib.BuildTarget('target')
- generate_breakpad_symbols_patch.assert_called_with(
- mock.ANY, build_target, debug=True)
- gather_symbol_files_patch.assert_called()
-
- # Verify response proto contents and output directory contents.
- self.assertEqual(
- [artifact.path for artifact in self.response.artifacts],
- [os.path.join(self.output_dir, constants.DEBUG_SYMBOLS_TAR)])
- files = os.listdir(self.output_dir)
- self.assertEqual(files, [constants.DEBUG_SYMBOLS_TAR])
-
- def testBundleGceTarballNoImageDir(self):
- """BundleDebugSymbols dies when image dir does not exist."""
- self.PatchObject(os.path, 'exists', return_value=False)
- with self.assertRaises(cros_build_lib.DieSystemExit):
- artifacts.BundleDebugSymbols(self.target_request, self.response,
- self.api_config)
diff --git a/api/controller/controller_util.py b/api/controller/controller_util.py
index 4a39f93..c86dd85 100644
--- a/api/controller/controller_util.py
+++ b/api/controller/controller_util.py
@@ -11,7 +11,7 @@
from chromite.lib import constants
from chromite.lib.parser import package_info
from chromite.lib.chroot_lib import Chroot
-
+from chromite.lib.sysroot_lib import Sysroot
class Error(Exception):
"""Base error class for the module."""
@@ -58,6 +58,24 @@
return chroot
+
+def ParseSysroot(sysroot_message):
+ """Create a sysroot object from the sysroot message.
+
+ Args:
+ sysroot_message (commmon_pb2.Sysroot): The sysroot message.
+
+ Returns:
+ Sysroot: The parsed sysroot object.
+
+ Raises:
+ AssertionError: When the message is not a Sysroot message.
+ """
+ assert isinstance(sysroot_message, sysroot_pb2.Sysroot)
+
+ return Sysroot(sysroot_message.path)
+
+
def ParseGomaConfig(goma_message, chroot_path):
"""Parse a goma config message."""
assert isinstance(goma_message, common_pb2.GomaConfig)
diff --git a/api/controller/controller_util_unittest.py b/api/controller/controller_util_unittest.py
index 0786958..038e5d8 100644
--- a/api/controller/controller_util_unittest.py
+++ b/api/controller/controller_util_unittest.py
@@ -12,6 +12,7 @@
from chromite.lib import cros_test_lib
from chromite.lib.parser import package_info
from chromite.lib.chroot_lib import Chroot
+from chromite.lib.sysroot_lib import Sysroot
class ParseChrootTest(cros_test_lib.MockTestCase):
@@ -44,6 +45,20 @@
with self.assertRaises(AssertionError):
controller_util.ParseChroot(common_pb2.BuildTarget())
+class ParseSysrootTest(cros_test_lib.MockTestCase):
+ """ParseSysroot tests."""
+
+ def testSuccess(self):
+ """test successful handling case."""
+ path = '/build/rare_pokemon'
+ sysroot_message = sysroot_pb2.Sysroot(path=path)
+ expected = Sysroot(path=path)
+ result = controller_util.ParseSysroot(sysroot_message)
+ self.assertEqual(expected, result)
+
+ def testWrongMessage(self):
+ with self.assertRaises(AssertionError):
+ controller_util.ParseSysroot(common_pb2.BuildTarget())
class ParseBuildTargetTest(cros_test_lib.TestCase):
"""ParseBuildTarget tests."""
diff --git a/api/controller/image.py b/api/controller/image.py
index 09e02db..32012d9 100644
--- a/api/controller/image.py
+++ b/api/controller/image.py
@@ -15,10 +15,13 @@
from chromite.api.controller import controller_util
from chromite.api.gen.chromiumos import common_pb2
from chromite.api.metrics import deserialize_metrics_log
+from chromite.lib import build_target_lib
+from chromite.lib import chroot_lib
from chromite.lib import cros_build_lib
from chromite.lib import constants
from chromite.lib import image_lib
from chromite.lib import cros_logging as logging
+from chromite.lib import sysroot_lib
from chromite.scripts import pushimage
from chromite.service import image
from chromite.utils import metrics
@@ -84,6 +87,56 @@
new_image.build_target.name = board
+def ExampleGetResponse():
+ """Give an example response to assemble upstream in caller artifacts."""
+ uabs = common_pb2.UploadedArtifactsByService
+ cabs = common_pb2.ArtifactsByService
+ return uabs.Sysroot(artifacts=[
+ uabs.Image.ArtifactPaths(
+ artifact_type=cabs.Image.ArtifactType.DLC_IMAGE,
+ paths=[
+ common_pb2.Path(
+ path='/tmp/dlc/dlc.img', location=common_pb2.Path.OUTSIDE)
+ ])
+ ])
+
+
+def GetArtifacts(in_proto: common_pb2.ArtifactsByService.Image,
+ chroot: chroot_lib.Chroot, sysroot_class: sysroot_lib.Sysroot,
+ _build_target: build_target_lib.BuildTarget, output_dir) -> list:
+ """Builds and copies images to specified output_dir.
+
+ Copies (after optionally bundling) all required images into the output_dir,
+ returning a mapping of image type to a list of (output_dir) paths to
+ the desired files. Note that currently it is only processing one image (DLC),
+ but the future direction is to process all required images. Required images
+ are located within output_artifact.artifact_type.
+
+ Args:
+ in_proto: Proto request defining reqs.
+ chroot: The chroot proto used for these artifacts.
+ sysroot_class: The sysroot proto used for these artifacts.
+ build_target: The build target used for these artifacts.
+ output_dir: The path to write artifacts to.
+
+ Returns:
+ A list of dictionary mappings of ArtifactType to list of paths.
+ """
+ generated = []
+ base_path = chroot.full_path(sysroot_class.path)
+
+ for output_artifact in in_proto.output_artifacts:
+ if in_proto.ArtifactType.DLC_IMAGE in output_artifact.artifact_types:
+ # Handling DLC copying.
+ result_paths = image.copy_dlc_image(base_path, output_dir)
+ if result_paths:
+ generated.append({
+ 'paths': result_paths,
+ 'type': in_proto.ArtifactType.DLC_IMAGE,
+ })
+ return generated
+
+
def _CreateResponse(_input_proto, output_proto, _config):
"""Set output_proto success field on a successful Create response."""
output_proto.success = True
diff --git a/api/controller/sysroot.py b/api/controller/sysroot.py
index 2f5c73b..0dc1f1e 100644
--- a/api/controller/sysroot.py
+++ b/api/controller/sysroot.py
@@ -13,6 +13,8 @@
from chromite.api.controller import controller_util
from chromite.api.metrics import deserialize_metrics_log
from chromite.lib import binpkg
+from chromite.lib import build_target_lib
+from chromite.lib import chroot_lib
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import goma_lib
@@ -22,10 +24,72 @@
from chromite.service import sysroot
from chromite.utils import metrics
-
_ACCEPTED_LICENSES = '@CHROMEOS'
+def ExampleGetResponse():
+ """Give an example response to assemble upstream in caller artifacts."""
+ uabs = common_pb2.UploadedArtifactsByService
+ cabs = common_pb2.ArtifactsByService
+ return uabs.Sysroot(artifacts=[
+ uabs.Sysroot.ArtifactPaths(
+ artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
+ paths=[
+ common_pb2.Path(
+ path='/tmp/debug.tgz', location=common_pb2.Path.OUTSIDE)
+ ],
+ ),
+ uabs.Sysroot.ArtifactPaths(
+ artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
+ paths=[
+ common_pb2.Path(
+ path='/tmp/debug_breakpad.tar.xz',
+ location=common_pb2.Path.OUTSIDE)
+ ])
+ ])
+
+
+def GetArtifacts(in_proto: common_pb2.ArtifactsByService.Sysroot,
+ chroot: chroot_lib.Chroot, sysroot_class: sysroot_lib.Sysroot,
+ build_target: build_target_lib.BuildTarget, output_dir: str) -> list:
+ """Builds and copies sysroot artifacts to specified output_dir.
+
+ Copies sysroot artifacts to output_dir, returning a list of (output_dir: str)
+ paths to the desired files.
+
+ Args:
+ in_proto: Proto request defining reqs.
+ chroot: The chroot class used for these artifacts.
+ sysroot_class: The sysroot class used for these artifacts.
+ build_target: The build target used for these artifacts.
+ output_dir: The path to write artifacts to.
+
+ Returns:
+ A list of dictionary mappings of ArtifactType to list of paths.
+ """
+ generated = []
+ for output_artifact in in_proto.output_artifacts:
+ if (in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS
+ in output_artifact.artifact_types):
+ result_path = sysroot.BundleBreakpadSymbols(chroot, sysroot_class,
+ build_target, output_dir)
+ if result_path:
+ generated.append({
+ 'paths': [result_path],
+ 'type': in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
+ })
+ if in_proto.ArtifactType.DEBUG_SYMBOLS in output_artifact.artifact_types:
+ result_path = sysroot.BundleDebugSymbols(chroot, sysroot_class,
+ build_target, output_dir)
+ if result_path:
+ generated.append({
+ 'paths': [result_path],
+ 'type': in_proto.ArtifactType.DEBUG_SYMBOLS,
+ })
+ return generated
+
+
+
@faux.all_empty
@validate.require('build_target.name')
@validate.validation_complete