api: Add extra failure fields in Install* methods
Provide an expanded set of data points for each package which failed to
build in the InstallPackages and InstallToolchain endpoints.
BUG=b:204816060
TEST=unit
Change-Id: I2b7e6e2750b9d8dd6041acdc191198237f237335
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/3279481
Tested-by: Lizzy Presland <zland@google.com>
Auto-Submit: Lizzy Presland <zland@google.com>
Commit-Queue: Lizzy Presland <zland@google.com>
Reviewed-by: Alex Klein <saklein@chromium.org>
diff --git a/api/controller/sysroot.py b/api/controller/sysroot.py
index 9057f38..e28eed2 100644
--- a/api/controller/sysroot.py
+++ b/api/controller/sysroot.py
@@ -4,6 +4,7 @@
"""Sysroot controller."""
+import glob
import logging
import os
@@ -164,6 +165,20 @@
pkg2.category = 'foo'
pkg2.version = '3.7-r99'
+ fail = output_proto.failed_package_data.add()
+ fail.name.package_name = 'package'
+ fail.name.category = 'category'
+ fail.name.version = '1.0.0_rc-r1'
+ fail.log_path.path = '/path/to/package:category-1.0.0_rc-r1:20210609-1337.log'
+ fail.log_path.location = common_pb2.Path.INSIDE
+
+ fail2 = output_proto.failed_package_data.add()
+ fail2.name.package_name = 'bar'
+ fail2.name.category = 'foo'
+ fail2.name.version = '3.7-r99'
+ fail2.log_path.path = '/path/to/foo:bar-3.7-r99:20210609-1620.log'
+ fail2.log_path.location = common_pb2.Path.INSIDE
+
@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
@@ -189,8 +204,24 @@
except sysroot_lib.ToolchainInstallError as e:
# Error installing - populate the failed package info.
for pkg_info in e.failed_toolchain_info:
+ # TODO(b/206514844): remove when field is deleted
package_info_msg = output_proto.failed_packages.add()
controller_util.serialize_package_info(pkg_info, package_info_msg)
+ # Grab the paths to the log files for each failed package from the
+ # sysroot.
+ # Logs currently exist within the sysroot in the form of:
+ # /build/${BOARD}/tmp/portage/logs/$CATEGORY:$PF:$TIMESTAMP.log
+ failed_pkg_data_msg = output_proto.failed_package_data.add()
+ controller_util.serialize_package_info(pkg_info, failed_pkg_data_msg.name)
+ glob_path = os.path.join(target_sysroot.path, 'tmp', 'portage', 'logs',
+ f'{pkg_info.category}:{pkg_info.package}:*.log')
+ log_files = glob.glob(glob_path)
+ log_files.sort(reverse=True)
+ # Omit path if files don't exist for some reason
+ if not log_files:
+ continue
+ failed_pkg_data_msg.log_path.path = log_files[0]
+ failed_pkg_data_msg.log_path.location = common_pb2.Path.INSIDE
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
@@ -255,8 +286,24 @@
# We need to report the failed packages.
for pkg_info in e.failed_packages:
+ # TODO(b/206514844): remove when field is deleted
package_info_msg = output_proto.failed_packages.add()
controller_util.serialize_package_info(pkg_info, package_info_msg)
+ # Grab the paths to the log files for each failed package from the
+ # sysroot.
+ # Logs currently exist within the sysroot in the form of:
+ # /build/${BOARD}/tmp/portage/logs/$CATEGORY:$PF:$TIMESTAMP.log
+ failed_pkg_data_msg = output_proto.failed_package_data.add()
+ controller_util.serialize_package_info(pkg_info, failed_pkg_data_msg.name)
+ glob_path = os.path.join(target_sysroot.path, 'tmp', 'portage', 'logs',
+ f'{pkg_info.category}:{pkg_info.package}:*.log')
+ log_files = glob.glob(glob_path)
+ log_files.sort(reverse=True)
+ # Omit path if files don't exist for some reason
+ if not log_files:
+ continue
+ failed_pkg_data_msg.log_path.path = log_files[0]
+ failed_pkg_data_msg.log_path.location = common_pb2.Path.INSIDE
return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
diff --git a/api/controller/sysroot_unittest.py b/api/controller/sysroot_unittest.py
index 9115eb1..cfae62d 100644
--- a/api/controller/sysroot_unittest.py
+++ b/api/controller/sysroot_unittest.py
@@ -247,6 +247,9 @@
self.sysroot = os.path.join(self.tempdir, 'board')
self.invalid_sysroot = os.path.join(self.tempdir, 'invalid', 'sysroot')
osutils.SafeMakedirs(self.sysroot)
+ # Set up portage log directory.
+ self.portage_dir = os.path.join(self.sysroot, 'tmp', 'portage', 'logs')
+ osutils.SafeMakedirs(self.portage_dir)
def _InputProto(self, build_target=None, sysroot_path=None,
compile_source=False):
@@ -265,6 +268,22 @@
"""Helper to build output proto instance."""
return sysroot_pb2.InstallToolchainResponse()
+ def _CreatePortageLogFile(self, root, pkg_info, timestamp):
+ """Creates a log file for testing for individual packages built by Portage.
+
+ Args:
+ root (pathlike): the sysroot path
+ pkg_info (PackageInfo): name components for log file.
+ timestamp (datetime): timestamp used to name the file.
+ """
+ path = os.path.join(root, 'tmp', 'portage', 'logs',
+ f'{pkg_info.category}:{pkg_info.package}:' \
+ f'{timestamp.strftime("%Y%m%d-%H%M%S")}.log')
+ osutils.WriteFile(path,
+ f'Test log file for package {pkg_info.category}/'
+ f'{pkg_info.package} written to {path}')
+ return path
+
def testValidateOnly(self):
"""Sanity check that a validate only call does not execute any logic."""
patch = self.PatchObject(sysroot_service, 'InstallToolchain')
@@ -347,6 +366,16 @@
err_pkgs = ['cat/pkg', 'cat2/pkg2']
err_cpvs = [package_info.parse(pkg) for pkg in err_pkgs]
expected = [('cat', 'pkg'), ('cat2', 'pkg2')]
+
+ new_logs = {}
+ for i, pkg in enumerate(err_pkgs):
+ self._CreatePortageLogFile(self.sysroot, err_cpvs[i],
+ datetime.datetime(2021, 6, 9, 13, 37, 0))
+ new_logs[pkg] = self._CreatePortageLogFile(self.sysroot, err_cpvs[i],
+ datetime.datetime(2021, 6, 9,
+ 16, 20, 0)
+ )
+
err = sysroot_lib.ToolchainInstallError('Error',
cros_build_lib.CommandResult(),
tc_info=err_cpvs)
@@ -356,6 +385,16 @@
self.api_config)
self.assertEqual(controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE, rc)
self.assertTrue(out_proto.failed_packages)
+ self.assertTrue(out_proto.failed_package_data)
+ # This needs to return 2 to indicate the available error response.
+ self.assertEqual(controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE, rc)
+ for data in out_proto.failed_package_data:
+ package = controller_util.deserialize_package_info(data.name)
+ cat_pkg = (data.name.category, data.name.package_name)
+ self.assertIn(cat_pkg, expected)
+ self.assertEqual(data.log_path.path, new_logs[package.atom])
+
+ # TODO(b/206514844): remove when field is deleted
for package in out_proto.failed_packages:
cat_pkg = (package.category, package.package_name)
self.assertIn(cat_pkg, expected)
@@ -372,6 +411,9 @@
self.build_target = 'board'
self.sysroot = os.path.join(self.tempdir, 'build', 'board')
osutils.SafeMakedirs(self.sysroot)
+ # Set up portage log directory.
+ self.portage_dir = os.path.join(self.sysroot, 'tmp', 'portage', 'logs')
+ osutils.SafeMakedirs(self.portage_dir)
# Set up goma directories.
self.goma_dir = os.path.join(self.tempdir, 'goma_dir')
osutils.SafeMakedirs(self.goma_dir)
@@ -428,6 +470,21 @@
path,
timestamp.strftime('Goma log file created at: %Y/%m/%d %H:%M:%S'))
+ def _CreatePortageLogFile(self, root, pkg_info, timestamp):
+ """Creates a log file for testing for individual packages built by Portage.
+
+ Args:
+ root (pathlike): the root path, taken from a BuildTarget object.
+ pkg_info (PackageInfo): name components for log file.
+ timestamp (datetime): timestamp used to name the file.
+ """
+ path = os.path.join(root, 'tmp', 'portage', 'logs',
+ f'{pkg_info.category}:{pkg_info.package}:' \
+ f'{timestamp.strftime("%Y%m%d-%H%M%S")}.log')
+ osutils.WriteFile(path, f'Test log file for package {pkg_info.category}/'
+ f'{pkg_info.package} written to {path}')
+ return path
+
def testValidateOnly(self):
"""Sanity check that a validate only call does not execute any logic."""
patch = self.PatchObject(sysroot_service, 'BuildPackages')
@@ -685,9 +742,17 @@
# Failed package info and expected list for verification.
err_pkgs = ['cat/pkg', 'cat2/pkg2']
- err_cpvs = [package_info.SplitCPV(cpv, strict=False) for cpv in err_pkgs]
+ err_cpvs = [package_info.parse(cpv) for cpv in err_pkgs]
expected = [('cat', 'pkg'), ('cat2', 'pkg2')]
+ new_logs = {}
+ for i, pkg in enumerate(err_pkgs):
+ self._CreatePortageLogFile(self.sysroot, err_cpvs[i],
+ datetime.datetime(2021, 6, 9, 13, 37, 0))
+ new_logs[pkg] = self._CreatePortageLogFile(self.sysroot, err_cpvs[i],
+ datetime.datetime(2021, 6, 9,
+ 16, 20, 0)
+ )
# Force error to be raised with the packages.
error = sysroot_lib.PackageInstallError('Error',
cros_build_lib.CommandResult(),
@@ -698,6 +763,13 @@
self.api_config)
# This needs to return 2 to indicate the available error response.
self.assertEqual(controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE, rc)
+ for data in out_proto.failed_package_data:
+ package = controller_util.deserialize_package_info(data.name)
+ cat_pkg = (data.name.category, data.name.package_name)
+ self.assertIn(cat_pkg, expected)
+ self.assertEqual(data.log_path.path, new_logs[package.atom])
+
+ # TODO(b/206514844): remove when field is deleted
for package in out_proto.failed_packages:
cat_pkg = (package.category, package.package_name)
self.assertIn(cat_pkg, expected)