Formatting: Format all python code with black.
This CL is probably not what you're looking for, it's only
automated formatting. Ignore it with
`git blame --ignore-rev <revision>` for this commit.
BUG=b:233893248
TEST=CQ
Change-Id: I66591d7a738d241aed3290138c0f68065ab10a6d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/3879174
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Tested-by: Alex Klein <saklein@chromium.org>
diff --git a/api/controller/sysroot.py b/api/controller/sysroot.py
index 332b7a3..0a07647 100644
--- a/api/controller/sysroot.py
+++ b/api/controller/sysroot.py
@@ -25,309 +25,339 @@
from chromite.service import sysroot
-_ACCEPTED_LICENSES = '@CHROMEOS'
+_ACCEPTED_LICENSES = "@CHROMEOS"
DEFAULT_BACKTRACK = 30
def _GetGomaLogDirectory():
- """Get goma's log directory based on the env variables.
+ """Get goma's log directory based on the env variables.
- Returns:
- a string of a directory name where goma's log may exist, or None if no
- potential directories exist.
- """
- # TODO(crbug.com/1045001): Replace environment variable with query to
- # goma object after goma refactoring allows this.
- candidates = [
- 'GLOG_log_dir',
- 'GOOGLE_LOG_DIR',
- 'TEST_TMPDIR',
- 'TMPDIR',
- 'TMP',
- ]
- for candidate in candidates:
- value = os.environ.get(candidate)
- if value and os.path.isdir(value):
- return value
+ Returns:
+ a string of a directory name where goma's log may exist, or None if no
+ potential directories exist.
+ """
+ # TODO(crbug.com/1045001): Replace environment variable with query to
+ # goma object after goma refactoring allows this.
+ candidates = [
+ "GLOG_log_dir",
+ "GOOGLE_LOG_DIR",
+ "TEST_TMPDIR",
+ "TMPDIR",
+ "TMP",
+ ]
+ for candidate in candidates:
+ value = os.environ.get(candidate)
+ if value and os.path.isdir(value):
+ return value
- # "/tmp" will always exist.
- return '/tmp'
+ # "/tmp" will always exist.
+ return "/tmp"
def ExampleGetResponse():
- """Give an example response to assemble upstream in caller artifacts."""
- uabs = common_pb2.UploadedArtifactsByService
- cabs = common_pb2.ArtifactsByService
- return uabs.Sysroot(artifacts=[
- uabs.Sysroot.ArtifactPaths(
- artifact_type=cabs.Sysroot.ArtifactType.SIMPLE_CHROME_SYSROOT,
- paths=[
- common_pb2.Path(
- path='/tmp/sysroot_chromeos-base_chromeos-chrome.tar.xz',
- location=common_pb2.Path.OUTSIDE)
- ],
- ),
- uabs.Sysroot.ArtifactPaths(
- artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
- paths=[
- common_pb2.Path(
- path='/tmp/debug.tgz', location=common_pb2.Path.OUTSIDE)
- ],
- ),
- uabs.Sysroot.ArtifactPaths(
- artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
- paths=[
- common_pb2.Path(
- path='/tmp/debug_breakpad.tar.xz',
- location=common_pb2.Path.OUTSIDE)
- ])
- ])
+ """Give an example response to assemble upstream in caller artifacts."""
+ uabs = common_pb2.UploadedArtifactsByService
+ cabs = common_pb2.ArtifactsByService
+ return uabs.Sysroot(
+ artifacts=[
+ uabs.Sysroot.ArtifactPaths(
+ artifact_type=cabs.Sysroot.ArtifactType.SIMPLE_CHROME_SYSROOT,
+ paths=[
+ common_pb2.Path(
+ path="/tmp/sysroot_chromeos-base_chromeos-chrome.tar.xz",
+ location=common_pb2.Path.OUTSIDE,
+ )
+ ],
+ ),
+ uabs.Sysroot.ArtifactPaths(
+ artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
+ paths=[
+ common_pb2.Path(
+ path="/tmp/debug.tgz", location=common_pb2.Path.OUTSIDE
+ )
+ ],
+ ),
+ uabs.Sysroot.ArtifactPaths(
+ artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
+ paths=[
+ common_pb2.Path(
+ path="/tmp/debug_breakpad.tar.xz",
+ location=common_pb2.Path.OUTSIDE,
+ )
+ ],
+ ),
+ ]
+ )
-def GetArtifacts(in_proto: common_pb2.ArtifactsByService.Sysroot,
- chroot: chroot_lib.Chroot, sysroot_class: sysroot_lib.Sysroot,
- build_target: build_target_lib.BuildTarget,
- output_dir: str) -> list:
- """Builds and copies sysroot artifacts to specified output_dir.
+def GetArtifacts(
+ in_proto: common_pb2.ArtifactsByService.Sysroot,
+ chroot: chroot_lib.Chroot,
+ sysroot_class: sysroot_lib.Sysroot,
+ build_target: build_target_lib.BuildTarget,
+ output_dir: str,
+) -> list:
+ """Builds and copies sysroot artifacts to specified output_dir.
- Copies sysroot artifacts to output_dir, returning a list of (output_dir: str)
- paths to the desired files.
+ Copies sysroot artifacts to output_dir, returning a list of (output_dir: str)
+ paths to the desired files.
- Args:
- in_proto: Proto request defining reqs.
- chroot: The chroot class used for these artifacts.
- sysroot_class: The sysroot class used for these artifacts.
- build_target: The build target used for these artifacts.
- output_dir: The path to write artifacts to.
+ Args:
+ in_proto: Proto request defining reqs.
+ chroot: The chroot class used for these artifacts.
+ sysroot_class: The sysroot class used for these artifacts.
+ build_target: The build target used for these artifacts.
+ output_dir: The path to write artifacts to.
- Returns:
- A list of dictionary mappings of ArtifactType to list of paths.
- """
- generated = []
- artifact_types = {
- in_proto.ArtifactType.SIMPLE_CHROME_SYSROOT:
- sysroot.CreateSimpleChromeSysroot,
- in_proto.ArtifactType.CHROME_EBUILD_ENV:
- sysroot.CreateChromeEbuildEnv,
- in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS:
- sysroot.BundleBreakpadSymbols,
- in_proto.ArtifactType.DEBUG_SYMBOLS:
- sysroot.BundleDebugSymbols,
- }
+ Returns:
+ A list of dictionary mappings of ArtifactType to list of paths.
+ """
+ generated = []
+ artifact_types = {
+ in_proto.ArtifactType.SIMPLE_CHROME_SYSROOT: sysroot.CreateSimpleChromeSysroot,
+ in_proto.ArtifactType.CHROME_EBUILD_ENV: sysroot.CreateChromeEbuildEnv,
+ in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS: sysroot.BundleBreakpadSymbols,
+ in_proto.ArtifactType.DEBUG_SYMBOLS: sysroot.BundleDebugSymbols,
+ }
- for output_artifact in in_proto.output_artifacts:
- for artifact_type, func in artifact_types.items():
- if artifact_type in output_artifact.artifact_types:
- result = func(chroot, sysroot_class, build_target, output_dir)
- if result:
- generated.append({
- 'paths': [result] if isinstance(result, str) else result,
- 'type': artifact_type,
- })
+ for output_artifact in in_proto.output_artifacts:
+ for artifact_type, func in artifact_types.items():
+ if artifact_type in output_artifact.artifact_types:
+ result = func(chroot, sysroot_class, build_target, output_dir)
+ if result:
+ generated.append(
+ {
+ "paths": [result]
+ if isinstance(result, str)
+ else result,
+ "type": artifact_type,
+ }
+ )
- return generated
+ return generated
@faux.all_empty
-@validate.require('build_target.name')
+@validate.require("build_target.name")
@validate.validation_complete
def Create(input_proto, output_proto, _config):
- """Create or replace a sysroot."""
- update_chroot = not input_proto.flags.chroot_current
- replace_sysroot = input_proto.flags.replace
+ """Create or replace a sysroot."""
+ update_chroot = not input_proto.flags.chroot_current
+ replace_sysroot = input_proto.flags.replace
- build_target = controller_util.ParseBuildTarget(input_proto.build_target,
- input_proto.profile)
- package_indexes = [
- binpkg.PackageIndexInfo.from_protobuf(x)
- for x in input_proto.package_indexes
- ]
- run_configs = sysroot.SetupBoardRunConfig(
- force=replace_sysroot,
- upgrade_chroot=update_chroot,
- package_indexes=package_indexes,
- backtrack=DEFAULT_BACKTRACK,
- )
+ build_target = controller_util.ParseBuildTarget(
+ input_proto.build_target, input_proto.profile
+ )
+ package_indexes = [
+ binpkg.PackageIndexInfo.from_protobuf(x)
+ for x in input_proto.package_indexes
+ ]
+ run_configs = sysroot.SetupBoardRunConfig(
+ force=replace_sysroot,
+ upgrade_chroot=update_chroot,
+ package_indexes=package_indexes,
+ backtrack=DEFAULT_BACKTRACK,
+ )
- try:
- created = sysroot.Create(
- build_target, run_configs, accept_licenses=_ACCEPTED_LICENSES)
- except sysroot.Error as e:
- cros_build_lib.Die(e)
+ try:
+ created = sysroot.Create(
+ build_target, run_configs, accept_licenses=_ACCEPTED_LICENSES
+ )
+ except sysroot.Error as e:
+ cros_build_lib.Die(e)
- output_proto.sysroot.path = created.path
- output_proto.sysroot.build_target.name = build_target.name
+ output_proto.sysroot.path = created.path
+ output_proto.sysroot.build_target.name = build_target.name
- return controller.RETURN_CODE_SUCCESS
+ return controller.RETURN_CODE_SUCCESS
@faux.all_empty
-@validate.require('build_target.name', 'packages')
-@validate.require_each('packages', ['category', 'package_name'])
+@validate.require("build_target.name", "packages")
+@validate.require_each("packages", ["category", "package_name"])
@validate.validation_complete
def GenerateArchive(input_proto, output_proto, _config):
- """Generate a sysroot. Typically used by informational builders."""
- build_target_name = input_proto.build_target.name
- pkg_list = []
- for package in input_proto.packages:
- pkg_list.append('%s/%s' % (package.category, package.package_name))
+ """Generate a sysroot. Typically used by informational builders."""
+ build_target_name = input_proto.build_target.name
+ pkg_list = []
+ for package in input_proto.packages:
+ pkg_list.append("%s/%s" % (package.category, package.package_name))
- with osutils.TempDir(delete=False) as temp_output_dir:
- sysroot_tar_path = sysroot.GenerateArchive(temp_output_dir,
- build_target_name, pkg_list)
+ with osutils.TempDir(delete=False) as temp_output_dir:
+ sysroot_tar_path = sysroot.GenerateArchive(
+ temp_output_dir, build_target_name, pkg_list
+ )
- # By assigning this Path variable to the tar path, the tar file will be
- # copied out to the input_proto's ResultPath location.
- output_proto.sysroot_archive.path = sysroot_tar_path
- output_proto.sysroot_archive.location = common_pb2.Path.INSIDE
+ # By assigning this Path variable to the tar path, the tar file will be
+ # copied out to the input_proto's ResultPath location.
+ output_proto.sysroot_archive.path = sysroot_tar_path
+ output_proto.sysroot_archive.location = common_pb2.Path.INSIDE
def _MockFailedPackagesResponse(_input_proto, output_proto, _config):
- """Mock error response that populates failed packages."""
- fail = output_proto.failed_package_data.add()
- fail.name.package_name = 'package'
- fail.name.category = 'category'
- fail.name.version = '1.0.0_rc-r1'
- fail.log_path.path = '/path/to/package:category-1.0.0_rc-r1:20210609-1337.log'
- fail.log_path.location = common_pb2.Path.INSIDE
+ """Mock error response that populates failed packages."""
+ fail = output_proto.failed_package_data.add()
+ fail.name.package_name = "package"
+ fail.name.category = "category"
+ fail.name.version = "1.0.0_rc-r1"
+ fail.log_path.path = (
+ "/path/to/package:category-1.0.0_rc-r1:20210609-1337.log"
+ )
+ fail.log_path.location = common_pb2.Path.INSIDE
- fail2 = output_proto.failed_package_data.add()
- fail2.name.package_name = 'bar'
- fail2.name.category = 'foo'
- fail2.name.version = '3.7-r99'
- fail2.log_path.path = '/path/to/foo:bar-3.7-r99:20210609-1620.log'
- fail2.log_path.location = common_pb2.Path.INSIDE
+ fail2 = output_proto.failed_package_data.add()
+ fail2.name.package_name = "bar"
+ fail2.name.category = "foo"
+ fail2.name.version = "3.7-r99"
+ fail2.log_path.path = "/path/to/foo:bar-3.7-r99:20210609-1620.log"
+ fail2.log_path.location = common_pb2.Path.INSIDE
@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
-@validate.require('sysroot.path', 'sysroot.build_target.name')
-@validate.exists('sysroot.path')
+@validate.require("sysroot.path", "sysroot.build_target.name")
+@validate.exists("sysroot.path")
@validate.validation_complete
def InstallToolchain(input_proto, output_proto, _config):
- """Install the toolchain into a sysroot."""
- compile_source = (
- input_proto.flags.compile_source or input_proto.flags.toolchain_changed)
+ """Install the toolchain into a sysroot."""
+ compile_source = (
+ input_proto.flags.compile_source or input_proto.flags.toolchain_changed
+ )
- sysroot_path = input_proto.sysroot.path
+ sysroot_path = input_proto.sysroot.path
- build_target = controller_util.ParseBuildTarget(
- input_proto.sysroot.build_target)
- target_sysroot = sysroot_lib.Sysroot(sysroot_path)
- run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)
+ build_target = controller_util.ParseBuildTarget(
+ input_proto.sysroot.build_target
+ )
+ target_sysroot = sysroot_lib.Sysroot(sysroot_path)
+ run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)
- _LogBinhost(build_target.name)
+ _LogBinhost(build_target.name)
- try:
- sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
- except sysroot_lib.ToolchainInstallError as e:
- controller_util.retrieve_package_log_paths(e.failed_toolchain_info,
- output_proto, target_sysroot)
+ try:
+ sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
+ except sysroot_lib.ToolchainInstallError as e:
+ controller_util.retrieve_package_log_paths(
+ e.failed_toolchain_info, output_proto, target_sysroot
+ )
- return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
+ return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
- return controller.RETURN_CODE_SUCCESS
+ return controller.RETURN_CODE_SUCCESS
@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
-@validate.require('sysroot.build_target.name')
-@validate.exists('sysroot.path')
-@validate.require_each('packages', ['category', 'package_name'])
-@validate.require_each('use_flags', ['flag'])
+@validate.require("sysroot.build_target.name")
+@validate.exists("sysroot.path")
+@validate.require_each("packages", ["category", "package_name"])
+@validate.require_each("use_flags", ["flag"])
@validate.validation_complete
@metrics_lib.collect_metrics
def InstallPackages(input_proto, output_proto, _config):
- """Install packages into a sysroot, building as necessary and permitted."""
- compile_source = (
- input_proto.flags.compile_source or input_proto.flags.toolchain_changed)
+ """Install packages into a sysroot, building as necessary and permitted."""
+ compile_source = (
+ input_proto.flags.compile_source or input_proto.flags.toolchain_changed
+ )
- use_remoteexec = bool(input_proto.remoteexec_config.reproxy_cfg_file and
- input_proto.remoteexec_config.reclient_dir)
+ use_remoteexec = bool(
+ input_proto.remoteexec_config.reproxy_cfg_file
+ and input_proto.remoteexec_config.reclient_dir
+ )
- # Testing if Goma will support unknown compilers now.
- use_goma = input_proto.flags.use_goma and not use_remoteexec
+ # Testing if Goma will support unknown compilers now.
+ use_goma = input_proto.flags.use_goma and not use_remoteexec
- target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
- build_target = controller_util.ParseBuildTarget(
- input_proto.sysroot.build_target)
+ target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
+ build_target = controller_util.ParseBuildTarget(
+ input_proto.sysroot.build_target
+ )
- # Get the package atom for each specified package. The field is optional, so
- # error only when we cannot parse an atom for each of the given packages.
- packages = [
- controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
- ]
+ # Get the package atom for each specified package. The field is optional, so
+ # error only when we cannot parse an atom for each of the given packages.
+ packages = [
+ controller_util.PackageInfoToCPV(x).cp for x in input_proto.packages
+ ]
- package_indexes = [
- binpkg.PackageIndexInfo.from_protobuf(x)
- for x in input_proto.package_indexes
- ]
+ package_indexes = [
+ binpkg.PackageIndexInfo.from_protobuf(x)
+ for x in input_proto.package_indexes
+ ]
- # Calculate which packages would have been merged, but don't install anything.
- dryrun = input_proto.flags.dryrun
+ # Calculate which packages would have been merged, but don't install anything.
+ dryrun = input_proto.flags.dryrun
- if not target_sysroot.IsToolchainInstalled():
- cros_build_lib.Die('Toolchain must first be installed.')
+ if not target_sysroot.IsToolchainInstalled():
+ cros_build_lib.Die("Toolchain must first be installed.")
- _LogBinhost(build_target.name)
+ _LogBinhost(build_target.name)
- use_flags = [u.flag for u in input_proto.use_flags]
- build_packages_config = sysroot.BuildPackagesRunConfig(
- use_any_chrome=False,
- usepkg=not compile_source,
- install_debug_symbols=True,
- packages=packages,
- package_indexes=package_indexes,
- use_flags=use_flags,
- use_goma=use_goma,
- use_remoteexec=use_remoteexec,
- incremental_build=False,
- dryrun=dryrun,
- backtrack=DEFAULT_BACKTRACK,
- )
+ use_flags = [u.flag for u in input_proto.use_flags]
+ build_packages_config = sysroot.BuildPackagesRunConfig(
+ use_any_chrome=False,
+ usepkg=not compile_source,
+ install_debug_symbols=True,
+ packages=packages,
+ package_indexes=package_indexes,
+ use_flags=use_flags,
+ use_goma=use_goma,
+ use_remoteexec=use_remoteexec,
+ incremental_build=False,
+ dryrun=dryrun,
+ backtrack=DEFAULT_BACKTRACK,
+ )
- try:
- sysroot.BuildPackages(build_target, target_sysroot, build_packages_config)
- except sysroot_lib.PackageInstallError as e:
- if not e.failed_packages:
- # No packages to report, so just exit with an error code.
- return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
+ try:
+ sysroot.BuildPackages(
+ build_target, target_sysroot, build_packages_config
+ )
+ except sysroot_lib.PackageInstallError as e:
+ if not e.failed_packages:
+ # No packages to report, so just exit with an error code.
+ return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY
- controller_util.retrieve_package_log_paths(e.failed_packages, output_proto,
- target_sysroot)
+ controller_util.retrieve_package_log_paths(
+ e.failed_packages, output_proto, target_sysroot
+ )
- return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
- finally:
- # Copy goma logs to specified directory if there is a goma_config and
- # it contains a log_dir to store artifacts.
- if input_proto.goma_config.log_dir.dir:
- log_source_dir = _GetGomaLogDirectory()
- archiver = goma_lib.LogsArchiver(
- log_source_dir,
- dest_dir=input_proto.goma_config.log_dir.dir,
- stats_file=input_proto.goma_config.stats_file,
- counterz_file=input_proto.goma_config.counterz_file)
- archiver_tuple = archiver.Archive()
- if archiver_tuple.stats_file:
- output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file
- if archiver_tuple.counterz_file:
- output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file
- output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
+ return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
+ finally:
+ # Copy goma logs to specified directory if there is a goma_config and
+ # it contains a log_dir to store artifacts.
+ if input_proto.goma_config.log_dir.dir:
+ log_source_dir = _GetGomaLogDirectory()
+ archiver = goma_lib.LogsArchiver(
+ log_source_dir,
+ dest_dir=input_proto.goma_config.log_dir.dir,
+ stats_file=input_proto.goma_config.stats_file,
+ counterz_file=input_proto.goma_config.counterz_file,
+ )
+ archiver_tuple = archiver.Archive()
+ if archiver_tuple.stats_file:
+ output_proto.goma_artifacts.stats_file = (
+ archiver_tuple.stats_file
+ )
+ if archiver_tuple.counterz_file:
+ output_proto.goma_artifacts.counterz_file = (
+ archiver_tuple.counterz_file
+ )
+ output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files
- # Return without populating the response if it is a dryrun.
- if dryrun:
- return controller.RETURN_CODE_SUCCESS
+ # Return without populating the response if it is a dryrun.
+ if dryrun:
+ return controller.RETURN_CODE_SUCCESS
- # Read metric events log and pipe them into output_proto.events.
- deserialize_metrics_log(output_proto.events, prefix=build_target.name)
+ # Read metric events log and pipe them into output_proto.events.
+ deserialize_metrics_log(output_proto.events, prefix=build_target.name)
def _LogBinhost(board):
- """Log the portage binhost for the given board."""
- binhost = portage_util.PortageqEnvvar(
- 'PORTAGE_BINHOST', board=board, allow_undefined=True)
- if not binhost:
- logging.warning('Portage Binhost not found.')
- else:
- logging.info('Portage Binhost: %s', binhost)
+ """Log the portage binhost for the given board."""
+ binhost = portage_util.PortageqEnvvar(
+ "PORTAGE_BINHOST", board=board, allow_undefined=True
+ )
+ if not binhost:
+ logging.warning("Portage Binhost not found.")
+ else:
+ logging.info("Portage Binhost: %s", binhost)