api/controller/sysroot.py

# Copyright 2019 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sysroot controller."""

import logging
import os
import traceback

from chromite.api import controller
from chromite.api import faux
from chromite.api import validate
from chromite.api.controller import controller_util
from chromite.api.gen.chromiumos import common_pb2
from chromite.api.metrics import deserialize_metrics_log
from chromite.lib import build_target_lib
from chromite.lib import chroot_lib
from chromite.lib import cros_build_lib
from chromite.lib import goma_lib
from chromite.lib import metrics_lib
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import sysroot_lib
from chromite.service import sysroot


_ACCEPTED_LICENSES = "@CHROMEOS"

DEFAULT_BACKTRACK = 30


def _GetGomaLogDirectory():
    """Get goma's log directory based on the env variables.

    Returns:
        a string of a directory name where goma's log may exist, or None if no
        potential directories exist.
    """
    # TODO(crbug.com/1045001): Replace environment variable with query to
    # goma object after goma refactoring allows this.
    candidates = [
        "GLOG_log_dir",
        "GOOGLE_LOG_DIR",
        "TEST_TMPDIR",
        "TMPDIR",
        "TMP",
    ]
    for candidate in candidates:
        value = os.environ.get(candidate)
        if value and os.path.isdir(value):
            return value

    # "/tmp" will always exist.
    return "/tmp"


def ExampleGetResponse():
    """Give an example response to assemble upstream in caller artifacts."""
    uabs = common_pb2.UploadedArtifactsByService
    cabs = common_pb2.ArtifactsByService
    return uabs.Sysroot(
        artifacts=[
            uabs.Sysroot.ArtifactPaths(
                artifact_type=cabs.Sysroot.ArtifactType.SIMPLE_CHROME_SYSROOT,
                paths=[
                    common_pb2.Path(
                        path=(
                            "/tmp/sysroot_chromeos-base_chromeos-chrome.tar.xz"
                        ),
                        location=common_pb2.Path.OUTSIDE,
                    )
                ],
            ),
            uabs.Sysroot.ArtifactPaths(
                artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
                paths=[
                    common_pb2.Path(
                        path="/tmp/debug.tgz", location=common_pb2.Path.OUTSIDE
                    )
                ],
            ),
            uabs.Sysroot.ArtifactPaths(
                artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
                paths=[
                    common_pb2.Path(
                        path="/tmp/debug_breakpad.tar.xz",
                        location=common_pb2.Path.OUTSIDE,
                    )
                ],
            ),
        ]
    )


def GetArtifacts(
    in_proto: common_pb2.ArtifactsByService.Sysroot,
    chroot: chroot_lib.Chroot,
    sysroot_class: sysroot_lib.Sysroot,
    build_target: build_target_lib.BuildTarget,
    output_dir: str,
) -> list:
    """Builds and copies sysroot artifacts to specified output_dir.

    Copies sysroot artifacts to output_dir, returning a list of
    (output_dir: str) paths to the desired files.

    Args:
        in_proto: Proto request defining reqs.
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.

    Returns:
        A list of dictionary mappings of ArtifactType to list of paths.
    """

    def _BundleBreakpadSymbols(chroot, sysroot_class, build_target, output_dir):
        # pylint: disable=line-too-long
        ignore_breakpad_symbol_generation_expected_files = [
            common_pb2.ArtifactsByService.Sysroot.BreakpadSymbolGenerationExpectedFile.Name(
                x
            )
            for x in in_proto.ignore_breakpad_symbol_generation_expected_files
            if x
            != common_pb2.ArtifactsByService.Sysroot.BreakpadSymbolGenerationExpectedFile.EXPECTED_FILE_UNSET
            and x
            in common_pb2.ArtifactsByService.Sysroot.BreakpadSymbolGenerationExpectedFile.values()
        ]
        # pylint: enable=line-too-long

        ignore_breakpad_symbol_generation_expected_files = [
            x[len("EXPECTED_FILE_") :]
            for x in ignore_breakpad_symbol_generation_expected_files
        ]

        return sysroot.BundleBreakpadSymbols(
            chroot,
            sysroot_class,
            build_target,
            output_dir,
            in_proto.ignore_breakpad_symbol_generation_errors,
            ignore_breakpad_symbol_generation_expected_files,
        )

    generated = []
    # pylint: disable=line-too-long
    artifact_types = {
        in_proto.ArtifactType.SIMPLE_CHROME_SYSROOT: sysroot.CreateSimpleChromeSysroot,
        in_proto.ArtifactType.CHROME_EBUILD_ENV: sysroot.CreateChromeEbuildEnv,
        in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS: _BundleBreakpadSymbols,
        in_proto.ArtifactType.DEBUG_SYMBOLS: sysroot.BundleDebugSymbols,
        in_proto.ArtifactType.FUZZER_SYSROOT: sysroot.CreateFuzzerSysroot,
        in_proto.ArtifactType.SYSROOT_ARCHIVE: sysroot.ArchiveSysroot,
    }
    # pylint: enable=line-too-long

    for output_artifact in in_proto.output_artifacts:
        for artifact_type, func in artifact_types.items():
            if artifact_type in output_artifact.artifact_types:
                try:
                    result = func(
                        chroot, sysroot_class, build_target, output_dir
                    )
                except Exception as e:
                    generated.append(
                        {
                            "type": artifact_type,
                            "failed": True,
                            "failure_reason": str(e),
                        }
                    )
                    artifact_name = (
                        common_pb2.ArtifactsByService.Sysroot.ArtifactType.Name(
                            artifact_type
                        )
                    )
                    logging.warning(
                        "%s artifact generation failed with exception %s",
                        artifact_name,
                        e,
                    )
                    logging.warning("traceback:\n%s", traceback.format_exc())
                    continue
                if result:
                    generated.append(
                        {
                            "paths": [str(result)]
                            if isinstance(result, (os.PathLike, str))
                            else result,
                            "type": artifact_type,
                        }
                    )

    return generated


@faux.all_empty
@validate.require("build_target.name")
@validate.validation_complete
def Create(input_proto, output_proto, _config):
    """Create or replace a sysroot."""
    update_chroot = not input_proto.flags.chroot_current
    replace_sysroot = input_proto.flags.replace
    use_cq_prebuilts = input_proto.flags.use_cq_prebuilts

    build_target = controller_util.ParseBuildTarget(
        input_proto.build_target, input_proto.profile
    )
    package_indexes = [
        controller_util.deserialize_package_index_info(x)
        for x in input_proto.package_indexes
    ]
    run_configs = sysroot.SetupBoardRunConfig(
        force=replace_sysroot,
        upgrade_chroot=update_chroot,
        package_indexes=package_indexes,
        use_cq_prebuilts=use_cq_prebuilts,
        backtrack=DEFAULT_BACKTRACK,
    )

    try:
        created = sysroot.Create(
            build_target, run_configs, accept_licenses=_ACCEPTED_LICENSES
        )
    except sysroot.Error as e:
        cros_build_lib.Die(e)

    output_proto.sysroot.path = created.path
    output_proto.sysroot.build_target.name = build_target.name

    return controller.RETURN_CODE_SUCCESS


@faux.all_empty
@validate.require("build_target.name", "packages")
@validate.require_each("packages", ["category", "package_name"])
@validate.validation_complete
def GenerateArchive(input_proto, output_proto, _config):
    """Generate a sysroot. Typically used by informational builders."""
    build_target_name = input_proto.build_target.name
    pkg_list = []
    for package in input_proto.packages:
        pkg_list.append("%s/%s" % (package.category, package.package_name))

    with osutils.TempDir(delete=False) as temp_output_dir:
        sysroot_tar_path = sysroot.GenerateArchive(
            temp_output_dir, build_target_name, pkg_list
        )

    # By assigning this Path variable to the tar path, the tar file will be
    # copied out to the input_proto's ResultPath location.
    output_proto.sysroot_archive.path = sysroot_tar_path
    output_proto.sysroot_archive.location = common_pb2.Path.INSIDE


def _MockFailedPackagesResponse(_input_proto, output_proto, _config):
    """Mock error response that populates failed packages."""
    fail = output_proto.failed_package_data.add()
    fail.name.package_name = "package"
    fail.name.category = "category"
    fail.name.version = "1.0.0_rc-r1"
    fail.log_path.path = (
        "/path/to/package:category-1.0.0_rc-r1:20210609-1337.log"
    )
    fail.log_path.location = common_pb2.Path.INSIDE

    fail2 = output_proto.failed_package_data.add()
    fail2.name.package_name = "bar"
    fail2.name.category = "foo"
    fail2.name.version = "3.7-r99"
    fail2.log_path.path = "/path/to/foo:bar-3.7-r99:20210609-1620.log"
    fail2.log_path.location = common_pb2.Path.INSIDE


@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
@validate.require("sysroot.path", "sysroot.build_target.name")
@validate.exists("sysroot.path")
@validate.validation_complete
def InstallToolchain(input_proto, output_proto, _config):
    """Install the toolchain into a sysroot."""
    compile_source = (
        input_proto.flags.compile_source or input_proto.flags.toolchain_changed
    )

    sysroot_path = input_proto.sysroot.path

    build_target = controller_util.ParseBuildTarget(
        input_proto.sysroot.build_target
    )
    target_sysroot = sysroot_lib.Sysroot(sysroot_path)
    run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)

    _LogBinhost(build_target.name)

    try:
        sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
    except sysroot_lib.ToolchainInstallError as e:
        controller_util.retrieve_package_log_paths(
            e.failed_toolchain_info, output_proto, target_sysroot
        )

        return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE

    return controller.RETURN_CODE_SUCCESS


@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
@validate.require("sysroot.build_target.name")
@validate.exists("sysroot.path")
@validate.require_each("packages", ["category", "package_name"])
@validate.require_each("use_flags", ["flag"])
@validate.validation_complete
@metrics_lib.collect_metrics
def InstallPackages(input_proto, output_proto, _config):
    """Install packages into a sysroot, building as necessary and permitted."""
    compile_source = (
        input_proto.flags.compile_source or input_proto.flags.toolchain_changed
    )

    use_remoteexec = bool(
        input_proto.remoteexec_config.reproxy_cfg_file
        and input_proto.remoteexec_config.reclient_dir
    )

    # Testing if Goma will support unknown compilers now.
    use_goma = input_proto.flags.use_goma and not use_remoteexec

    target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
    build_target = controller_util.ParseBuildTarget(
        input_proto.sysroot.build_target
    )

    # Get the package atom for each specified package. The field is optional, so
    # error only when we cannot parse an atom for each of the given packages.
    packages = [
        controller_util.deserialize_package_info(x).atom
        for x in input_proto.packages
    ]

    package_indexes = [
        controller_util.deserialize_package_index_info(x)
        for x in input_proto.package_indexes
    ]

    # Calculate which packages would have been merged, but don't install
    # anything.
    dryrun = input_proto.flags.dryrun

    # Allow cros workon packages to build from the unstable ebuilds.
    workon = input_proto.flags.workon

    # Use Bazel to build packages.
    bazel = input_proto.flags.bazel

    if not target_sysroot.IsToolchainInstalled():
        cros_build_lib.Die("Toolchain must first be installed.")

    _LogBinhost(build_target.name)

    use_flags = [u.flag for u in input_proto.use_flags]
    build_packages_config = sysroot.BuildPackagesRunConfig(
        use_any_chrome=False,
        usepkg=not compile_source,
        install_debug_symbols=True,
        packages=packages,
        package_indexes=package_indexes,
        use_flags=use_flags,
        use_goma=use_goma,
        use_remoteexec=use_remoteexec,
        incremental_build=False,
        dryrun=dryrun,
        backtrack=DEFAULT_BACKTRACK,
        workon=workon,
        bazel=bazel,
    )

    try:
        sysroot.BuildPackages(
            build_target, target_sysroot, build_packages_config
        )
    except sysroot_lib.PackageInstallError as e:
        if not e.failed_packages:
            # No packages to report, so just exit with an error code.
            return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY

        controller_util.retrieve_package_log_paths(
            e.failed_packages, output_proto, target_sysroot
        )

        return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
    finally:
        # Copy goma logs to specified directory if there is a goma_config and
        # it contains a log_dir to store artifacts.
        if input_proto.goma_config.log_dir.dir:
            log_source_dir = _GetGomaLogDirectory()
            archiver = goma_lib.LogsArchiver(
                log_source_dir,
                dest_dir=input_proto.goma_config.log_dir.dir,
                stats_file=input_proto.goma_config.stats_file,
                counterz_file=input_proto.goma_config.counterz_file,
            )
            archiver_tuple = archiver.Archive()
            if archiver_tuple.stats_file:
                output_proto.goma_artifacts.stats_file = (
                    archiver_tuple.stats_file
                )
            if archiver_tuple.counterz_file:
                output_proto.goma_artifacts.counterz_file = (
                    archiver_tuple.counterz_file
                )
            output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files

    # Return without populating the response if it is a dryrun.
    if dryrun:
        return controller.RETURN_CODE_SUCCESS

    # Read metric events log and pipe them into output_proto.events.
    deserialize_metrics_log(output_proto.events, prefix=build_target.name)


def _LogBinhost(board):
    """Log the portage binhost for the given board."""
    binhost = portage_util.PortageqEnvvar(
        "PORTAGE_BINHOST", board=board, allow_undefined=True
    )
    if not binhost:
        logging.warning("Portage Binhost not found.")
    else:
        logging.info("Portage Binhost: %s", binhost)