api/controller/sysroot.py

# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sysroot controller."""

import glob
import logging
import os

from chromite.api import controller
from chromite.api import faux
from chromite.api import validate
from chromite.api.controller import controller_util
from chromite.api.gen.chromiumos import common_pb2
from chromite.api.metrics import deserialize_metrics_log
from chromite.lib import binpkg
from chromite.lib import build_target_lib
from chromite.lib import chroot_lib
from chromite.lib import cros_build_lib
from chromite.lib import goma_lib
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import sysroot_lib
from chromite.service import sysroot
from chromite.utils import metrics


_ACCEPTED_LICENSES = '@CHROMEOS'


def ExampleGetResponse():
  """Give an example response to assemble upstream in caller artifacts."""
  uabs = common_pb2.UploadedArtifactsByService
  cabs = common_pb2.ArtifactsByService
  return uabs.Sysroot(artifacts=[
      uabs.Sysroot.ArtifactPaths(
          artifact_type=cabs.Sysroot.ArtifactType.SIMPLE_CHROME_SYSROOT,
          paths=[
              common_pb2.Path(
                  path='/tmp/sysroot_chromeos-base_chromeos-chrome.tar.xz',
                  location=common_pb2.Path.OUTSIDE)
          ],
      ),
      uabs.Sysroot.ArtifactPaths(
          artifact_type=cabs.Sysroot.ArtifactType.DEBUG_SYMBOLS,
          paths=[
              common_pb2.Path(
                  path='/tmp/debug.tgz', location=common_pb2.Path.OUTSIDE)
          ],
      ),
      uabs.Sysroot.ArtifactPaths(
          artifact_type=cabs.Sysroot.ArtifactType.BREAKPAD_DEBUG_SYMBOLS,
          paths=[
              common_pb2.Path(
                  path='/tmp/debug_breakpad.tar.xz',
                  location=common_pb2.Path.OUTSIDE)
          ])
  ])


def GetArtifacts(in_proto: common_pb2.ArtifactsByService.Sysroot,
                 chroot: chroot_lib.Chroot, sysroot_class: sysroot_lib.Sysroot,
                 build_target: build_target_lib.BuildTarget, output_dir: str
                 ) -> list:
  """Builds and copies sysroot artifacts to specified output_dir.

  Copies sysroot artifacts to output_dir, returning a list of (output_dir: str)
  paths to the desired files.

  Args:
    in_proto: Proto request defining reqs.
    chroot: The chroot class used for these artifacts.
    sysroot_class: The sysroot class used for these artifacts.
    build_target: The build target used for these artifacts.
    output_dir: The path to write artifacts to.

  Returns:
    A list of dictionary mappings of ArtifactType to list of paths.
  """
  generated = []
  artifact_types = {
      in_proto.ArtifactType.SIMPLE_CHROME_SYSROOT:
          sysroot.CreateSimpleChromeSysroot,
      in_proto.ArtifactType.CHROME_EBUILD_ENV: sysroot.CreateChromeEbuildEnv,
      in_proto.ArtifactType.BREAKPAD_DEBUG_SYMBOLS:
          sysroot.BundleBreakpadSymbols,
      in_proto.ArtifactType.DEBUG_SYMBOLS: sysroot.BundleDebugSymbols,
  }

  for output_artifact in in_proto.output_artifacts:
    for artifact_type, func in artifact_types.items():
      if artifact_type in output_artifact.artifact_types:
        result = func(chroot, sysroot_class, build_target, output_dir)
        if result:
          generated.append({
              'paths': [result] if isinstance(result, str) else result,
              'type': artifact_type,
          })

  return generated


@faux.all_empty
@validate.require('build_target.name')
@validate.validation_complete
def Create(input_proto, output_proto, _config):
  """Create or replace a sysroot."""
  update_chroot = not input_proto.flags.chroot_current
  replace_sysroot = input_proto.flags.replace

  build_target = controller_util.ParseBuildTarget(input_proto.build_target,
                                                  input_proto.profile)
  package_indexes = [
      binpkg.PackageIndexInfo.from_protobuf(x)
      for x in input_proto.package_indexes
  ]
  run_configs = sysroot.SetupBoardRunConfig(
      force=replace_sysroot, upgrade_chroot=update_chroot,
      package_indexes=package_indexes)

  try:
    created = sysroot.Create(build_target, run_configs,
                             accept_licenses=_ACCEPTED_LICENSES)
  except sysroot.Error as e:
    cros_build_lib.Die(e)

  output_proto.sysroot.path = created.path
  output_proto.sysroot.build_target.name = build_target.name

  return controller.RETURN_CODE_SUCCESS


@faux.all_empty
@validate.require('build_target.name', 'packages')
@validate.require_each('packages', ['category', 'package_name'])
@validate.validation_complete
def GenerateArchive(input_proto, output_proto, _config):
  """Generate a sysroot. Typically used by informational builders."""
  build_target_name = input_proto.build_target.name
  pkg_list = []
  for package in input_proto.packages:
    pkg_list.append('%s/%s' % (package.category, package.package_name))

  with osutils.TempDir(delete=False) as temp_output_dir:
    sysroot_tar_path = sysroot.GenerateArchive(temp_output_dir,
                                               build_target_name,
                                               pkg_list)

  # By assigning this Path variable to the tar path, the tar file will be
  # copied out to the input_proto's ResultPath location.
  output_proto.sysroot_archive.path = sysroot_tar_path
  output_proto.sysroot_archive.location = common_pb2.Path.INSIDE


def _MockFailedPackagesResponse(_input_proto, output_proto, _config):
  """Mock error response that populates failed packages."""
  pkg = output_proto.failed_packages.add()
  pkg.package_name = 'package'
  pkg.category = 'category'
  pkg.version = '1.0.0_rc-r1'

  pkg2 = output_proto.failed_packages.add()
  pkg2.package_name = 'bar'
  pkg2.category = 'foo'
  pkg2.version = '3.7-r99'

  fail = output_proto.failed_package_data.add()
  fail.name.package_name = 'package'
  fail.name.category = 'category'
  fail.name.version = '1.0.0_rc-r1'
  fail.log_path.path = '/path/to/package:category-1.0.0_rc-r1:20210609-1337.log'
  fail.log_path.location = common_pb2.Path.INSIDE

  fail2 = output_proto.failed_package_data.add()
  fail2.name.package_name = 'bar'
  fail2.name.category = 'foo'
  fail2.name.version = '3.7-r99'
  fail2.log_path.path = '/path/to/foo:bar-3.7-r99:20210609-1620.log'
  fail2.log_path.location = common_pb2.Path.INSIDE


@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
@validate.require('sysroot.path', 'sysroot.build_target.name')
@validate.exists('sysroot.path')
@validate.validation_complete
def InstallToolchain(input_proto, output_proto, _config):
  """Install the toolchain into a sysroot."""
  compile_source = (
      input_proto.flags.compile_source or input_proto.flags.toolchain_changed)

  sysroot_path = input_proto.sysroot.path

  build_target = controller_util.ParseBuildTarget(
      input_proto.sysroot.build_target)
  target_sysroot = sysroot_lib.Sysroot(sysroot_path)
  run_configs = sysroot.SetupBoardRunConfig(usepkg=not compile_source)

  _LogBinhost(build_target.name)

  try:
    sysroot.InstallToolchain(build_target, target_sysroot, run_configs)
  except sysroot_lib.ToolchainInstallError as e:
    # Error installing - populate the failed package info.
    for pkg_info in e.failed_toolchain_info:
      # TODO(b/206514844): remove when field is deleted
      package_info_msg = output_proto.failed_packages.add()
      controller_util.serialize_package_info(pkg_info, package_info_msg)
      # Grab the paths to the log files for each failed package from the
      # sysroot.
      # Logs currently exist within the sysroot in the form of:
      # /build/${BOARD}/tmp/portage/logs/$CATEGORY:$PF:$TIMESTAMP.log
      failed_pkg_data_msg = output_proto.failed_package_data.add()
      controller_util.serialize_package_info(pkg_info, failed_pkg_data_msg.name)
      glob_path = os.path.join(target_sysroot.portage_logdir,
                               f'{pkg_info.category}:{pkg_info.pvr}:*.log')
      log_files = glob.glob(glob_path)
      log_files.sort(reverse=True)
      # Omit path if files don't exist for some reason
      if not log_files:
        logging.warning('Log file for %s was not found. Search path: %s',
                        pkg_info.cpvr, glob_path)
        continue
      failed_pkg_data_msg.log_path.path = log_files[0]
      failed_pkg_data_msg.log_path.location = common_pb2.Path.INSIDE

    return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE

  return controller.RETURN_CODE_SUCCESS


@faux.empty_success
@faux.error(_MockFailedPackagesResponse)
@validate.require('sysroot.build_target.name')
@validate.exists('sysroot.path')
@validate.require_each('packages', ['category', 'package_name'])
@validate.require_each('use_flags', ['flag'])
@validate.validation_complete
@metrics.collect_metrics
def InstallPackages(input_proto, output_proto, _config):
  """Install packages into a sysroot, building as necessary and permitted."""
  compile_source = (
      input_proto.flags.compile_source or input_proto.flags.toolchain_changed)

  use_remoteexec = bool(input_proto.remoteexec_config.reproxy_cfg_file and
                        input_proto.remoteexec_config.reclient_dir)

  # Testing if Goma will support unknown compilers now.
  use_goma = input_proto.flags.use_goma and not use_remoteexec

  target_sysroot = sysroot_lib.Sysroot(input_proto.sysroot.path)
  build_target = controller_util.ParseBuildTarget(
      input_proto.sysroot.build_target)

  # Get the package atom for each specified package. The field is optional, so
  # error only when we cannot parse an atom for each of the given packages.
  packages = [controller_util.PackageInfoToCPV(x).cp
              for x in input_proto.packages]

  package_indexes = [
      binpkg.PackageIndexInfo.from_protobuf(x)
      for x in input_proto.package_indexes
  ]

  # Calculate which packages would have been merged, but don't install anything.
  dryrun = input_proto.flags.dryrun

  if not target_sysroot.IsToolchainInstalled():
    cros_build_lib.Die('Toolchain must first be installed.')

  _LogBinhost(build_target.name)

  use_flags = [u.flag for u in input_proto.use_flags]
  build_packages_config = sysroot.BuildPackagesRunConfig(
      usepkg=not compile_source,
      install_debug_symbols=True,
      packages=packages,
      package_indexes=package_indexes,
      use_flags=use_flags,
      use_goma=use_goma,
      use_remoteexec=use_remoteexec,
      incremental_build=False,
      setup_board=False,
      dryrun=dryrun)

  try:
    sysroot.BuildPackages(build_target, target_sysroot, build_packages_config)
  except sysroot_lib.PackageInstallError as e:
    if not e.failed_packages:
      # No packages to report, so just exit with an error code.
      return controller.RETURN_CODE_COMPLETED_UNSUCCESSFULLY

    # We need to report the failed packages.
    for pkg_info in e.failed_packages:
      # TODO(b/206514844): remove when field is deleted
      package_info_msg = output_proto.failed_packages.add()
      controller_util.serialize_package_info(pkg_info, package_info_msg)
      # Grab the paths to the log files for each failed package from the
      # sysroot.
      # Logs currently exist within the sysroot in the form of:
      # /build/${BOARD}/tmp/portage/logs/$CATEGORY:$PF:$TIMESTAMP.log
      failed_pkg_data_msg = output_proto.failed_package_data.add()
      controller_util.serialize_package_info(pkg_info, failed_pkg_data_msg.name)
      glob_path = os.path.join(target_sysroot.portage_logdir,
                               f'{pkg_info.category}:{pkg_info.pvr}:*.log')
      log_files = glob.glob(glob_path)
      log_files.sort(reverse=True)
      # Omit path if files don't exist for some reason
      if not log_files:
        logging.warning('Log file for %s was not found. Search path: %s',
                        pkg_info.cpvr, glob_path)
        continue
      failed_pkg_data_msg.log_path.path = log_files[0]
      failed_pkg_data_msg.log_path.location = common_pb2.Path.INSIDE

    return controller.RETURN_CODE_UNSUCCESSFUL_RESPONSE_AVAILABLE
  finally:
    # Copy goma logs to specified directory if there is a goma_config and
    # it contains a log_dir to store artifacts.
    if input_proto.goma_config.log_dir.dir:
      # Get the goma log directory based on the GLOG_log_dir env variable.
      # TODO(crbug.com/1045001): Replace environment variable with query to
      # goma object after goma refactoring allows this.
      log_source_dir = os.getenv('GLOG_log_dir')
      if not log_source_dir:
        cros_build_lib.Die('GLOG_log_dir must be defined.')
      archiver = goma_lib.LogsArchiver(
          log_source_dir,
          dest_dir=input_proto.goma_config.log_dir.dir,
          stats_file=input_proto.goma_config.stats_file,
          counterz_file=input_proto.goma_config.counterz_file)
      archiver_tuple = archiver.Archive()
      if archiver_tuple.stats_file:
        output_proto.goma_artifacts.stats_file = archiver_tuple.stats_file
      if archiver_tuple.counterz_file:
        output_proto.goma_artifacts.counterz_file = archiver_tuple.counterz_file
      output_proto.goma_artifacts.log_files[:] = archiver_tuple.log_files

  # Return without populating the response if it is a dryrun.
  if dryrun:
    return controller.RETURN_CODE_SUCCESS

  # Read metric events log and pipe them into output_proto.events.
  deserialize_metrics_log(output_proto.events, prefix=build_target.name)


def _LogBinhost(board):
  """Log the portage binhost for the given board."""
  binhost = portage_util.PortageqEnvvar('PORTAGE_BINHOST', board=board,
                                        allow_undefined=True)
  if not binhost:
    logging.warning('Portage Binhost not found.')
  else:
    logging.info('Portage Binhost: %s', binhost)