api/controller/toolchain.py

# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Toolchain-related operations."""

import collections
import logging
from pathlib import Path

from chromite.api import controller
from chromite.api import faux
from chromite.api import validate
from chromite.api.controller import controller_util
from chromite.api.gen.chromite.api import toolchain_pb2
from chromite.api.gen.chromite.api.artifacts_pb2 import PrepareForBuildResponse
from chromite.api.gen.chromiumos.builder_config_pb2 import BuilderConfig
from chromite.lib import toolchain_util
from chromite.service import toolchain


# TODO(b/229665884): Move the implementation details for most/all endpoints to:
#   chromite/services/toolchain.py
# This migration has been done for linting endpoints but not yet for others.

_Handlers = collections.namedtuple('_Handlers', ['name', 'prepare', 'bundle'])
_TOOLCHAIN_ARTIFACT_HANDLERS = {
    BuilderConfig.Artifacts.UNVERIFIED_CHROME_LLVM_ORDERFILE:
        _Handlers('UnverifiedChromeLlvmOrderfile',
                  toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.VERIFIED_CHROME_LLVM_ORDERFILE:
        _Handlers('VerifiedChromeLlvmOrderfile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.CHROME_CLANG_WARNINGS_FILE:
        _Handlers('ChromeClangWarningsFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.UNVERIFIED_LLVM_PGO_FILE:
        _Handlers('UnverifiedLlvmPgoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.UNVERIFIED_CHROME_BENCHMARK_AFDO_FILE:
        _Handlers('UnverifiedChromeBenchmarkAfdoFile',
                  toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.CHROME_DEBUG_BINARY:
        _Handlers('ChromeDebugBinary', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.UNVERIFIED_CHROME_BENCHMARK_PERF_FILE:
        _Handlers('UnverifiedChromeBenchmarkPerfFile',
                  toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.VERIFIED_CHROME_BENCHMARK_AFDO_FILE:
        _Handlers('VerifiedChromeBenchmarkAfdoFile',
                  toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.UNVERIFIED_KERNEL_CWP_AFDO_FILE:
        _Handlers('UnverifiedKernelCwpAfdoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.VERIFIED_KERNEL_CWP_AFDO_FILE:
        _Handlers('VerifiedKernelCwpAfdoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.UNVERIFIED_CHROME_CWP_AFDO_FILE:
        _Handlers('UnverifiedChromeCwpAfdoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.VERIFIED_CHROME_CWP_AFDO_FILE:
        _Handlers('VerifiedChromeCwpAfdoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.VERIFIED_RELEASE_AFDO_FILE:
        _Handlers('VerifiedReleaseAfdoFile', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.TOOLCHAIN_WARNING_LOGS:
        _Handlers('ToolchainWarningLogs', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.CHROME_AFDO_PROFILE_FOR_ANDROID_LINUX:
        _Handlers('ChromeAFDOProfileForAndroidLinux',
                  toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.CLANG_CRASH_DIAGNOSES:
        _Handlers('ClangCrashDiagnoses', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
    BuilderConfig.Artifacts.COMPILER_RUSAGE_LOG:
        _Handlers('CompilerRusageLogs', toolchain_util.PrepareForBuild,
                  toolchain_util.BundleArtifacts),
}

_TOOLCHAIN_COMMIT_HANDLERS = {
    BuilderConfig.Artifacts.VERIFIED_KERNEL_CWP_AFDO_FILE:
        'VerifiedKernelCwpAfdoFile'
}


# TODO(crbug/1031213): When @faux is expanded to have more than success/failure,
# this should be changed.
@faux.all_empty
@validate.require('artifact_types')
# Note: chroot and sysroot are unspecified the first time that the build_target
# recipe calls PrepareForBuild.  The second time, they are specified.  No
# validation check because "all" values are valid.
@validate.validation_complete
def PrepareForBuild(input_proto, output_proto, _config):
  """Prepare to build toolchain artifacts.

  The handlers (from _TOOLCHAIN_ARTIFACT_HANDLERS above) are called with:
      artifact_name (str): name of the artifact type.
      chroot (chroot_lib.Chroot): chroot.  Will be None if the chroot has not
          yet been created.
      sysroot_path (str): sysroot path inside the chroot (e.g., /build/atlas).
          Will be an empty string if the sysroot has not yet been created.
      build_target_name (str): name of the build target (e.g., atlas).  Will be
          an empty string if the sysroot has not yet been created.
      input_artifacts ({(str) name:[str gs_locations]}): locations for possible
          input artifacts.  The handler is expected to know which keys it should
          be using, and ignore any keys that it does not understand.
      profile_info ({(str) name: (str) value}) Dictionary containing profile
          information.

  They locate and modify any ebuilds and/or source required for the artifact
  being created, then return a value from toolchain_util.PrepareForBuildReturn.

  This function sets output_proto.build_relevance to the result.

  Args:
    input_proto (PrepareForToolchainBuildRequest): The input proto
    output_proto (PrepareForToolchainBuildResponse): The output proto
    _config (api_config.ApiConfig): The API call config.
  """
  if input_proto.chroot.path:
    chroot = controller_util.ParseChroot(input_proto.chroot)
  else:
    chroot = None

  input_artifacts = collections.defaultdict(list)
  for art in input_proto.input_artifacts:
    item = _TOOLCHAIN_ARTIFACT_HANDLERS.get(art.input_artifact_type)
    if item:
      input_artifacts[item.name].extend(
          ['gs://%s' % str(x) for x in art.input_artifact_gs_locations])

  profile_info = _GetProfileInfoDict(input_proto.profile_info)

  results = set()
  sysroot_path = input_proto.sysroot.path
  build_target = input_proto.sysroot.build_target.name
  for artifact_type in input_proto.artifact_types:
    # Unknown artifact_types are an error.
    handler = _TOOLCHAIN_ARTIFACT_HANDLERS[artifact_type]
    if handler.prepare:
      results.add(
          handler.prepare(handler.name, chroot, sysroot_path, build_target,
                          input_artifacts, profile_info))

  # Translate the returns from the handlers we called.
  #   If any NEEDED => NEEDED
  #   elif any UNKNOWN => UNKNOWN
  #   elif any POINTLESS => POINTLESS
  #   else UNKNOWN.
  if toolchain_util.PrepareForBuildReturn.NEEDED in results:
    output_proto.build_relevance = PrepareForBuildResponse.NEEDED
  elif toolchain_util.PrepareForBuildReturn.UNKNOWN in results:
    output_proto.build_relevance = PrepareForBuildResponse.UNKNOWN
  elif toolchain_util.PrepareForBuildReturn.POINTLESS in results:
    output_proto.build_relevance = PrepareForBuildResponse.POINTLESS
  else:
    output_proto.build_relevance = PrepareForBuildResponse.UNKNOWN
  return controller.RETURN_CODE_SUCCESS


# TODO(crbug/1031213): When @faux is expanded to have more than success/failure,
# this should be changed.
@faux.all_empty
@validate.require('chroot.path', 'output_dir', 'artifact_types')
@validate.exists('output_dir')
@validate.validation_complete
def BundleArtifacts(input_proto, output_proto, _config):
  """Bundle valid toolchain artifacts.

  The handlers (from _TOOLCHAIN_ARTIFACT_HANDLERS above) are called with:
      artifact_name (str): name of the artifact type
      chroot (chroot_lib.Chroot): chroot
      sysroot_path (str): sysroot path inside the chroot (e.g., /build/atlas),
          or None.
      chrome_root (str): path to chrome root. (e.g., /b/s/w/ir/k/chrome)
      build_target_name (str): name of the build target (e.g., atlas), or None.
      output_dir (str): absolute path where artifacts are being bundled.
        (e.g., /b/s/w/ir/k/recipe_cleanup/artifactssptfMU)
      profile_info ({(str) name: (str) value}) Dictionary containing profile
          information.

  Note: the actual upload to GS is done by CI, not here.

  Args:
    input_proto (BundleToolchainRequest): The input proto
    output_proto (BundleToolchainResponse): The output proto
    _config (api_config.ApiConfig): The API call config.
  """
  chroot = controller_util.ParseChroot(input_proto.chroot)

  profile_info = _GetProfileInfoDict(input_proto.profile_info)

  output_path = Path(input_proto.output_dir)

  for artifact_type in input_proto.artifact_types:
    if artifact_type not in _TOOLCHAIN_ARTIFACT_HANDLERS:
      logging.error('%s not understood', artifact_type)
      return controller.RETURN_CODE_UNRECOVERABLE

    handler = _TOOLCHAIN_ARTIFACT_HANDLERS[artifact_type]
    if not handler or not handler.bundle:
      logging.warning('%s does not have a handler with a bundle function.',
                      artifact_type)
      continue

    artifacts = handler.bundle(handler.name, chroot, input_proto.sysroot.path,
                               input_proto.sysroot.build_target.name,
                               input_proto.output_dir, profile_info)
    if not artifacts:
      continue

    # Filter out artifacts that do not exist or are empty.
    usable_artifacts = []
    for artifact in artifacts:
      artifact_path = output_path / artifact
      if not artifact_path.exists():
        logging.warning('%s is not in the output directory.', artifact)
      elif not artifact_path.stat().st_size:
        logging.warning('%s is empty.', artifact)
      else:
        usable_artifacts.append(artifact)

    if not usable_artifacts:
      logging.warning('No usable artifacts for artifact type %s', artifact_type)
      continue

    # Add all usable artifacts.
    art_info = output_proto.artifacts_info.add()
    art_info.artifact_type = artifact_type
    for artifact in usable_artifacts:
      art_info.artifacts.add().path = artifact


def _GetUpdatedFilesResponse(_input_proto, output_proto, _config):
  """Add successful status to the faux response."""
  file_info = output_proto.updated_files.add()
  file_info.path = '/any/modified/file'
  output_proto.commit_message = 'Commit message'


@faux.empty_error
@faux.success(_GetUpdatedFilesResponse)
@validate.require('uploaded_artifacts')
@validate.validation_complete
def GetUpdatedFiles(input_proto, output_proto, _config):
  """Use uploaded artifacts to update some updates in a chromeos checkout.

  The function will call toolchain_util.GetUpdatedFiles using the type of
  uploaded artifacts to make some changes in a checkout, and return the list
  of change files together with commit message.
     updated_artifacts: A list of UpdatedArtifacts type which contains a tuple
        of artifact info and profile info.
  Note: the actual creation of the commit is done by CI, not here.

  Args:
    input_proto (GetUpdatedFilesRequest): The input proto
    output_proto (GetUpdatedFilesResponse): The output proto
    _config (api_config.ApiConfig): The API call config.
  """
  commit_message = ''
  for artifact in input_proto.uploaded_artifacts:
    artifact_type = artifact.artifact_info.artifact_type
    if artifact_type not in _TOOLCHAIN_COMMIT_HANDLERS:
      logging.error('%s not understood', artifact_type)
      return controller.RETURN_CODE_UNRECOVERABLE
    artifact_name = _TOOLCHAIN_COMMIT_HANDLERS[artifact_type]
    if artifact_name:
      assert len(artifact.artifact_info.artifacts) == 1, (
          'Only one file to update per each artifact')
      updated_files, message = toolchain_util.GetUpdatedFiles(
          artifact_name, artifact.artifact_info.artifacts[0].path,
          _GetProfileInfoDict(artifact.profile_info))
      for f in updated_files:
        file_info = output_proto.updated_files.add()
        file_info.path = f

      commit_message += message + '\n'
    output_proto.commit_message = commit_message
    # No commit footer is added for now. Can add more here if needed


def _GetProfileInfoDict(profile_info):
  """Convert profile_info to a dict.

  Args:
    profile_info (ArtifactProfileInfo): The artifact profile_info.

  Returns:
    A dictionary containing profile info.
  """
  ret = {}
  which = profile_info.WhichOneof('artifact_profile_info')
  if which:
    value = getattr(profile_info, which)
    # If it is a message, then use the contents of the message.  This works as
    # long as simple types do not have a 'DESCRIPTOR' attribute. (And protobuf
    # messages do.)
    if getattr(value, 'DESCRIPTOR', None):
      ret.update({k.name: v for k, v in value.ListFields()})
    else:
      ret[which] = value
  return ret


LINTER_CODES = {
    'clang_tidy': toolchain_pb2.LinterFinding.CLANG_TIDY,
    'cargo_clippy': toolchain_pb2.LinterFinding.CARGO_CLIPPY
}


@faux.all_empty
@validate.exists('sysroot.path')
@validate.require('packages')
@validate.validation_complete
def EmergeWithLinting(input_proto, output_proto, _config):
  """Emerge packages with linter features enabled and retrieves all findings.

  Args:
    input_proto (LinterRequest): The nput proto with package and sysroot info.
    output_proto (LinterResponse): The output proto where findings are stored.
    _config (api_config.ApiConfig): The API call config (unused).
  """
  packages = [
      controller_util.deserialize_package_info(package)
      for package in input_proto.packages
  ]

  build_linter = toolchain.BuildLinter(
      packages,
      input_proto.sysroot.path,
      differential=input_proto.filter_modified)

  findings = build_linter.emerge_with_linting(use_clippy=True, use_tidy=True)

  for finding in findings:
    locations = []
    for location in finding.locations:
      locations.append(
          toolchain_pb2.LinterFindingLocation(
              filepath=location.filepath,
              line_start=location.line_start,
              line_end=location.line_end))
    output_proto.findings.append(
        toolchain_pb2.LinterFinding(
            message=finding.message,
            locations=locations,
            linter=LINTER_CODES[finding.linter]))