scripts/cros_install_debug_syms.py

# -*- coding: utf-8 -*-
# Copyright 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Install debug symbols for specified packages.

Only reinstall the debug symbols if they are not already installed to save time.

The debug symbols are packaged outside of the prebuilt package in a
.debug.tbz2 archive when FEATURES=separatedebug is set (by default on
builders). On local machines, separatedebug is not set and the debug symbols
are part of the prebuilt package.
"""

from __future__ import print_function

import argparse
import functools
import multiprocessing
import os
import pickle
import sys
import tempfile

from six.moves import urllib

from chromite.lib import binpkg
from chromite.lib import cache
from chromite.lib import commandline
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import gs
from chromite.utils import outcap

if cros_build_lib.IsInsideChroot():
  # pylint: disable=import-error
  from portage import create_trees


assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'


DEBUG_SYMS_EXT = '.debug.tbz2'

# We cache the package indexes. When the format of what we store changes,
# bump the cache version to avoid problems.
CACHE_VERSION = '1'


class DebugSymbolsInstaller(object):
  """Container for enviromnent objects, needed to make multiprocessing work."""

  def __init__(self, vartree, gs_context, sysroot, stdout_to_null):
    self._vartree = vartree
    self._gs_context = gs_context
    self._sysroot = sysroot
    self._stdout_to_null = stdout_to_null
    self._capturer = outcap.OutputCapturer()

  def __enter__(self):
    if self._stdout_to_null:
      self._capturer.StartCapturing()

    return self

  def __exit__(self, _exc_type, _exc_val, _exc_tb):
    if self._stdout_to_null:
      self._capturer.StopCapturing()

  def Install(self, cpv, url):
    """Install the debug symbols for |cpv|.

    This will install the debug symbols tarball in PKGDIR so that it can be
    used later.

    Args:
      cpv: the cpv of the package to build. This assumes that the cpv is
        installed in the sysroot.
      url: url of the debug symbols archive. This could be a Google Storage url
        or a local path.
    """
    archive = os.path.join(self._vartree.settings['PKGDIR'],
                           cpv + DEBUG_SYMS_EXT)
    # GsContext does not understand file:// scheme so we need to extract the
    # path ourselves.
    parsed_url = urllib.parse.urlsplit(url)
    if not parsed_url.scheme or parsed_url.scheme == 'file':
      url = parsed_url.path

    if not os.path.isfile(archive):
      self._gs_context.Copy(url, archive, debug_level=logging.DEBUG)

    with osutils.TempDir(sudo_rm=True) as tempdir:
      cros_build_lib.sudo_run(
          ['tar', '-I', 'bzip2 -q', '-xf', archive, '-C', tempdir], quiet=True)

      with open(self._vartree.getpath(cpv, filename='CONTENTS'),
                'a') as content_file:
        # Merge the content of the temporary dir into the sysroot.
        # pylint: disable=protected-access
        link = self._vartree.dbapi._dblink(cpv)
        link.mergeme(tempdir, self._sysroot, content_file, None, '', {}, None)


def ShouldGetSymbols(cpv, vardb, remote_symbols):
  """Return True if the symbols for cpv are available and are not installed.

  We try to check if the symbols are installed before checking availability as
  a GS request is more expensive than checking locally.

  Args:
    cpv: cpv of the package
    vardb: a vartree dbapi
    remote_symbols: a mapping from cpv to debug symbols url

  Returns:
    True if |cpv|'s debug symbols are not installed and are available
  """
  features, contents = vardb.aux_get(cpv, ['FEATURES', 'CONTENTS'])

  return ('separatedebug' in features and not '/usr/lib/debug/' in contents and
          cpv in remote_symbols)


def RemoteSymbols(vartree, binhost_cache=None):
  """Get the cpv to debug symbols mapping.

  If several binhost contain debug symbols for the same cpv, keep only the
  highest priority one.

  Args:
    vartree: a vartree
    binhost_cache: a cache containing the cpv to debug symbols url for all
      known binhosts. None if we are not caching binhosts.

  Returns:
    a dictionary mapping the cpv to a remote debug symbols gsurl.
  """
  symbols_mapping = {}
  for binhost in vartree.settings['PORTAGE_BINHOST'].split():
    if binhost:
      symbols_mapping.update(ListBinhost(binhost, binhost_cache))
  return symbols_mapping


def GetPackageIndex(binhost, binhost_cache=None):
  """Get the packages index for |binhost|.

  If a cache is provided, use it to a cache remote packages index.

  Args:
    binhost: a portage binhost, local, google storage or http.
    binhost_cache: a cache for the remote packages index.

  Returns:
    A PackageIndex object.
  """
  key = binhost.split('://')[-1]
  key = key.rstrip('/').split('/')

  if binhost_cache and binhost_cache.Lookup(key).Exists():
    with open(binhost_cache.Lookup(key).path, 'rb') as f:
      return pickle.load(f)

  pkgindex = binpkg.GrabRemotePackageIndex(binhost, quiet=True)
  if pkgindex and binhost_cache:
    # Only cache remote binhosts as local binhosts can change.
    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
      pickle.dump(pkgindex, temp_file)
      temp_file.file.close()
      binhost_cache.Lookup(key).Assign(temp_file.name)
  elif pkgindex is None:
    urlparts = urllib.parse.urlsplit(binhost)
    if urlparts.scheme not in ('file', ''):
      # Don't fail the build on network errors. Print a warning message and
      # continue.
      logging.warning('Could not get package index %s', binhost)
      return None

    binhost = urlparts.path
    if not os.path.isdir(binhost):
      raise ValueError('unrecognized binhost format for %s.')
    pkgindex = binpkg.GrabLocalPackageIndex(binhost)

  return pkgindex


def ListBinhost(binhost, binhost_cache=None):
  """Return the cpv to debug symbols mapping for a given binhost.

  List the content of the binhost to extract the cpv to debug symbols
  mapping. If --cachebinhost is set, we cache the result to avoid the
  cost of gsutil every time.

  Args:
    binhost: a portage binhost, local or on google storage.
    binhost_cache: a cache containing mappings cpv to debug symbols url for a
      given binhost (None if we don't want to cache).

  Returns:
    A cpv to debug symbols url mapping.
  """

  symbols = {}
  pkgindex = GetPackageIndex(binhost, binhost_cache)
  if pkgindex is None:
    return symbols

  for p in pkgindex.packages:
    if p.get('DEBUG_SYMBOLS') == 'yes':
      path = p.get('PATH', p['CPV'] + '.tbz2')
      base_url = pkgindex.header.get('URI', binhost)
      symbols[p['CPV']] = os.path.join(base_url,
                                       path.replace('.tbz2', DEBUG_SYMS_EXT))

  return symbols


def GetMatchingCPV(package, vardb):
  """Return the cpv of the installed package matching |package|.

  Args:
    package: package name
    vardb: a vartree dbapi

  Returns:
    The cpv of the installed package whose name matchex |package|.
  """
  matches = vardb.match(package)
  if not matches:
    cros_build_lib.Die('Could not find package %s' % package)
  if len(matches) != 1:
    cros_build_lib.Die('Ambiguous package name: %s.\n'
                       'Matching: %s' % (package, ' '.join(matches)))
  return matches[0]


def GetVartree(sysroot):
  """Get the portage vartree.

  This must be called in subprocesses only. The vartree is not serializable,
  and is implemented as a singleton in portage, so it always breaks the
  parallel call when initialized before it is called.
  """
  trees = create_trees(target_root=sysroot, config_root=sysroot)
  return trees[sysroot]['vartree']


def GetBinhostCache(options):
  """Get and optionally clear the binhost cache."""
  cache_dir = os.path.join(path_util.FindCacheDir(),
                           'cros_install_debug_syms-v' + CACHE_VERSION)
  if options.clearcache:
    osutils.RmDir(cache_dir, ignore_missing=True)

  binhost_cache = None
  if options.cachebinhost:
    binhost_cache = cache.DiskCache(cache_dir)

  return binhost_cache


def GetInstallArgs(options, sysroot):
  """Resolve the packages that are to have their debug symbols installed.

  This function must be called in subprocesses only since it requires the
  portage vartree.
  See: GetVartree

  Returns:
    list[(pkg, binhost_url)]
  """
  vartree = GetVartree(sysroot)
  symbols_mapping = RemoteSymbols(vartree, GetBinhostCache(options))

  if options.all:
    to_install = vartree.dbapi.cpv_all()
  else:
    to_install = [GetMatchingCPV(p, vartree.dbapi) for p in options.packages]

  to_install = [
      p for p in to_install
      if ShouldGetSymbols(p, vartree.dbapi, symbols_mapping)
  ]

  return [(p, symbols_mapping[p]) for p in to_install]


def ListInstallArgs(options, sysroot):
  """List the args for the calling process."""
  lines = ['%s %s' % arg for arg in GetInstallArgs(options, sysroot)]
  print('\n'.join(lines))


def GetInstallArgsList(argv):
  """Get the install args from the --list reexec of the command."""
  # Insert the --list as the first argument to prevent parsing --list as a
  # package when a package is given.
  cmd = [argv[0]] + ['--list'] + argv[1:]
  result = cros_build_lib.run(cmd, capture_output=True, encoding='utf-8')
  lines = result.output.splitlines()
  return [line.split() for line in lines if line]


def _InstallOne(sysroot, debug, args):
  """Parallelizable wrapper for the DebugSymbolsInstaller.Install method."""
  vartree = GetVartree(sysroot)
  gs_context = gs.GSContext(boto_file=vartree.settings['BOTO_CONFIG'])
  with DebugSymbolsInstaller(vartree, gs_context, sysroot,
                             not debug) as installer:
    installer.Install(*args)


def GetParser():
  """Build the argument parser."""
  parser = commandline.ArgumentParser(description=__doc__)
  parser.add_argument('--board', help='Board name (required).', required=True)
  parser.add_argument(
      '--all',
      action='store_true',
      default=False,
      help='Install the debug symbols for all installed packages')
  parser.add_argument(
      'packages',
      nargs=argparse.REMAINDER,
      help='list of packages that need the debug symbols.')

  advanced = parser.add_argument_group('Advanced options')
  advanced.add_argument(
      '--nocachebinhost',
      dest='cachebinhost',
      default=True,
      action='store_false',
      help="Don't cache the list of files contained in binhosts. "
           '(Default: cache)')
  advanced.add_argument(
      '--clearcache',
      action='store_true',
      default=False,
      help='Clear the binhost cache.')
  advanced.add_argument(
      '-j',
      '--jobs',
      default=multiprocessing.cpu_count(),
      type=int,
      help='Number of processes to run in parallel.')
  advanced.add_argument(
      '--list',
      action='store_true',
      default=False,
      help='List the packages whose debug symbols would be installed and '
           'their binhost path.')

  return parser


def ParseArgs(argv):
  """Parse and validate arguments."""
  parser = GetParser()

  options = parser.parse_args(argv)
  if options.all and options.packages:
    parser.error('Cannot use --all with a list of packages')

  options.Freeze()

  return options


def main(argv):
  if not cros_build_lib.IsInsideChroot():
    raise commandline.ChrootRequiredError(argv)

  cmd = [os.path.join(constants.CHROMITE_BIN_DIR,
                      'cros_install_debug_syms')] + argv
  if os.geteuid() != 0:
    cros_build_lib.sudo_run(cmd)
    return

  options = ParseArgs(argv)

  # sysroot must have a trailing / as the tree dictionary produced by
  # create_trees in indexed with a trailing /.
  sysroot = cros_build_lib.GetSysroot(options.board) + '/'

  if options.list:
    ListInstallArgs(options, sysroot)
    return

  args = GetInstallArgsList(cmd)

  if not args:
    logging.info('No packages found needing debug symbols.')
    return

  # Partial to simplify the arguments to parallel since the first two are the
  # same for every call.
  partial_install = functools.partial(_InstallOne, sysroot, options.debug)
  pool = multiprocessing.Pool(processes=options.jobs)
  pool.map(partial_install, args)

  logging.debug('installation done, updating packages index file')
  packages_dir = os.path.join(sysroot, 'packages')
  packages_file = os.path.join(packages_dir, 'Packages')
  # binpkg will set DEBUG_SYMBOLS automatically if it detects the debug symbols
  # in the packages dir.
  pkgindex = binpkg.GrabLocalPackageIndex(packages_dir)
  with open(packages_file, 'w') as p:
    pkgindex.Write(p)