scripts/upload_symbols.py

# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Upload all debug symbols required for crash reporting purposes.

This script need only be used to upload release builds symbols or to debug
crashes on non-release builds (in which case try to only upload the symbols
for those executables involved)."""

import ctypes
import logging
import multiprocessing
import os
import random
import textwrap
import tempfile
import time

from chromite.buildbot import constants
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import osutils
from chromite.lib import parallel


# URLs used for uploading symbols.
OFFICIAL_UPLOAD_URL = 'http://clients2.google.com/cr/symbol'
STAGING_UPLOAD_URL = 'http://clients2.google.com/cr/staging_symbol'


# The crash server rejects files that are this big.
CRASH_SERVER_FILE_LIMIT = 350 * 1024 * 1024
# Give ourselves a little breathing room from what the server expects.
DEFAULT_FILE_LIMIT = CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024)


# How long to wait (in seconds) for a single upload to complete.  This has
# to allow for symbols that are up to CRASH_SERVER_FILE_LIMIT in size.
UPLOAD_TIMEOUT = 30 * 60


# Sleep for 200ms in between uploads to avoid DoS'ing symbol server.
DEFAULT_SLEEP_DELAY = 0.2


# Number of seconds to wait before retrying an upload.  The delay will double
# for each subsequent retry of the same symbol file.
INITIAL_RETRY_DELAY = 1

# Allow up to 7 attempts to upload a symbol file (total delay may be
# 1+2+4+8+16+32=63 seconds).
MAX_RETRIES = 6

# Number of total errors, TOTAL_ERROR_COUNT, before retries are no longer
# attempted.  This is used to avoid lots of errors causing unreasonable delays.
MAX_TOTAL_ERRORS_FOR_RETRY = 3


def SymUpload(sym_file, upload_url):
  """Run breakpad sym_upload helper"""
  # TODO(vapier): Rewrite to use native python HTTP libraries.  This tool
  # reads the sym_file and does a HTTP post to URL with a few fields set.
  # See the tiny breakpad/tools/linux/symupload/sym_upload.cc for details.
  cmd = ['sym_upload', sym_file, upload_url]
  with cros_build_lib.SubCommandTimeout(UPLOAD_TIMEOUT):
    return cros_build_lib.RunCommandCaptureOutput(
        cmd, debug_level=logging.DEBUG)


def TestingSymUpload(sym_file, upload_url):
  """A stub version of SymUpload for --testing usage"""
  cmd = ['sym_upload', sym_file, upload_url]
  # Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
  returncode = random.randint(1, 100) <= 80
  cros_build_lib.Debug('would run (and return %i): %s', returncode,
                       ' '.join(map(repr, cmd)))
  if returncode:
    output = 'Failed to send the symbol file.'
  else:
    output = 'Successfully sent the symbol file.'
  result = cros_build_lib.CommandResult(cmd=cmd, error=None, output=output,
                                        returncode=returncode)
  if returncode:
    raise cros_build_lib.RunCommandError('forced test fail', result)
  else:
    return result


def UploadSymbol(sym_file, upload_url, file_limit=DEFAULT_FILE_LIMIT,
                 sleep=0, num_errors=None):
  """Upload |sym_file| to |upload_url|

  Args:
    sym_file: The full path to the breakpad symbol to upload
    upload_url: The crash server to upload things to
    file_limit: The max file size of a symbol file before we try to strip it
    sleep: Number of seconds to sleep before running
  """
  if num_errors is None:
    num_errors = ctypes.c_int()
  elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
    # Abandon ship!  It's on fire!  NOoooooooooooOOOoooooo.
    return 0

  upload_file = sym_file

  if sleep:
    # Keeps us from DoS-ing the symbol server.
    time.sleep(sleep)

  cros_build_lib.Debug('uploading %s' % sym_file)

  # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
  with tempfile.NamedTemporaryFile(prefix='upload_symbols',
                                   bufsize=0) as temp_sym_file:
    if file_limit:
      # If the symbols size is too big, strip out the call frame info.  The CFI
      # is unnecessary for 32bit x86 targets where the frame pointer is used (as
      # all of ours have) and it accounts for over half the size of the symbols
      # uploaded.
      file_size = os.path.getsize(sym_file)
      if file_size > file_limit:
        cros_build_lib.Warning('stripping CFI from %s due to size %s > %s',
                               sym_file, file_size, file_limit)
        temp_sym_file.writelines([x for x in open(sym_file, 'rb').readlines()
                                  if not x.startswith('STACK CFI')])
        upload_file = temp_sym_file.name

    # Hopefully the crash server will let it through.  But it probably won't.
    # Not sure what the best answer is in this case.
    file_size = os.path.getsize(upload_file)
    if file_size > CRASH_SERVER_FILE_LIMIT:
      cros_build_lib.PrintBuildbotStepWarnings()
      cros_build_lib.Error('upload file %s is awfully large, risking rejection '
                           'by symbol server (%s > %s)', sym_file, file_size,
                           CRASH_SERVER_FILE_LIMIT)
      num_errors.value += 1

    # Upload the symbol file.
    try:
      cros_build_lib.RetryCommand(SymUpload, MAX_RETRIES, upload_file,
                                  upload_url, sleep=INITIAL_RETRY_DELAY)
      cros_build_lib.Info('successfully uploaded %10i bytes: %s', file_size,
                          os.path.basename(sym_file))
    except cros_build_lib.RunCommandError as e:
      cros_build_lib.Warning('could not upload: %s:\n{stdout} %s\n{stderr} %s',
                             os.path.basename(sym_file), e.result.output,
                             e.result.error)
      num_errors.value += 1

  return num_errors.value


def UploadSymbols(board, official=False, breakpad_dir=None,
                  file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY,
                  upload_count=None):
  """Upload all the generated symbols for |board| to the crash server

  Args:
    board: The board whose symbols we wish to upload
    official: Use the official symbol server rather than the staging one
    breakpad_dir: The full path to the breakpad directory where symbols live
    file_limit: The max file size of a symbol file before we try to strip it
    sleep: How long to sleep in between uploads
    upload_count: If set, only upload this many symbols (meant for testing)
  Returns:
    False if some errors were encountered, True otherwise.
  """
  num_errors = 0

  if official:
    upload_url = OFFICIAL_UPLOAD_URL
  else:
    cros_build_lib.Warning('unofficial builds upload to the staging server')
    upload_url = STAGING_UPLOAD_URL

  if breakpad_dir is None:
    breakpad_dir = FindBreakpadDir(board)
  cros_build_lib.Info('uploading symbols to %s from %s', upload_url,
                      breakpad_dir)

  cros_build_lib.Info('uploading all breakpad symbol files')
  # We need to limit ourselves to one upload at a time to avoid the server
  # kicking in DoS protection.  See these bugs for more details:
  # http://crbug.com/209442
  # http://crbug.com/212496
  bg_errors = multiprocessing.Value('i')
  with parallel.BackgroundTaskRunner(UploadSymbol, file_limit=file_limit,
                                     sleep=sleep, num_errors=bg_errors,
                                     processes=1) as queue:
    for root, _, files in os.walk(breakpad_dir):
      if upload_count == 0:
        break

      for sym_file in files:
        if sym_file.endswith('.sym'):
          sym_file = os.path.join(root, sym_file)
          queue.put([sym_file, upload_url])

          if upload_count is not None:
            upload_count -= 1
            if upload_count == 0:
              break
  num_errors += bg_errors.value

  return num_errors


def GenerateBreakpadSymbols(board, breakpad_dir=None):
  """Generate all the symbols for this board

  Note: this should be merged with buildbot_commands.GenerateBreakpadSymbols()
  once we rewrite cros_generate_breakpad_symbols in python.

  Args:
    board: The board whose symbols we wish to generate
    breakpad_dir: The full path to the breakpad directory where symbols live
  """
  if breakpad_dir is None:
    breakpad_dir = FindBreakpadDir(board)

  cros_build_lib.Info('clearing out %s', breakpad_dir)
  osutils.RmDir(breakpad_dir, ignore_missing=True, sudo=True)

  cros_build_lib.Info('generating all breakpad symbol files')
  cmd = [os.path.join(constants.CROSUTILS_DIR,
                      'cros_generate_breakpad_symbols'),
         '--board', board]
  if cros_build_lib.logger.getEffectiveLevel() < logging.INFO:
    cmd += ['--verbose']
  result = cros_build_lib.RunCommand(cmd, error_code_ok=True)
  if result.returncode:
    cros_build_lib.Warning('errors hit while generating symbols; '
                           'uploading anyways')
    return 1

  return 0


def FindBreakpadDir(board):
  """Given a |board|, return the path to the breakpad dir for it"""
  return os.path.join('/build', board, 'usr', 'lib', 'debug', 'breakpad')


def main(argv):
  parser = commandline.ArgumentParser(description=__doc__)

  parser.add_argument('--board', default=None,
                      help='board to build packages for')
  parser.add_argument('--breakpad_root', type='path', default=None,
                      help='root directory for breakpad symbols')
  parser.add_argument('--official_build', action='store_true', default=False,
                      help='point to official symbol server')
  parser.add_argument('--regenerate', action='store_true', default=False,
                      help='regenerate all symbols')
  parser.add_argument('--upload-count', type=int, default=None,
                      help='only upload # number of symbols')
  parser.add_argument('--strip_cfi', type=int,
                      default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
                      help='strip CFI data for files above this size')
  parser.add_argument('--testing', action='store_true', default=False,
                      help='run in testing mode')
  parser.add_argument('--yes', action='store_true', default=False,
                      help='answer yes to all prompts')

  opts = parser.parse_args(argv)

  if opts.board is None:
    cros_build_lib.Die('--board is required')

  if opts.breakpad_root and opts.regenerate:
    cros_build_lib.Die('--regenerate may not be used with --breakpad_root')

  if opts.testing:
    # TODO(build): Kill off --testing mode once unittests are up-to-snuff.
    cros_build_lib.Info('running in testing mode')
    # pylint: disable=W0601,W0603
    global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
    INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
    SymUpload = TestingSymUpload

  if not opts.yes:
    query = textwrap.wrap(textwrap.dedent("""
        Uploading symbols for an entire Chromium OS build is really only
        necessary for release builds and in a few cases for developers
        to debug problems.  It will take considerable time to run.  For
        developer debugging purposes, consider instead passing specific
        files to upload.
    """), 80)
    cros_build_lib.Warning('\n%s', '\n'.join(query))
    if not cros_build_lib.BooleanPrompt(
        prompt='Are you sure you want to upload all build symbols',
        default=False):
      cros_build_lib.Die('better safe than sorry')

  ret = 0
  if opts.regenerate:
    ret += GenerateBreakpadSymbols(opts.board, breakpad_dir=opts.breakpad_root)

  ret += UploadSymbols(opts.board, official=opts.official_build,
                       breakpad_dir=opts.breakpad_root,
                       file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY,
                       upload_count=opts.upload_count)
  if ret:
    cros_build_lib.Error('encountered %i problem(s)', ret)
    # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
    # return 0 in case we are a multiple of the mask.
    ret = 1

  return ret