upload_symbols: rewrite in python
We want to do more stuff with upload_symbols. The current bash code is
limiting us though. Rewrite it in python and use chromite libraries.
BUG=chromium:209442
BUG=chromium:213204
TEST=`./buildbot/run_tests` passes
CQ-DEPEND=CL:57182
Change-Id: Ib10cb9c48bf2bf6f06a7b0400af4b1fe54ede856
Reviewed-on: https://gerrit.chromium.org/gerrit/58625
Reviewed-by: David James <davidjames@chromium.org>
Tested-by: Mike Frysinger <vapier@chromium.org>
Commit-Queue: Mike Frysinger <vapier@chromium.org>
diff --git a/scripts/upload_symbols.py b/scripts/upload_symbols.py
new file mode 100644
index 0000000..2b89a83
--- /dev/null
+++ b/scripts/upload_symbols.py
@@ -0,0 +1,304 @@
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Upload all debug symbols required for crash reporting purposes.
+
+This script need only be used to upload release builds symbols or to debug
+crashes on non-release builds (in which case try to only upload the symbols
+for those executables involved)."""
+
+import ctypes
+import logging
+import multiprocessing
+import os
+import random
+import textwrap
+import tempfile
+import time
+
+from chromite.buildbot import constants
+from chromite.lib import commandline
+from chromite.lib import cros_build_lib
+from chromite.lib import osutils
+from chromite.lib import parallel
+
+
+# URLs used for uploading symbols.
+OFFICIAL_UPLOAD_URL = 'http://clients2.google.com/cr/symbol'
+STAGING_UPLOAD_URL = 'http://clients2.google.com/cr/staging_symbol'
+
+
+# The crash server rejects files that are this big.
+CRASH_SERVER_FILE_LIMIT = 350 * 1024 * 1024
+# Give ourselves a little breathing room from what the server expects.
+DEFAULT_FILE_LIMIT = CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024)
+
+
+# Sleep for 200ms in between uploads to avoid DoS'ing symbol server.
+DEFAULT_SLEEP_DELAY = 0.2
+
+
+# Number of seconds to wait before retrying an upload. The delay will double
+# for each subsequent retry of the same symbol file.
+INITIAL_RETRY_DELAY = 1
+
+# Allow up to 7 attempts to upload a symbol file (total delay may be
+# 1+2+4+8+16+32=63 seconds).
+MAX_RETRIES = 6
+
+# Number of total errors, TOTAL_ERROR_COUNT, before retries are no longer
+# attempted. This is used to avoid lots of errors causing unreasonable delays.
+MAX_TOTAL_ERRORS_FOR_RETRY = 3
+
+
+def SymUpload(sym_file, upload_url):
+ """Run breakpad sym_upload helper"""
+ # TODO(vapier): Rewrite to use native python HTTP libraries. This tool
+ # reads the sym_file and does a HTTP post to URL with a few fields set.
+ # See the tiny breakpad/tools/linux/symupload/sym_upload.cc for details.
+ cmd = ['sym_upload', sym_file, upload_url]
+ return cros_build_lib.RunCommandCaptureOutput(cmd)
+
+
+def TestingSymUpload(sym_file, upload_url):
+ """A stub version of SymUpload for --testing usage"""
+ cmd = ['sym_upload', sym_file, upload_url]
+ # Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
+ returncode = random.randint(1, 100) <= 80
+ cros_build_lib.Debug('would run (and return %i): %s', returncode,
+ ' '.join(map(repr, cmd)))
+ if returncode:
+ output = 'Failed to send the symbol file.'
+ else:
+ output = 'Successfully sent the symbol file.'
+ result = cros_build_lib.CommandResult(cmd=cmd, error=None, output=output,
+ returncode=returncode)
+ if returncode:
+ raise cros_build_lib.RunCommandError('forced test fail', result)
+ else:
+ return result
+
+
+def UploadSymbol(sym_file, upload_url, file_limit=DEFAULT_FILE_LIMIT,
+ sleep=0, num_errors=None):
+ """Upload |sym_file| to |upload_url|
+
+ Args:
+ sym_file: The full path to the breakpad symbol to upload
+ upload_url: The crash server to upload things to
+ file_limit: The max file size of a symbol file before we try to strip it
+ sleep: Number of seconds to sleep before running
+ """
+ if num_errors is None:
+ num_errors = ctypes.c_int()
+ elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
+ # Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
+ return 0
+
+ upload_file = sym_file
+
+ if sleep:
+ # Keeps us from DoS-ing the symbol server.
+ time.sleep(sleep)
+
+ cros_build_lib.Debug('uploading %s' % sym_file)
+
+ # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
+ with tempfile.NamedTemporaryFile(prefix='upload_symbols',
+ bufsize=0) as temp_sym_file:
+ if file_limit:
+ # If the symbols size is too big, strip out the call frame info. The CFI
+ # is unnecessary for 32bit x86 targets where the frame pointer is used (as
+ # all of ours have) and it accounts for over half the size of the symbols
+ # uploaded.
+ file_size = os.path.getsize(sym_file)
+ if file_size > file_limit:
+ cros_build_lib.Warning('stripping CFI from %s due to size %s > %s',
+ sym_file, file_size, file_limit)
+ temp_sym_file.writelines([x for x in open(sym_file, 'rb').readlines()
+ if not x.startswith('STACK CFI')])
+ upload_file = temp_sym_file.name
+
+ # Hopefully the crash server will let it through. But it probably won't.
+ # Not sure what the best answer is in this case.
+ file_size = os.path.getsize(upload_file)
+ if file_size > CRASH_SERVER_FILE_LIMIT:
+ cros_build_lib.PrintBuildbotStepWarnings()
+ cros_build_lib.Error('upload file %s is awfully large, risking rejection '
+ 'by symbol server (%s > %s)', sym_file, file_size,
+ CRASH_SERVER_FILE_LIMIT)
+ num_errors.value += 1
+
+ # Upload the symbol file.
+ try:
+ cros_build_lib.RetryCommand(SymUpload, MAX_RETRIES, upload_file,
+ upload_url, sleep=INITIAL_RETRY_DELAY)
+ cros_build_lib.Info('successfully uploaded %10i bytes: %s', file_size,
+ os.path.basename(sym_file))
+ except cros_build_lib.RunCommandError as e:
+ cros_build_lib.Warning('could not upload: %s:\n{stdout} %s\n{stderr} %s',
+ os.path.basename(sym_file), e.result.output,
+ e.result.error)
+ num_errors.value += 1
+
+ return num_errors.value
+
+
+def UploadSymbols(board, official=False, breakpad_dir=None,
+ file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY,
+ upload_count=None):
+ """Upload all the generated symbols for |board| to the crash server
+
+ Args:
+ board: The board whose symbols we wish to upload
+ official: Use the official symbol server rather than the staging one
+ breakpad_dir: The full path to the breakpad directory where symbols live
+ file_limit: The max file size of a symbol file before we try to strip it
+ sleep: How long to sleep in between uploads
+ upload_count: If set, only upload this many symbols (meant for testing)
+ Returns:
+ False if some errors were encountered, True otherwise.
+ """
+ num_errors = 0
+
+ if official:
+ upload_url = OFFICIAL_UPLOAD_URL
+ else:
+ cros_build_lib.Warning('unofficial builds upload to the staging server')
+ upload_url = STAGING_UPLOAD_URL
+
+ if breakpad_dir is None:
+ breakpad_dir = FindBreakpadDir(board)
+ cros_build_lib.Info('uploading symbols to %s from %s', upload_url,
+ breakpad_dir)
+
+ cros_build_lib.Info('uploading all breakpad symbol files')
+ # We need to limit ourselves to one upload at a time to avoid the server
+ # kicking in DoS protection. See these bugs for more details:
+ # http://crbug.com/209442
+ # http://crbug.com/212496
+ bg_errors = multiprocessing.Value('i')
+ with parallel.BackgroundTaskRunner(UploadSymbol, file_limit=file_limit,
+ sleep=sleep, num_errors=bg_errors,
+ processes=1) as queue:
+ for root, _, files in os.walk(breakpad_dir):
+ if upload_count == 0:
+ break
+
+ for sym_file in files:
+ if sym_file.endswith('.sym'):
+ sym_file = os.path.join(root, sym_file)
+ queue.put([sym_file, upload_url])
+
+ if upload_count is not None:
+ upload_count -= 1
+ if upload_count == 0:
+ break
+ num_errors += bg_errors.value
+
+ return num_errors
+
+
+def GenerateBreakpadSymbols(board, breakpad_dir=None):
+ """Generate all the symbols for this board
+
+ Note: this should be merged with buildbot_commands.GenerateBreakpadSymbols()
+ once we rewrite cros_generate_breakpad_symbols in python.
+
+ Args:
+ board: The board whose symbols we wish to generate
+ breakpad_dir: The full path to the breakpad directory where symbols live
+ """
+ if breakpad_dir is None:
+ breakpad_dir = FindBreakpadDir(board)
+
+ cros_build_lib.Info('clearing out %s', breakpad_dir)
+ osutils.RmDir(breakpad_dir, ignore_missing=True, sudo=True)
+
+ cros_build_lib.Info('generating all breakpad symbol files')
+ cmd = [os.path.join(constants.CROSUTILS_DIR,
+ 'cros_generate_breakpad_symbols'),
+ '--board', board]
+ if cros_build_lib.logger.getEffectiveLevel() < logging.INFO:
+ cmd += ['--verbose']
+ result = cros_build_lib.RunCommand(cmd, error_code_ok=True)
+ if result.returncode:
+ cros_build_lib.Warning('errors hit while generating symbols; '
+ 'uploading anyways')
+ return 1
+
+ return 0
+
+
+def FindBreakpadDir(board):
+ """Given a |board|, return the path to the breakpad dir for it"""
+ return os.path.join('/build', board, 'usr', 'lib', 'debug', 'breakpad')
+
+
+def main(argv):
+ parser = commandline.ArgumentParser(description=__doc__)
+
+ parser.add_argument('--board', default=None,
+ help='board to build packages for')
+ parser.add_argument('--breakpad_root', type='path', default=None,
+ help='root directory for breakpad symbols')
+ parser.add_argument('--official_build', action='store_true', default=False,
+ help='point to official symbol server')
+ parser.add_argument('--regenerate', action='store_true', default=False,
+ help='regenerate all symbols')
+ parser.add_argument('--upload-count', type=int, default=None,
+ help='only upload # number of symbols')
+ parser.add_argument('--strip_cfi', type=int,
+ default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
+ help='strip CFI data for files above this size')
+ parser.add_argument('--testing', action='store_true', default=False,
+ help='run in testing mode')
+ parser.add_argument('--yes', action='store_true', default=False,
+ help='answer yes to all prompts')
+
+ opts = parser.parse_args(argv)
+
+ if opts.board is None:
+ cros_build_lib.Die('--board is required')
+
+ if opts.breakpad_root and opts.regenerate:
+ cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
+
+ if opts.testing:
+ # TODO(build): Kill off --testing mode once unittests are up-to-snuff.
+ cros_build_lib.Info('running in testing mode')
+ # pylint: disable=W0601,W0603
+ global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
+ INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
+ SymUpload = TestingSymUpload
+
+ if not opts.yes:
+ query = textwrap.wrap(textwrap.dedent("""
+ Uploading symbols for an entire Chromium OS build is really only
+ necessary for release builds and in a few cases for developers
+ to debug problems. It will take considerable time to run. For
+ developer debugging purposes, consider instead passing specific
+ files to upload.
+ """), 80)
+ cros_build_lib.Warning('\n%s', '\n'.join(query))
+ if not cros_build_lib.BooleanPrompt(
+ prompt='Are you sure you want to upload all build symbols',
+ default=False):
+ cros_build_lib.Die('better safe than sorry')
+
+ ret = 0
+ if opts.regenerate:
+ ret += GenerateBreakpadSymbols(opts.board, breakpad_dir=opts.breakpad_root)
+
+ ret += UploadSymbols(opts.board, official=opts.official_build,
+ breakpad_dir=opts.breakpad_root,
+ file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY,
+ upload_count=opts.upload_count)
+ if ret:
+ cros_build_lib.Error('encountered %i problem(s)', ret)
+ # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
+ # return 0 in case we are a multiple of the mask.
+ ret = 1
+
+ return ret