Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Upload all debug symbols required for crash reporting purposes. |
| 6 | |
| 7 | This script need only be used to upload release builds symbols or to debug |
| 8 | crashes on non-release builds (in which case try to only upload the symbols |
Mike Frysinger | 02e1e07 | 2013-11-10 22:11:34 -0500 | [diff] [blame] | 9 | for those executables involved). |
| 10 | """ |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 11 | |
Mike Frysinger | a4fa1e8 | 2014-01-15 01:45:56 -0500 | [diff] [blame] | 12 | from __future__ import print_function |
| 13 | |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 14 | import collections |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 15 | import ctypes |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 16 | import datetime |
Mike Frysinger | dad2a03 | 2014-05-11 23:05:11 -0400 | [diff] [blame] | 17 | import errno |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 18 | import functools |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 19 | import hashlib |
Mike Frysinger | a4fa1e8 | 2014-01-15 01:45:56 -0500 | [diff] [blame] | 20 | import httplib |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 21 | import multiprocessing |
| 22 | import os |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 23 | import poster |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 24 | try: |
| 25 | import Queue |
| 26 | except ImportError: |
| 27 | # Python-3 renamed to "queue". We still use Queue to avoid collisions |
| 28 | # with naming variables as "queue". Maybe we'll transition at some point. |
| 29 | # pylint: disable=F0401 |
| 30 | import queue as Queue |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 31 | import random |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 32 | import signal |
Mike Frysinger | fd35565 | 2014-01-23 02:57:48 -0500 | [diff] [blame] | 33 | import socket |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 34 | import sys |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 35 | import textwrap |
| 36 | import tempfile |
| 37 | import time |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 38 | import urllib2 |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 39 | import urlparse |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 40 | |
Don Garrett | 88b8d78 | 2014-05-13 17:30:55 -0700 | [diff] [blame] | 41 | from chromite.cbuildbot import constants |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 42 | from chromite.lib import cache |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 43 | from chromite.lib import commandline |
| 44 | from chromite.lib import cros_build_lib |
Ralph Nathan | 5a582ff | 2015-03-20 18:18:30 -0700 | [diff] [blame] | 45 | from chromite.lib import cros_logging as logging |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 46 | from chromite.lib import gs |
| 47 | from chromite.lib import osutils |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 48 | from chromite.lib import parallel |
Gilad Arnold | 83233ed | 2015-05-08 12:12:13 -0700 | [diff] [blame] | 49 | from chromite.lib import path_util |
David James | c93e6a4d | 2014-01-13 11:37:36 -0800 | [diff] [blame] | 50 | from chromite.lib import retry_util |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 51 | from chromite.lib import signals |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 52 | from chromite.lib import timeout_util |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 53 | from chromite.scripts import cros_generate_breakpad_symbols |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 54 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 55 | # Needs to be after chromite imports. |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 56 | # We don't want to import the general keyring module as that will implicitly |
| 57 | # try to import & connect to a dbus server. That's a waste of time. |
| 58 | sys.modules['keyring'] = None |
| 59 | import isolateserver |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 60 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 61 | |
| 62 | # URLs used for uploading symbols. |
| 63 | OFFICIAL_UPLOAD_URL = 'http://clients2.google.com/cr/symbol' |
| 64 | STAGING_UPLOAD_URL = 'http://clients2.google.com/cr/staging_symbol' |
| 65 | |
| 66 | |
| 67 | # The crash server rejects files that are this big. |
| 68 | CRASH_SERVER_FILE_LIMIT = 350 * 1024 * 1024 |
| 69 | # Give ourselves a little breathing room from what the server expects. |
| 70 | DEFAULT_FILE_LIMIT = CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024) |
| 71 | |
| 72 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 73 | # The batch limit when talking to the dedup server. We avoid sending one at a |
| 74 | # time as the round trip overhead will dominate. Conversely, we avoid sending |
| 75 | # all at once so we can start uploading symbols asap -- the symbol server is a |
| 76 | # bit slow and will take longer than anything else. |
| 77 | # TODO: A better algorithm would be adaptive. If we have more than one symbol |
| 78 | # in the upload queue waiting, we could send more symbols to the dedupe server |
| 79 | # at a time. |
| 80 | DEDUPE_LIMIT = 100 |
| 81 | |
| 82 | # How long to wait for the server to respond with the results. Note that the |
| 83 | # larger the limit above, the larger this will need to be. So we give it ~1 |
| 84 | # second per item max. |
| 85 | DEDUPE_TIMEOUT = DEDUPE_LIMIT |
| 86 | |
Mike Frysinger | 4dd462e | 2014-04-30 16:21:51 -0400 | [diff] [blame] | 87 | # How long to wait for the notification to finish (in minutes). If it takes |
| 88 | # longer than this, we'll stop notifiying, but that's not a big deal as we |
| 89 | # will be able to recover in later runs. |
| 90 | DEDUPE_NOTIFY_TIMEOUT = 20 |
| 91 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 92 | # The unique namespace in the dedupe server that only we use. Helps avoid |
| 93 | # collisions with all the hashed values and unrelated content. |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 94 | OFFICIAL_DEDUPE_NAMESPACE_TMPL = '%s-upload-symbols' |
| 95 | STAGING_DEDUPE_NAMESPACE_TMPL = '%s-staging' % OFFICIAL_DEDUPE_NAMESPACE_TMPL |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 96 | |
| 97 | |
Mike Frysinger | 7104666 | 2014-09-12 18:15:15 -0700 | [diff] [blame] | 98 | # The minimum average rate (in bytes per second) that we expect to maintain |
| 99 | # when uploading symbols. This has to allow for symbols that are up to |
| 100 | # CRASH_SERVER_FILE_LIMIT in size. |
| 101 | UPLOAD_MIN_RATE = CRASH_SERVER_FILE_LIMIT / (30 * 60) |
| 102 | |
| 103 | # The lowest timeout (in seconds) we'll allow. If the server is overloaded, |
| 104 | # then there might be a delay in setting up the connection, not just with the |
| 105 | # transfer. So even a small file might need a larger value. |
| 106 | UPLOAD_MIN_TIMEOUT = 2 * 60 |
Mike Frysinger | cd78a08 | 2013-06-26 17:13:04 -0400 | [diff] [blame] | 107 | |
| 108 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 109 | # Sleep for 200ms in between uploads to avoid DoS'ing symbol server. |
| 110 | DEFAULT_SLEEP_DELAY = 0.2 |
| 111 | |
| 112 | |
| 113 | # Number of seconds to wait before retrying an upload. The delay will double |
| 114 | # for each subsequent retry of the same symbol file. |
| 115 | INITIAL_RETRY_DELAY = 1 |
| 116 | |
| 117 | # Allow up to 7 attempts to upload a symbol file (total delay may be |
| 118 | # 1+2+4+8+16+32=63 seconds). |
| 119 | MAX_RETRIES = 6 |
| 120 | |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 121 | # Number of total errors, before uploads are no longer attempted. |
| 122 | # This is used to avoid lots of errors causing unreasonable delays. |
| 123 | # See the related, but independent, error values below. |
| 124 | MAX_TOTAL_ERRORS_FOR_RETRY = 30 |
| 125 | |
| 126 | # A watermark of transient errors which we allow recovery from. If we hit |
| 127 | # errors infrequently, overall we're probably doing fine. For example, if |
| 128 | # we have one failure every 100 passes, then we probably don't want to fail |
| 129 | # right away. But if we hit a string of failures in a row, we want to abort. |
| 130 | # |
| 131 | # The watermark starts at 0 (and can never go below that). When this error |
| 132 | # level is exceeded, we stop uploading. When a failure happens, we add the |
| 133 | # fail adjustment, and when an upload succeeds, we add the pass adjustment. |
| 134 | # We want to penalize failures more so that we ramp up when there is a string |
| 135 | # of them, but then slowly back off as things start working. |
| 136 | # |
| 137 | # A quick example: |
| 138 | # 0.0: Starting point. |
| 139 | # 0.0: Upload works, so add -0.5, and then clamp to 0. |
| 140 | # 1.0: Upload fails, so add 1.0. |
| 141 | # 2.0: Upload fails, so add 1.0. |
| 142 | # 1.5: Upload works, so add -0.5. |
| 143 | # 1.0: Upload works, so add -0.5. |
| 144 | ERROR_WATERMARK = 3.0 |
| 145 | ERROR_ADJUST_FAIL = 1.0 |
| 146 | ERROR_ADJUST_PASS = -0.5 |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 147 | |
| 148 | |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 149 | # A named tuple which hold a SymbolItem object and |
| 150 | # a isolateserver._IsolateServerPushState item. |
| 151 | SymbolElement = collections.namedtuple( |
| 152 | 'SymbolElement', ('symbol_item', 'opaque_push_state')) |
| 153 | |
| 154 | |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 155 | def GetUploadTimeout(path): |
| 156 | """How long to wait for a specific file to upload to the crash server. |
| 157 | |
| 158 | This is a function largely to make unittesting easier. |
| 159 | |
| 160 | Args: |
| 161 | path: The path to the file to calculate the timeout for |
| 162 | |
| 163 | Returns: |
| 164 | Timeout length (in seconds) |
| 165 | """ |
| 166 | # Scale the timeout based on the filesize. |
| 167 | return max(os.path.getsize(path) / UPLOAD_MIN_RATE, UPLOAD_MIN_TIMEOUT) |
| 168 | |
| 169 | |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 170 | def SymUpload(upload_url, sym_item, product_name): |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 171 | """Upload a symbol file to a HTTP server |
| 172 | |
| 173 | The upload is a multipart/form-data POST with the following parameters: |
| 174 | code_file: the basename of the module, e.g. "app" |
| 175 | code_identifier: the module file's identifier |
| 176 | debug_file: the basename of the debugging file, e.g. "app" |
| 177 | debug_identifier: the debug file's identifier, usually consisting of |
| 178 | the guid and age embedded in the pdb, e.g. |
| 179 | "11111111BBBB3333DDDD555555555555F" |
| 180 | version: the file version of the module, e.g. "1.2.3.4" |
| 181 | product: HTTP-friendly product name |
| 182 | os: the operating system that the module was built for |
| 183 | cpu: the CPU that the module was built for |
| 184 | symbol_file: the contents of the breakpad-format symbol file |
| 185 | |
| 186 | Args: |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 187 | upload_url: The crash URL to POST the |sym_file| to |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 188 | sym_item: A SymbolItem containing the path to the breakpad symbol to upload |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 189 | product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'. |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 190 | """ |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 191 | sym_header = sym_item.sym_header |
| 192 | sym_file = sym_item.sym_file |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 193 | |
| 194 | fields = ( |
| 195 | ('code_file', sym_header.name), |
| 196 | ('debug_file', sym_header.name), |
| 197 | ('debug_identifier', sym_header.id.replace('-', '')), |
Mike Frysinger | b8a966b | 2014-03-19 17:36:18 -0400 | [diff] [blame] | 198 | # The product/version fields are used by the server only for statistic |
| 199 | # purposes. They do not impact symbolization, so they're safe to set |
| 200 | # to any value all the time. |
| 201 | # In this case, we use it to help see the load our build system is |
| 202 | # placing on the server. |
| 203 | # Not sure what to set for the version. Maybe the git sha1 of this file. |
| 204 | # Note: the server restricts this to 30 chars. |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 205 | #('version', None), |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 206 | ('product', product_name), |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 207 | ('os', sym_header.os), |
| 208 | ('cpu', sym_header.cpu), |
| 209 | poster.encode.MultipartParam.from_file('symbol_file', sym_file), |
| 210 | ) |
| 211 | |
| 212 | data, headers = poster.encode.multipart_encode(fields) |
| 213 | request = urllib2.Request(upload_url, data, headers) |
| 214 | request.add_header('User-agent', 'chromite.upload_symbols') |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 215 | urllib2.urlopen(request, timeout=GetUploadTimeout(sym_file)) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 216 | |
| 217 | |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 218 | def TestingSymUpload(upload_url, sym_item, _product_name): |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 219 | """A stub version of SymUpload for --testing usage""" |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 220 | cmd = ['sym_upload', sym_item.sym_file, upload_url] |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 221 | # Randomly fail 80% of the time (the retry logic makes this 80%/3 per file). |
| 222 | returncode = random.randint(1, 100) <= 80 |
Ralph Nathan | 5a582ff | 2015-03-20 18:18:30 -0700 | [diff] [blame] | 223 | logging.debug('would run (and return %i): %s', returncode, |
| 224 | cros_build_lib.CmdToStr(cmd)) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 225 | if returncode: |
| 226 | output = 'Failed to send the symbol file.' |
| 227 | else: |
| 228 | output = 'Successfully sent the symbol file.' |
| 229 | result = cros_build_lib.CommandResult(cmd=cmd, error=None, output=output, |
| 230 | returncode=returncode) |
| 231 | if returncode: |
Mike Frysinger | a4fa1e8 | 2014-01-15 01:45:56 -0500 | [diff] [blame] | 232 | exceptions = ( |
Mike Frysinger | fd35565 | 2014-01-23 02:57:48 -0500 | [diff] [blame] | 233 | socket.error('[socket.error] forced test fail'), |
Mike Frysinger | a4fa1e8 | 2014-01-15 01:45:56 -0500 | [diff] [blame] | 234 | httplib.BadStatusLine('[BadStatusLine] forced test fail'), |
| 235 | urllib2.HTTPError(upload_url, 400, '[HTTPError] forced test fail', |
| 236 | {}, None), |
| 237 | urllib2.URLError('[URLError] forced test fail'), |
| 238 | ) |
| 239 | raise random.choice(exceptions) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 240 | else: |
| 241 | return result |
| 242 | |
| 243 | |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 244 | def ErrorLimitHit(num_errors, watermark_errors): |
| 245 | """See if our error limit has been hit |
| 246 | |
| 247 | Args: |
| 248 | num_errors: A multiprocessing.Value of the raw number of failures. |
| 249 | watermark_errors: A multiprocessing.Value of the current rate of failures. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 250 | |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 251 | Returns: |
| 252 | True if our error limits have been exceeded. |
| 253 | """ |
| 254 | return ((num_errors is not None and |
| 255 | num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY) or |
| 256 | (watermark_errors is not None and |
| 257 | watermark_errors.value > ERROR_WATERMARK)) |
| 258 | |
| 259 | |
| 260 | def _UpdateCounter(counter, adj): |
| 261 | """Update |counter| by |adj| |
| 262 | |
| 263 | Handle atomic updates of |counter|. Also make sure it does not |
| 264 | fall below 0. |
| 265 | |
| 266 | Args: |
| 267 | counter: A multiprocessing.Value to update |
| 268 | adj: The value to add to |counter| |
| 269 | """ |
| 270 | def _Update(): |
| 271 | clamp = 0 if type(adj) is int else 0.0 |
| 272 | counter.value = max(clamp, counter.value + adj) |
| 273 | |
| 274 | if hasattr(counter, 'get_lock'): |
| 275 | with counter.get_lock(): |
| 276 | _Update() |
| 277 | elif counter is not None: |
| 278 | _Update() |
| 279 | |
| 280 | |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 281 | def UploadSymbol(upload_url, symbol_element, product_name, |
| 282 | file_limit=DEFAULT_FILE_LIMIT, sleep=0, num_errors=None, |
| 283 | watermark_errors=None, failed_queue=None, passed_queue=None): |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 284 | """Upload |sym_element.symbol_item| to |upload_url| |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 285 | |
| 286 | Args: |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 287 | upload_url: The crash server to upload things to |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 288 | symbol_element: A SymbolElement tuple. symbol_element.symbol_item is a |
| 289 | SymbolItem object containing the path to the breakpad symbol |
| 290 | to upload. symbol_element.opaque_push_state is an object of |
| 291 | _IsolateServerPushState or None if the item doesn't have |
| 292 | a push state. |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 293 | product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'. |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 294 | file_limit: The max file size of a symbol file before we try to strip it |
| 295 | sleep: Number of seconds to sleep before running |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 296 | num_errors: An object to update with the error count (needs a .value member) |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 297 | watermark_errors: An object to track current error behavior (needs a .value) |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 298 | failed_queue: When a symbol fails, add it to this queue |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 299 | passed_queue: When a symbol passes, add it to this queue |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 300 | |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 301 | Returns: |
| 302 | The number of errors that were encountered. |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 303 | """ |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 304 | sym_file = symbol_element.symbol_item.sym_file |
| 305 | upload_item = symbol_element.symbol_item |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 306 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 307 | if num_errors is None: |
| 308 | num_errors = ctypes.c_int() |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 309 | if ErrorLimitHit(num_errors, watermark_errors): |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 310 | # Abandon ship! It's on fire! NOoooooooooooOOOoooooo. |
Mike Frysinger | 7f9be14 | 2014-01-15 02:16:42 -0500 | [diff] [blame] | 311 | if failed_queue: |
| 312 | failed_queue.put(sym_file) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 313 | return 0 |
| 314 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 315 | if sleep: |
| 316 | # Keeps us from DoS-ing the symbol server. |
| 317 | time.sleep(sleep) |
| 318 | |
Ralph Nathan | 5a582ff | 2015-03-20 18:18:30 -0700 | [diff] [blame] | 319 | logging.debug('uploading %s' % sym_file) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 320 | |
| 321 | # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use. |
| 322 | with tempfile.NamedTemporaryFile(prefix='upload_symbols', |
| 323 | bufsize=0) as temp_sym_file: |
| 324 | if file_limit: |
| 325 | # If the symbols size is too big, strip out the call frame info. The CFI |
| 326 | # is unnecessary for 32bit x86 targets where the frame pointer is used (as |
| 327 | # all of ours have) and it accounts for over half the size of the symbols |
| 328 | # uploaded. |
| 329 | file_size = os.path.getsize(sym_file) |
| 330 | if file_size > file_limit: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 331 | logging.warning('stripping CFI from %s due to size %s > %s', sym_file, |
| 332 | file_size, file_limit) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 333 | temp_sym_file.writelines([x for x in open(sym_file, 'rb').readlines() |
| 334 | if not x.startswith('STACK CFI')]) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 335 | |
| 336 | upload_item = FakeItem(sym_file=temp_sym_file.name, |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 337 | sym_header=symbol_element.symbol_item.sym_header) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 338 | |
| 339 | # Hopefully the crash server will let it through. But it probably won't. |
| 340 | # Not sure what the best answer is in this case. |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 341 | file_size = os.path.getsize(upload_item.sym_file) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 342 | if file_size > CRASH_SERVER_FILE_LIMIT: |
Prathmesh Prabhu | 17f0742 | 2015-07-17 11:40:40 -0700 | [diff] [blame] | 343 | logging.PrintBuildbotStepWarnings() |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 344 | logging.warning('upload file %s is awfully large, risking rejection by ' |
| 345 | 'the symbol server (%s > %s)', sym_file, file_size, |
| 346 | CRASH_SERVER_FILE_LIMIT) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 347 | |
| 348 | # Upload the symbol file. |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 349 | success = False |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 350 | try: |
Mike Frysinger | 9adcfd2 | 2013-10-24 12:01:40 -0400 | [diff] [blame] | 351 | cros_build_lib.TimedCommand( |
David James | c93e6a4d | 2014-01-13 11:37:36 -0800 | [diff] [blame] | 352 | retry_util.RetryException, |
Mike Frysinger | 9adcfd2 | 2013-10-24 12:01:40 -0400 | [diff] [blame] | 353 | (urllib2.HTTPError, urllib2.URLError), MAX_RETRIES, SymUpload, |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 354 | upload_url, upload_item, product_name, sleep=INITIAL_RETRY_DELAY, |
Mike Frysinger | f50187c | 2014-11-27 22:01:29 -0500 | [diff] [blame] | 355 | timed_log_msg=('upload of %10i bytes took %%(delta)s: %s' % |
Mike Frysinger | d6e2df0 | 2014-11-26 02:55:04 -0500 | [diff] [blame] | 356 | (file_size, os.path.basename(sym_file)))) |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 357 | success = True |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 358 | |
| 359 | if passed_queue: |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 360 | passed_queue.put(symbol_element) |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 361 | except urllib2.HTTPError as e: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 362 | logging.warning('could not upload: %s: HTTP %s: %s', |
| 363 | os.path.basename(sym_file), e.code, e.reason) |
Mike Frysinger | fd35565 | 2014-01-23 02:57:48 -0500 | [diff] [blame] | 364 | except (urllib2.URLError, httplib.HTTPException, socket.error) as e: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 365 | logging.warning('could not upload: %s: %s', os.path.basename(sym_file), e) |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 366 | finally: |
| 367 | if success: |
| 368 | _UpdateCounter(watermark_errors, ERROR_ADJUST_PASS) |
| 369 | else: |
| 370 | _UpdateCounter(num_errors, 1) |
| 371 | _UpdateCounter(watermark_errors, ERROR_ADJUST_FAIL) |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 372 | if failed_queue: |
| 373 | failed_queue.put(sym_file) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 374 | |
| 375 | return num_errors.value |
| 376 | |
| 377 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 378 | # A dummy class that allows for stubbing in tests and SymUpload. |
| 379 | FakeItem = cros_build_lib.Collection( |
| 380 | 'FakeItem', sym_file=None, sym_header=None, content=lambda x: '') |
| 381 | |
| 382 | |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 383 | class SymbolItem(isolateserver.BufferItem): |
| 384 | """Turn a sym_file into an isolateserver.Item""" |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 385 | |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 386 | ALGO = hashlib.sha1 |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 387 | |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 388 | def __init__(self, sym_file): |
| 389 | sym_header = cros_generate_breakpad_symbols.ReadSymsHeader(sym_file) |
| 390 | super(SymbolItem, self).__init__(str(sym_header), self.ALGO) |
| 391 | self.sym_header = sym_header |
| 392 | self.sym_file = sym_file |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 393 | |
| 394 | |
| 395 | def SymbolDeduplicatorNotify(dedupe_namespace, dedupe_queue): |
| 396 | """Send a symbol file to the swarming service |
| 397 | |
| 398 | Notify the swarming service of a successful upload. If the notification fails |
| 399 | for any reason, we ignore it. We don't care as it just means we'll upload it |
| 400 | again later on, and the symbol server will handle that graciously. |
| 401 | |
| 402 | This func runs in a different process from the main one, so we cannot share |
| 403 | the storage object. Instead, we create our own. This func stays alive for |
| 404 | the life of the process, so we only create one here overall. |
| 405 | |
| 406 | Args: |
| 407 | dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols. |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 408 | dedupe_queue: The queue to read SymbolElements from |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 409 | """ |
| 410 | if dedupe_queue is None: |
| 411 | return |
| 412 | |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 413 | sym_file = '' |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 414 | try: |
Mike Frysinger | 650e672 | 2014-04-28 18:29:15 -0400 | [diff] [blame] | 415 | with timeout_util.Timeout(DEDUPE_TIMEOUT): |
| 416 | storage = isolateserver.get_storage_api(constants.ISOLATESERVER, |
| 417 | dedupe_namespace) |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 418 | for symbol_element in iter(dedupe_queue.get, None): |
| 419 | if not symbol_element or not symbol_element.symbol_item: |
| 420 | continue |
| 421 | symbol_item = symbol_element.symbol_item |
| 422 | push_state = symbol_element.opaque_push_state |
| 423 | sym_file = symbol_item.sym_file if symbol_item.sym_file else '' |
| 424 | if push_state is not None: |
| 425 | with timeout_util.Timeout(DEDUPE_TIMEOUT): |
| 426 | logging.debug('sending %s to dedupe server', sym_file) |
| 427 | symbol_item.prepare(SymbolItem.ALGO) |
| 428 | storage.push(symbol_item, push_state, symbol_item.content()) |
| 429 | logging.debug('sent %s', sym_file) |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 430 | logging.info('dedupe notification finished; exiting') |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 431 | except Exception: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 432 | logging.warning('posting %s to dedupe server failed', |
| 433 | os.path.basename(sym_file), exc_info=True) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 434 | |
Mike Frysinger | 58312e9 | 2014-03-18 04:18:36 -0400 | [diff] [blame] | 435 | # Keep draining the queue though so it doesn't fill up. |
| 436 | while dedupe_queue.get() is not None: |
| 437 | continue |
| 438 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 439 | |
| 440 | def SymbolDeduplicator(storage, sym_paths): |
| 441 | """Filter out symbol files that we've already uploaded |
| 442 | |
| 443 | Using the swarming service, ask it to tell us which symbol files we've already |
| 444 | uploaded in previous runs and/or by other bots. If the query fails for any |
| 445 | reason, we'll just upload all symbols. This is fine as the symbol server will |
| 446 | do the right thing and this phase is purely an optimization. |
| 447 | |
| 448 | This code runs in the main thread which is why we can re-use the existing |
| 449 | storage object. Saves us from having to recreate one all the time. |
| 450 | |
| 451 | Args: |
| 452 | storage: An isolateserver.StorageApi object |
| 453 | sym_paths: List of symbol files to check against the dedupe server |
| 454 | |
| 455 | Returns: |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 456 | List of SymbolElement objects that have not been uploaded before |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 457 | """ |
| 458 | if not sym_paths: |
| 459 | return sym_paths |
| 460 | |
| 461 | items = [SymbolItem(x) for x in sym_paths] |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 462 | for item in items: |
| 463 | item.prepare(SymbolItem.ALGO) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 464 | if storage: |
| 465 | try: |
| 466 | with timeout_util.Timeout(DEDUPE_TIMEOUT): |
| 467 | items = storage.contains(items) |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 468 | return [SymbolElement(symbol_item=item, opaque_push_state=push_state) |
| 469 | for (item, push_state) in items.iteritems()] |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 470 | except Exception: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 471 | logging.warning('talking to dedupe server failed', exc_info=True) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 472 | |
Fang Deng | ba68046 | 2015-08-16 20:34:11 -0700 | [diff] [blame] | 473 | return [SymbolElement(symbol_item=item, opaque_push_state=None) |
| 474 | for item in items] |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 475 | |
| 476 | |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 477 | def IsTarball(path): |
| 478 | """Guess if this is a tarball based on the filename.""" |
| 479 | parts = path.split('.') |
| 480 | if len(parts) <= 1: |
| 481 | return False |
| 482 | |
| 483 | if parts[-1] == 'tar': |
| 484 | return True |
| 485 | |
| 486 | if parts[-2] == 'tar': |
| 487 | return parts[-1] in ('bz2', 'gz', 'xz') |
| 488 | |
| 489 | return parts[-1] in ('tbz2', 'tbz', 'tgz', 'txz') |
| 490 | |
| 491 | |
| 492 | def SymbolFinder(tempdir, paths): |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 493 | """Locate symbol files in |paths| |
| 494 | |
| 495 | Args: |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 496 | tempdir: Path to use for temporary files (caller will clean up). |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 497 | paths: A list of input paths to walk. Files are returned w/out any checks. |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 498 | Dirs are searched for files that end in ".sym". Urls are fetched and then |
| 499 | processed. Tarballs are unpacked and walked. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 500 | |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 501 | Returns: |
| 502 | Yield every viable sym file. |
| 503 | """ |
Gilad Arnold | 83233ed | 2015-05-08 12:12:13 -0700 | [diff] [blame] | 504 | cache_dir = path_util.GetCacheDir() |
Mike Frysinger | e847efd | 2015-01-08 03:57:24 -0500 | [diff] [blame] | 505 | common_path = os.path.join(cache_dir, constants.COMMON_CACHE) |
| 506 | tar_cache = cache.TarballCache(common_path) |
| 507 | |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 508 | for p in paths: |
Don Garrett | 25f309a | 2014-03-19 14:02:12 -0700 | [diff] [blame] | 509 | # Pylint is confused about members of ParseResult. |
Don Garrett | f8bf784 | 2014-03-20 17:03:42 -0700 | [diff] [blame] | 510 | |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 511 | o = urlparse.urlparse(p) |
Don Garrett | f8bf784 | 2014-03-20 17:03:42 -0700 | [diff] [blame] | 512 | if o.scheme: # pylint: disable=E1101 |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 513 | # Support globs of filenames. |
| 514 | ctx = gs.GSContext() |
| 515 | for p in ctx.LS(p): |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 516 | logging.info('processing files inside %s', p) |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 517 | o = urlparse.urlparse(p) |
Don Garrett | f8bf784 | 2014-03-20 17:03:42 -0700 | [diff] [blame] | 518 | key = ('%s%s' % (o.netloc, o.path)).split('/') # pylint: disable=E1101 |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 519 | # The common cache will not be LRU, removing the need to hold a read |
| 520 | # lock on the cached gsutil. |
| 521 | ref = tar_cache.Lookup(key) |
| 522 | try: |
| 523 | ref.SetDefault(p) |
| 524 | except cros_build_lib.RunCommandError as e: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 525 | logging.warning('ignoring %s\n%s', p, e) |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 526 | continue |
| 527 | for p in SymbolFinder(tempdir, [ref.path]): |
| 528 | yield p |
| 529 | |
| 530 | elif os.path.isdir(p): |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 531 | for root, _, files in os.walk(p): |
| 532 | for f in files: |
| 533 | if f.endswith('.sym'): |
| 534 | yield os.path.join(root, f) |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 535 | |
| 536 | elif IsTarball(p): |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 537 | logging.info('processing files inside %s', p) |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 538 | tardir = tempfile.mkdtemp(dir=tempdir) |
| 539 | cache.Untar(os.path.realpath(p), tardir) |
| 540 | for p in SymbolFinder(tardir, [tardir]): |
| 541 | yield p |
| 542 | |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 543 | else: |
| 544 | yield p |
| 545 | |
| 546 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 547 | def WriteQueueToFile(listing, queue, relpath=None): |
| 548 | """Write all the items in |queue| to the |listing|. |
| 549 | |
Mike Frysinger | 5e6dd71 | 2014-03-07 22:21:17 -0500 | [diff] [blame] | 550 | Note: The queue must have a sentinel None appended to the end. |
| 551 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 552 | Args: |
| 553 | listing: Where to write out the list of files. |
| 554 | queue: The queue of paths to drain. |
| 555 | relpath: If set, write out paths relative to this one. |
| 556 | """ |
| 557 | if not listing: |
Mike Frysinger | a0ddac6 | 2014-03-14 10:30:25 -0400 | [diff] [blame] | 558 | # Still drain the queue so we make sure the producer has finished |
| 559 | # before we return. Otherwise, the queue might get destroyed too |
| 560 | # quickly which will trigger a traceback in the producer. |
| 561 | while queue.get() is not None: |
| 562 | continue |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 563 | return |
| 564 | |
| 565 | with cros_build_lib.Open(listing, 'wb+') as f: |
Mike Frysinger | 5e6dd71 | 2014-03-07 22:21:17 -0500 | [diff] [blame] | 566 | while True: |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 567 | path = queue.get() |
Mike Frysinger | 5e6dd71 | 2014-03-07 22:21:17 -0500 | [diff] [blame] | 568 | if path is None: |
| 569 | return |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 570 | if relpath: |
| 571 | path = os.path.relpath(path, relpath) |
| 572 | f.write('%s\n' % path) |
| 573 | |
| 574 | |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 575 | def UploadSymbols(board=None, official=False, server=None, breakpad_dir=None, |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 576 | file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY, |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 577 | upload_limit=None, sym_paths=None, failed_list=None, |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 578 | root=None, retry=True, dedupe_namespace=None, |
| 579 | product_name='ChromeOS'): |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 580 | """Upload all the generated symbols for |board| to the crash server |
| 581 | |
Mike Frysinger | 9dcf9ae | 2013-08-10 15:17:09 -0400 | [diff] [blame] | 582 | You can use in a few ways: |
| 583 | * pass |board| to locate all of its symbols |
| 584 | * pass |breakpad_dir| to upload all the symbols in there |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 585 | * pass |sym_paths| to upload specific symbols (or dirs of symbols) |
Mike Frysinger | 9dcf9ae | 2013-08-10 15:17:09 -0400 | [diff] [blame] | 586 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 587 | Args: |
| 588 | board: The board whose symbols we wish to upload |
| 589 | official: Use the official symbol server rather than the staging one |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 590 | server: Explicit server to post symbols to |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 591 | breakpad_dir: The full path to the breakpad directory where symbols live |
| 592 | file_limit: The max file size of a symbol file before we try to strip it |
| 593 | sleep: How long to sleep in between uploads |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 594 | upload_limit: If set, only upload this many symbols (meant for testing) |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 595 | sym_paths: Specific symbol files (or dirs of sym files) to upload, |
| 596 | otherwise search |breakpad_dir| |
Mike Frysinger | 7f9be14 | 2014-01-15 02:16:42 -0500 | [diff] [blame] | 597 | failed_list: Write the names of all sym files we did not upload; can be a |
| 598 | filename or file-like object. |
Mike Frysinger | 118d250 | 2013-08-19 03:36:56 -0400 | [diff] [blame] | 599 | root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set) |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 600 | retry: Whether we should retry failures. |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 601 | dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols. |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 602 | product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'. |
Mike Frysinger | 1a736a8 | 2013-12-12 01:50:59 -0500 | [diff] [blame] | 603 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 604 | Returns: |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 605 | The number of errors that were encountered. |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 606 | """ |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 607 | if server is None: |
| 608 | if official: |
| 609 | upload_url = OFFICIAL_UPLOAD_URL |
| 610 | else: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 611 | logging.warning('unofficial builds upload to the staging server') |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 612 | upload_url = STAGING_UPLOAD_URL |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 613 | else: |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 614 | upload_url = server |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 615 | |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 616 | if sym_paths: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 617 | logging.info('uploading specified symbols to %s', upload_url) |
Mike Frysinger | 9dcf9ae | 2013-08-10 15:17:09 -0400 | [diff] [blame] | 618 | else: |
| 619 | if breakpad_dir is None: |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 620 | if root is None: |
| 621 | raise ValueError('breakpad_dir requires root to be set') |
Mike Frysinger | 118d250 | 2013-08-19 03:36:56 -0400 | [diff] [blame] | 622 | breakpad_dir = os.path.join( |
| 623 | root, |
| 624 | cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip('/')) |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 625 | logging.info('uploading all symbols to %s from %s', upload_url, |
| 626 | breakpad_dir) |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 627 | sym_paths = [breakpad_dir] |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 628 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 629 | # We use storage_query to ask the server about existing symbols. The |
| 630 | # storage_notify_proc process is used to post updates to the server. We |
| 631 | # cannot safely share the storage object between threads/processes, but |
| 632 | # we also want to minimize creating new ones as each object has to init |
| 633 | # new state (like server connections). |
Mike Frysinger | 650e672 | 2014-04-28 18:29:15 -0400 | [diff] [blame] | 634 | storage_query = None |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 635 | if dedupe_namespace: |
| 636 | dedupe_limit = DEDUPE_LIMIT |
| 637 | dedupe_queue = multiprocessing.Queue() |
Mike Frysinger | 650e672 | 2014-04-28 18:29:15 -0400 | [diff] [blame] | 638 | try: |
| 639 | with timeout_util.Timeout(DEDUPE_TIMEOUT): |
| 640 | storage_query = isolateserver.get_storage_api(constants.ISOLATESERVER, |
| 641 | dedupe_namespace) |
| 642 | except Exception: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 643 | logging.warning('initializing dedupe server connection failed', |
| 644 | exc_info=True) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 645 | else: |
| 646 | dedupe_limit = 1 |
Mike Frysinger | 650e672 | 2014-04-28 18:29:15 -0400 | [diff] [blame] | 647 | dedupe_queue = None |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 648 | # Can't use parallel.BackgroundTaskRunner because that'll create multiple |
| 649 | # processes and we want only one the whole time (see comment above). |
| 650 | storage_notify_proc = multiprocessing.Process( |
| 651 | target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue)) |
| 652 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 653 | bg_errors = multiprocessing.Value('i') |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 654 | watermark_errors = multiprocessing.Value('f') |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 655 | failed_queue = multiprocessing.Queue() |
| 656 | uploader = functools.partial( |
Don Garrett | cea3eea | 2015-09-29 17:20:46 -0700 | [diff] [blame] | 657 | UploadSymbol, upload_url, product_name=product_name, |
| 658 | file_limit=file_limit, sleep=sleep, num_errors=bg_errors, |
| 659 | watermark_errors=watermark_errors, failed_queue=failed_queue, |
| 660 | passed_queue=dedupe_queue) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 661 | |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 662 | start_time = datetime.datetime.now() |
| 663 | Counters = cros_build_lib.Collection( |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 664 | 'Counters', upload_limit=upload_limit, uploaded_count=0, deduped_count=0) |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 665 | counters = Counters() |
| 666 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 667 | def _Upload(queue, counters, files): |
| 668 | if not files: |
| 669 | return |
| 670 | |
| 671 | missing_count = 0 |
| 672 | for item in SymbolDeduplicator(storage_query, files): |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 673 | missing_count += 1 |
Mike Frysinger | d42e5f0 | 2014-03-14 11:19:37 -0400 | [diff] [blame] | 674 | |
| 675 | if counters.upload_limit == 0: |
| 676 | continue |
| 677 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 678 | queue.put((item,)) |
| 679 | counters.uploaded_count += 1 |
| 680 | if counters.upload_limit is not None: |
| 681 | counters.upload_limit -= 1 |
| 682 | |
| 683 | counters.deduped_count += (len(files) - missing_count) |
| 684 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 685 | try: |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 686 | storage_notify_proc.start() |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 687 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 688 | with osutils.TempDir(prefix='upload_symbols.') as tempdir: |
| 689 | # For the first run, we collect the symbols that failed. If the |
| 690 | # overall failure rate was low, we'll retry them on the second run. |
| 691 | for retry in (retry, False): |
| 692 | # We need to limit ourselves to one upload at a time to avoid the server |
| 693 | # kicking in DoS protection. See these bugs for more details: |
| 694 | # http://crbug.com/209442 |
| 695 | # http://crbug.com/212496 |
| 696 | with parallel.BackgroundTaskRunner(uploader, processes=1) as queue: |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 697 | dedupe_list = [] |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 698 | for sym_file in SymbolFinder(tempdir, sym_paths): |
| 699 | dedupe_list.append(sym_file) |
| 700 | dedupe_len = len(dedupe_list) |
| 701 | if dedupe_len < dedupe_limit: |
| 702 | if (counters.upload_limit is None or |
| 703 | dedupe_len < counters.upload_limit): |
| 704 | continue |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 705 | |
Mike Frysinger | 1010a89 | 2014-03-14 11:24:17 -0400 | [diff] [blame] | 706 | # We check the counter before _Upload so that we don't keep talking |
| 707 | # to the dedupe server. Otherwise, we end up sending one symbol at |
| 708 | # a time to it and that slows things down a lot. |
| 709 | if counters.upload_limit == 0: |
| 710 | break |
| 711 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 712 | _Upload(queue, counters, dedupe_list) |
| 713 | dedupe_list = [] |
| 714 | _Upload(queue, counters, dedupe_list) |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 715 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 716 | # See if we need to retry, and if we haven't failed too many times yet. |
| 717 | if not retry or ErrorLimitHit(bg_errors, watermark_errors): |
Mike Frysinger | 5e6dd71 | 2014-03-07 22:21:17 -0500 | [diff] [blame] | 718 | break |
Mike Frysinger | 5e6dd71 | 2014-03-07 22:21:17 -0500 | [diff] [blame] | 719 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 720 | sym_paths = [] |
| 721 | failed_queue.put(None) |
| 722 | while True: |
| 723 | sym_path = failed_queue.get() |
| 724 | if sym_path is None: |
| 725 | break |
| 726 | sym_paths.append(sym_path) |
Mike Frysinger | 02e9240 | 2013-11-22 16:22:02 -0500 | [diff] [blame] | 727 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 728 | if sym_paths: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 729 | logging.warning('retrying %i symbols', len(sym_paths)) |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 730 | if counters.upload_limit is not None: |
| 731 | counters.upload_limit += len(sym_paths) |
| 732 | # Decrement the error count in case we recover in the second pass. |
| 733 | assert bg_errors.value >= len(sym_paths), \ |
| 734 | 'more failed files than errors?' |
| 735 | bg_errors.value -= len(sym_paths) |
| 736 | else: |
| 737 | # No failed symbols, so just return now. |
| 738 | break |
Mike Frysinger | 7f9be14 | 2014-01-15 02:16:42 -0500 | [diff] [blame] | 739 | |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 740 | # If the user has requested it, save all the symbol files that we failed to |
| 741 | # upload to a listing file. This should help with recovery efforts later. |
| 742 | failed_queue.put(None) |
| 743 | WriteQueueToFile(failed_list, failed_queue, breakpad_dir) |
| 744 | |
| 745 | finally: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 746 | logging.info('finished uploading; joining background process') |
Mike Frysinger | 1387008 | 2014-03-14 10:41:20 -0400 | [diff] [blame] | 747 | if dedupe_queue: |
| 748 | dedupe_queue.put(None) |
Mike Frysinger | 4dd462e | 2014-04-30 16:21:51 -0400 | [diff] [blame] | 749 | |
| 750 | # The notification might be slow going, so give it some time to finish. |
| 751 | # We have to poll here as the process monitor is watching for output and |
| 752 | # will kill us if we go silent for too long. |
| 753 | wait_minutes = DEDUPE_NOTIFY_TIMEOUT |
| 754 | while storage_notify_proc.is_alive() and wait_minutes > 0: |
Aviv Keshet | d1f0463 | 2014-05-09 11:33:46 -0700 | [diff] [blame] | 755 | if dedupe_queue: |
| 756 | qsize = str(dedupe_queue.qsize()) |
| 757 | else: |
| 758 | qsize = '[None]' |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 759 | logging.info('waiting up to %i minutes for ~%s notifications', |
| 760 | wait_minutes, qsize) |
Mike Frysinger | 4dd462e | 2014-04-30 16:21:51 -0400 | [diff] [blame] | 761 | storage_notify_proc.join(60) |
| 762 | wait_minutes -= 1 |
| 763 | |
| 764 | # The process is taking too long, so kill it and complain. |
| 765 | if storage_notify_proc.is_alive(): |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 766 | logging.warning('notification process took too long') |
Prathmesh Prabhu | 17f0742 | 2015-07-17 11:40:40 -0700 | [diff] [blame] | 767 | logging.PrintBuildbotStepWarnings() |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 768 | |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 769 | # Kill it gracefully first (traceback) before tacking it down harder. |
| 770 | pid = storage_notify_proc.pid |
| 771 | for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL): |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 772 | logging.warning('sending %s to %i', signals.StrSignal(sig), pid) |
Mike Frysinger | dad2a03 | 2014-05-11 23:05:11 -0400 | [diff] [blame] | 773 | # The process might have exited between the last check and the |
| 774 | # actual kill below, so ignore ESRCH errors. |
| 775 | try: |
| 776 | os.kill(pid, sig) |
| 777 | except OSError as e: |
| 778 | if e.errno == errno.ESRCH: |
| 779 | break |
| 780 | else: |
| 781 | raise |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 782 | time.sleep(5) |
Mike Frysinger | dad2a03 | 2014-05-11 23:05:11 -0400 | [diff] [blame] | 783 | if not storage_notify_proc.is_alive(): |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 784 | break |
| 785 | |
| 786 | # Drain the queue so we don't hang when we finish. |
| 787 | try: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 788 | logging.warning('draining the notify queue manually') |
Mike Frysinger | 4f5ea83 | 2014-05-12 00:54:28 -0400 | [diff] [blame] | 789 | with timeout_util.Timeout(60): |
| 790 | try: |
| 791 | while dedupe_queue.get_nowait(): |
| 792 | pass |
| 793 | except Queue.Empty: |
| 794 | pass |
| 795 | except timeout_util.TimeoutError: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 796 | logging.warning('draining the notify queue failed; trashing it') |
Mike Frysinger | 4f5ea83 | 2014-05-12 00:54:28 -0400 | [diff] [blame] | 797 | dedupe_queue.cancel_join_thread() |
Mike Frysinger | 66e51e9 | 2014-05-03 16:52:00 -0400 | [diff] [blame] | 798 | |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 799 | logging.info('uploaded %i symbols (%i were deduped) which took: %s', |
| 800 | counters.uploaded_count, counters.deduped_count, |
| 801 | datetime.datetime.now() - start_time) |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 802 | |
Mike Frysinger | 9b2ff5c | 2013-11-22 10:01:12 -0500 | [diff] [blame] | 803 | return bg_errors.value |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 804 | |
| 805 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 806 | def main(argv): |
| 807 | parser = commandline.ArgumentParser(description=__doc__) |
| 808 | |
Mike Frysinger | d41938e | 2014-02-10 06:37:55 -0500 | [diff] [blame] | 809 | parser.add_argument('sym_paths', type='path_or_uri', nargs='*', default=None, |
| 810 | help='symbol file or directory or URL or tarball') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 811 | parser.add_argument('--board', default=None, |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 812 | help='Used to find default breakpad_root.') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 813 | parser.add_argument('--breakpad_root', type='path', default=None, |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 814 | help='full path to the breakpad symbol directory') |
| 815 | parser.add_argument('--root', type='path', default=None, |
| 816 | help='full path to the chroot dir') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 817 | parser.add_argument('--official_build', action='store_true', default=False, |
| 818 | help='point to official symbol server') |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 819 | parser.add_argument('--server', type=str, default=None, |
| 820 | help='URI for custom symbol server') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 821 | parser.add_argument('--regenerate', action='store_true', default=False, |
| 822 | help='regenerate all symbols') |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 823 | parser.add_argument('--upload-limit', type=int, default=None, |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 824 | help='only upload # number of symbols') |
| 825 | parser.add_argument('--strip_cfi', type=int, |
| 826 | default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024), |
| 827 | help='strip CFI data for files above this size') |
Mike Frysinger | 7f9be14 | 2014-01-15 02:16:42 -0500 | [diff] [blame] | 828 | parser.add_argument('--failed-list', type='path', |
| 829 | help='where to save a list of failed symbols') |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 830 | parser.add_argument('--dedupe', action='store_true', default=False, |
| 831 | help='use the swarming service to avoid re-uploading') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 832 | parser.add_argument('--testing', action='store_true', default=False, |
| 833 | help='run in testing mode') |
| 834 | parser.add_argument('--yes', action='store_true', default=False, |
| 835 | help='answer yes to all prompts') |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 836 | parser.add_argument('--product_name', type=str, default='ChromeOS', |
| 837 | help='Produce Name for breakpad stats.') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 838 | |
| 839 | opts = parser.parse_args(argv) |
Mike Frysinger | 90e49ca | 2014-01-14 14:42:07 -0500 | [diff] [blame] | 840 | opts.Freeze() |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 841 | |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 842 | if opts.sym_paths or opts.breakpad_root: |
Mike Frysinger | 9dcf9ae | 2013-08-10 15:17:09 -0400 | [diff] [blame] | 843 | if opts.regenerate: |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 844 | cros_build_lib.Die('--regenerate may not be used with specific files, ' |
| 845 | 'or breakpad_root') |
Mike Frysinger | 9dcf9ae | 2013-08-10 15:17:09 -0400 | [diff] [blame] | 846 | else: |
| 847 | if opts.board is None: |
| 848 | cros_build_lib.Die('--board is required') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 849 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 850 | if opts.testing: |
| 851 | # TODO(build): Kill off --testing mode once unittests are up-to-snuff. |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 852 | logging.info('running in testing mode') |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 853 | # pylint: disable=W0601,W0603 |
| 854 | global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY |
| 855 | INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0 |
| 856 | SymUpload = TestingSymUpload |
| 857 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 858 | dedupe_namespace = None |
| 859 | if opts.dedupe: |
| 860 | if opts.official_build and not opts.testing: |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 861 | dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE_TMPL % opts.product_name |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 862 | else: |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 863 | dedupe_namespace = STAGING_DEDUPE_NAMESPACE_TMPL % opts.product_name |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 864 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 865 | if not opts.yes: |
Mike Frysinger | c5de960 | 2014-02-09 02:42:36 -0500 | [diff] [blame] | 866 | prolog = '\n'.join(textwrap.wrap(textwrap.dedent(""" |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 867 | Uploading symbols for an entire Chromium OS build is really only |
| 868 | necessary for release builds and in a few cases for developers |
| 869 | to debug problems. It will take considerable time to run. For |
| 870 | developer debugging purposes, consider instead passing specific |
| 871 | files to upload. |
Mike Frysinger | c5de960 | 2014-02-09 02:42:36 -0500 | [diff] [blame] | 872 | """), 80)).strip() |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 873 | if not cros_build_lib.BooleanPrompt( |
| 874 | prompt='Are you sure you want to upload all build symbols', |
Mike Frysinger | c5de960 | 2014-02-09 02:42:36 -0500 | [diff] [blame] | 875 | default=False, prolog=prolog): |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 876 | cros_build_lib.Die('better safe than sorry') |
| 877 | |
| 878 | ret = 0 |
| 879 | if opts.regenerate: |
Mike Frysinger | 69cb41d | 2013-08-11 20:08:19 -0400 | [diff] [blame] | 880 | ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols( |
| 881 | opts.board, breakpad_dir=opts.breakpad_root) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 882 | |
| 883 | ret += UploadSymbols(opts.board, official=opts.official_build, |
Mike Frysinger | 3864754 | 2014-09-12 18:15:39 -0700 | [diff] [blame] | 884 | server=opts.server, breakpad_dir=opts.breakpad_root, |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 885 | file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY, |
Mike Frysinger | 8ec8c50 | 2014-02-10 00:19:13 -0500 | [diff] [blame] | 886 | upload_limit=opts.upload_limit, sym_paths=opts.sym_paths, |
Mike Frysinger | c5597f2 | 2014-11-27 15:39:15 -0500 | [diff] [blame] | 887 | failed_list=opts.failed_list, root=opts.root, |
Don Garrett | 747cc4b | 2015-10-07 14:48:48 -0700 | [diff] [blame] | 888 | dedupe_namespace=dedupe_namespace, |
| 889 | product_name=opts.product_name) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 890 | if ret: |
Ralph Nathan | 5990042 | 2015-03-24 10:41:17 -0700 | [diff] [blame] | 891 | logging.error('encountered %i problem(s)', ret) |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 892 | # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently |
| 893 | # return 0 in case we are a multiple of the mask. |
| 894 | ret = 1 |
| 895 | |
| 896 | return ret |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 897 | |
| 898 | |
| 899 | # We need this to run once per process. Do it at module import time as that |
| 900 | # will let us avoid doing it inline at function call time (see SymUpload) as |
| 901 | # that func might be called by the multiprocessing module which means we'll |
| 902 | # do the opener logic multiple times overall. Plus, if you're importing this |
| 903 | # module, it's a pretty good chance that you're going to need this. |
| 904 | poster.streaminghttp.register_openers() |