blob: dd4638f3234f062e1e02308ed898344817f97bd9 [file] [log] [blame]
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -04001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Upload all debug symbols required for crash reporting purposes.
6
7This script need only be used to upload release builds symbols or to debug
8crashes on non-release builds (in which case try to only upload the symbols
Mike Frysinger02e1e072013-11-10 22:11:34 -05009for those executables involved).
10"""
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040011
Mike Frysingera4fa1e82014-01-15 01:45:56 -050012from __future__ import print_function
13
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040014import ctypes
Mike Frysinger8ec8c502014-02-10 00:19:13 -050015import datetime
Mike Frysinger02e92402013-11-22 16:22:02 -050016import functools
Mike Frysinger0c0efa22014-02-09 23:32:23 -050017import hashlib
Mike Frysingera4fa1e82014-01-15 01:45:56 -050018import httplib
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040019import multiprocessing
20import os
Mike Frysinger094a2172013-08-14 12:54:35 -040021import poster
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040022import random
Mike Frysingerfd355652014-01-23 02:57:48 -050023import socket
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040024import textwrap
25import tempfile
26import time
Mike Frysinger094a2172013-08-14 12:54:35 -040027import urllib2
Mike Frysingerd41938e2014-02-10 06:37:55 -050028import urlparse
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040029
Mike Frysinger0c0efa22014-02-09 23:32:23 -050030from chromite.buildbot import constants
Mike Frysingerd41938e2014-02-10 06:37:55 -050031from chromite.lib import cache
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040032from chromite.lib import commandline
33from chromite.lib import cros_build_lib
Mike Frysingerd41938e2014-02-10 06:37:55 -050034from chromite.lib import gs
35from chromite.lib import osutils
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040036from chromite.lib import parallel
David Jamesc93e6a4d2014-01-13 11:37:36 -080037from chromite.lib import retry_util
Mike Frysinger0c0efa22014-02-09 23:32:23 -050038from chromite.lib import timeout_util
Mike Frysinger69cb41d2013-08-11 20:08:19 -040039from chromite.scripts import cros_generate_breakpad_symbols
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040040
Mike Frysinger0c0efa22014-02-09 23:32:23 -050041# Needs to be after chromite imports.
42# TODO(build): When doing the initial buildbot bootstrap, we won't have any
43# other repos available. So ignore isolateserver imports. But buildbot will
44# re-exec itself once it has done a full repo sync and then the module will
45# be available -- it isn't needed that early. http://crbug.com/341152
46try:
47 import isolateserver
48except ImportError:
49 isolateserver = None
50
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040051
52# URLs used for uploading symbols.
53OFFICIAL_UPLOAD_URL = 'http://clients2.google.com/cr/symbol'
54STAGING_UPLOAD_URL = 'http://clients2.google.com/cr/staging_symbol'
55
56
57# The crash server rejects files that are this big.
58CRASH_SERVER_FILE_LIMIT = 350 * 1024 * 1024
59# Give ourselves a little breathing room from what the server expects.
60DEFAULT_FILE_LIMIT = CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024)
61
62
Mike Frysinger0c0efa22014-02-09 23:32:23 -050063# The batch limit when talking to the dedup server. We avoid sending one at a
64# time as the round trip overhead will dominate. Conversely, we avoid sending
65# all at once so we can start uploading symbols asap -- the symbol server is a
66# bit slow and will take longer than anything else.
67# TODO: A better algorithm would be adaptive. If we have more than one symbol
68# in the upload queue waiting, we could send more symbols to the dedupe server
69# at a time.
70DEDUPE_LIMIT = 100
71
72# How long to wait for the server to respond with the results. Note that the
73# larger the limit above, the larger this will need to be. So we give it ~1
74# second per item max.
75DEDUPE_TIMEOUT = DEDUPE_LIMIT
76
77# The unique namespace in the dedupe server that only we use. Helps avoid
78# collisions with all the hashed values and unrelated content.
79OFFICIAL_DEDUPE_NAMESPACE = 'chromium-os-upload-symbols'
80STAGING_DEDUPE_NAMESPACE = '%s-staging' % OFFICIAL_DEDUPE_NAMESPACE
81
82
Mike Frysingercd78a082013-06-26 17:13:04 -040083# How long to wait (in seconds) for a single upload to complete. This has
84# to allow for symbols that are up to CRASH_SERVER_FILE_LIMIT in size.
85UPLOAD_TIMEOUT = 30 * 60
86
87
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040088# Sleep for 200ms in between uploads to avoid DoS'ing symbol server.
89DEFAULT_SLEEP_DELAY = 0.2
90
91
92# Number of seconds to wait before retrying an upload. The delay will double
93# for each subsequent retry of the same symbol file.
94INITIAL_RETRY_DELAY = 1
95
96# Allow up to 7 attempts to upload a symbol file (total delay may be
97# 1+2+4+8+16+32=63 seconds).
98MAX_RETRIES = 6
99
Mike Frysingereb753bf2013-11-22 16:05:35 -0500100# Number of total errors, before uploads are no longer attempted.
101# This is used to avoid lots of errors causing unreasonable delays.
102# See the related, but independent, error values below.
103MAX_TOTAL_ERRORS_FOR_RETRY = 30
104
105# A watermark of transient errors which we allow recovery from. If we hit
106# errors infrequently, overall we're probably doing fine. For example, if
107# we have one failure every 100 passes, then we probably don't want to fail
108# right away. But if we hit a string of failures in a row, we want to abort.
109#
110# The watermark starts at 0 (and can never go below that). When this error
111# level is exceeded, we stop uploading. When a failure happens, we add the
112# fail adjustment, and when an upload succeeds, we add the pass adjustment.
113# We want to penalize failures more so that we ramp up when there is a string
114# of them, but then slowly back off as things start working.
115#
116# A quick example:
117# 0.0: Starting point.
118# 0.0: Upload works, so add -0.5, and then clamp to 0.
119# 1.0: Upload fails, so add 1.0.
120# 2.0: Upload fails, so add 1.0.
121# 1.5: Upload works, so add -0.5.
122# 1.0: Upload works, so add -0.5.
123ERROR_WATERMARK = 3.0
124ERROR_ADJUST_FAIL = 1.0
125ERROR_ADJUST_PASS = -0.5
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400126
127
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500128def SymUpload(upload_url, sym_item):
Mike Frysinger094a2172013-08-14 12:54:35 -0400129 """Upload a symbol file to a HTTP server
130
131 The upload is a multipart/form-data POST with the following parameters:
132 code_file: the basename of the module, e.g. "app"
133 code_identifier: the module file's identifier
134 debug_file: the basename of the debugging file, e.g. "app"
135 debug_identifier: the debug file's identifier, usually consisting of
136 the guid and age embedded in the pdb, e.g.
137 "11111111BBBB3333DDDD555555555555F"
138 version: the file version of the module, e.g. "1.2.3.4"
139 product: HTTP-friendly product name
140 os: the operating system that the module was built for
141 cpu: the CPU that the module was built for
142 symbol_file: the contents of the breakpad-format symbol file
143
144 Args:
Mike Frysinger094a2172013-08-14 12:54:35 -0400145 upload_url: The crash URL to POST the |sym_file| to
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500146 sym_item: A SymbolItem containing the path to the breakpad symbol to upload
Mike Frysinger094a2172013-08-14 12:54:35 -0400147 """
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500148 sym_header = sym_item.sym_header
149 sym_file = sym_item.sym_file
Mike Frysinger094a2172013-08-14 12:54:35 -0400150
151 fields = (
152 ('code_file', sym_header.name),
153 ('debug_file', sym_header.name),
154 ('debug_identifier', sym_header.id.replace('-', '')),
155 # Should we set these fields? They aren't critical, but it might be nice?
156 # We'd have to figure out what file this symbol is coming from and what
157 # package provides it ...
158 #('version', None),
159 #('product', 'ChromeOS'),
160 ('os', sym_header.os),
161 ('cpu', sym_header.cpu),
162 poster.encode.MultipartParam.from_file('symbol_file', sym_file),
163 )
164
165 data, headers = poster.encode.multipart_encode(fields)
166 request = urllib2.Request(upload_url, data, headers)
167 request.add_header('User-agent', 'chromite.upload_symbols')
168 urllib2.urlopen(request, timeout=UPLOAD_TIMEOUT)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400169
170
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500171def TestingSymUpload(upload_url, sym_item):
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400172 """A stub version of SymUpload for --testing usage"""
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500173 cmd = ['sym_upload', sym_item.sym_file, upload_url]
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400174 # Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
175 returncode = random.randint(1, 100) <= 80
176 cros_build_lib.Debug('would run (and return %i): %s', returncode,
Matt Tennant7feda352013-12-20 14:03:40 -0800177 cros_build_lib.CmdToStr(cmd))
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400178 if returncode:
179 output = 'Failed to send the symbol file.'
180 else:
181 output = 'Successfully sent the symbol file.'
182 result = cros_build_lib.CommandResult(cmd=cmd, error=None, output=output,
183 returncode=returncode)
184 if returncode:
Mike Frysingera4fa1e82014-01-15 01:45:56 -0500185 exceptions = (
Mike Frysingerfd355652014-01-23 02:57:48 -0500186 socket.error('[socket.error] forced test fail'),
Mike Frysingera4fa1e82014-01-15 01:45:56 -0500187 httplib.BadStatusLine('[BadStatusLine] forced test fail'),
188 urllib2.HTTPError(upload_url, 400, '[HTTPError] forced test fail',
189 {}, None),
190 urllib2.URLError('[URLError] forced test fail'),
191 )
192 raise random.choice(exceptions)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400193 else:
194 return result
195
196
Mike Frysingereb753bf2013-11-22 16:05:35 -0500197def ErrorLimitHit(num_errors, watermark_errors):
198 """See if our error limit has been hit
199
200 Args:
201 num_errors: A multiprocessing.Value of the raw number of failures.
202 watermark_errors: A multiprocessing.Value of the current rate of failures.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500203
Mike Frysingereb753bf2013-11-22 16:05:35 -0500204 Returns:
205 True if our error limits have been exceeded.
206 """
207 return ((num_errors is not None and
208 num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY) or
209 (watermark_errors is not None and
210 watermark_errors.value > ERROR_WATERMARK))
211
212
213def _UpdateCounter(counter, adj):
214 """Update |counter| by |adj|
215
216 Handle atomic updates of |counter|. Also make sure it does not
217 fall below 0.
218
219 Args:
220 counter: A multiprocessing.Value to update
221 adj: The value to add to |counter|
222 """
223 def _Update():
224 clamp = 0 if type(adj) is int else 0.0
225 counter.value = max(clamp, counter.value + adj)
226
227 if hasattr(counter, 'get_lock'):
228 with counter.get_lock():
229 _Update()
230 elif counter is not None:
231 _Update()
232
233
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500234def UploadSymbol(upload_url, sym_item, file_limit=DEFAULT_FILE_LIMIT,
Mike Frysinger02e92402013-11-22 16:22:02 -0500235 sleep=0, num_errors=None, watermark_errors=None,
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500236 failed_queue=None, passed_queue=None):
237 """Upload |sym_item| to |upload_url|
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400238
239 Args:
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400240 upload_url: The crash server to upload things to
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500241 sym_item: A SymbolItem containing the path to the breakpad symbol to upload
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400242 file_limit: The max file size of a symbol file before we try to strip it
243 sleep: Number of seconds to sleep before running
Mike Frysinger69cb41d2013-08-11 20:08:19 -0400244 num_errors: An object to update with the error count (needs a .value member)
Mike Frysingereb753bf2013-11-22 16:05:35 -0500245 watermark_errors: An object to track current error behavior (needs a .value)
Mike Frysinger02e92402013-11-22 16:22:02 -0500246 failed_queue: When a symbol fails, add it to this queue
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500247 passed_queue: When a symbol passes, add it to this queue
Mike Frysinger1a736a82013-12-12 01:50:59 -0500248
Mike Frysinger69cb41d2013-08-11 20:08:19 -0400249 Returns:
250 The number of errors that were encountered.
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400251 """
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500252 sym_file = sym_item.sym_file
253 upload_item = sym_item
254
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400255 if num_errors is None:
256 num_errors = ctypes.c_int()
Mike Frysingereb753bf2013-11-22 16:05:35 -0500257 if ErrorLimitHit(num_errors, watermark_errors):
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400258 # Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
Mike Frysinger7f9be142014-01-15 02:16:42 -0500259 if failed_queue:
260 failed_queue.put(sym_file)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400261 return 0
262
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400263 if sleep:
264 # Keeps us from DoS-ing the symbol server.
265 time.sleep(sleep)
266
267 cros_build_lib.Debug('uploading %s' % sym_file)
268
269 # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
270 with tempfile.NamedTemporaryFile(prefix='upload_symbols',
271 bufsize=0) as temp_sym_file:
272 if file_limit:
273 # If the symbols size is too big, strip out the call frame info. The CFI
274 # is unnecessary for 32bit x86 targets where the frame pointer is used (as
275 # all of ours have) and it accounts for over half the size of the symbols
276 # uploaded.
277 file_size = os.path.getsize(sym_file)
278 if file_size > file_limit:
279 cros_build_lib.Warning('stripping CFI from %s due to size %s > %s',
280 sym_file, file_size, file_limit)
281 temp_sym_file.writelines([x for x in open(sym_file, 'rb').readlines()
282 if not x.startswith('STACK CFI')])
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500283
284 upload_item = FakeItem(sym_file=temp_sym_file.name,
285 sym_header=sym_item.sym_header)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400286
287 # Hopefully the crash server will let it through. But it probably won't.
288 # Not sure what the best answer is in this case.
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500289 file_size = os.path.getsize(upload_item.sym_file)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400290 if file_size > CRASH_SERVER_FILE_LIMIT:
291 cros_build_lib.PrintBuildbotStepWarnings()
Mike Frysinger02e92402013-11-22 16:22:02 -0500292 cros_build_lib.Warning('upload file %s is awfully large, risking '
293 'rejection by the symbol server (%s > %s)',
294 sym_file, file_size, CRASH_SERVER_FILE_LIMIT)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400295
296 # Upload the symbol file.
Mike Frysingereb753bf2013-11-22 16:05:35 -0500297 success = False
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400298 try:
Mike Frysinger9adcfd22013-10-24 12:01:40 -0400299 cros_build_lib.TimedCommand(
David Jamesc93e6a4d2014-01-13 11:37:36 -0800300 retry_util.RetryException,
Mike Frysinger9adcfd22013-10-24 12:01:40 -0400301 (urllib2.HTTPError, urllib2.URLError), MAX_RETRIES, SymUpload,
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500302 upload_url, upload_item, sleep=INITIAL_RETRY_DELAY,
Mike Frysinger9adcfd22013-10-24 12:01:40 -0400303 timed_log_msg='upload of %10i bytes took %%s: %s' %
304 (file_size, os.path.basename(sym_file)))
Mike Frysingereb753bf2013-11-22 16:05:35 -0500305 success = True
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500306
307 if passed_queue:
308 passed_queue.put(sym_item)
Mike Frysinger094a2172013-08-14 12:54:35 -0400309 except urllib2.HTTPError as e:
310 cros_build_lib.Warning('could not upload: %s: HTTP %s: %s',
311 os.path.basename(sym_file), e.code, e.reason)
Mike Frysingerfd355652014-01-23 02:57:48 -0500312 except (urllib2.URLError, httplib.HTTPException, socket.error) as e:
Mike Frysingerc4ab5782013-10-02 18:14:22 -0400313 cros_build_lib.Warning('could not upload: %s: %s',
314 os.path.basename(sym_file), e)
Mike Frysingereb753bf2013-11-22 16:05:35 -0500315 finally:
316 if success:
317 _UpdateCounter(watermark_errors, ERROR_ADJUST_PASS)
318 else:
319 _UpdateCounter(num_errors, 1)
320 _UpdateCounter(watermark_errors, ERROR_ADJUST_FAIL)
Mike Frysinger02e92402013-11-22 16:22:02 -0500321 if failed_queue:
322 failed_queue.put(sym_file)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400323
324 return num_errors.value
325
326
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500327# A dummy class that allows for stubbing in tests and SymUpload.
328FakeItem = cros_build_lib.Collection(
329 'FakeItem', sym_file=None, sym_header=None, content=lambda x: '')
330
331
332# TODO(build): Delete this if check. http://crbug.com/341152
333if isolateserver:
334 class SymbolItem(isolateserver.BufferItem):
335 """Turn a sym_file into an isolateserver.Item"""
336
337 ALGO = hashlib.sha1
338
339 def __init__(self, sym_file):
340 sym_header = cros_generate_breakpad_symbols.ReadSymsHeader(sym_file)
341 super(SymbolItem, self).__init__(str(sym_header), self.ALGO)
342 self.sym_header = sym_header
343 self.sym_file = sym_file
344
345
346def SymbolDeduplicatorNotify(dedupe_namespace, dedupe_queue):
347 """Send a symbol file to the swarming service
348
349 Notify the swarming service of a successful upload. If the notification fails
350 for any reason, we ignore it. We don't care as it just means we'll upload it
351 again later on, and the symbol server will handle that graciously.
352
353 This func runs in a different process from the main one, so we cannot share
354 the storage object. Instead, we create our own. This func stays alive for
355 the life of the process, so we only create one here overall.
356
357 Args:
358 dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
359 dedupe_queue: The queue to read SymbolItems from
360 """
361 if dedupe_queue is None:
362 return
363
364 item = None
365 try:
366 storage = isolateserver.get_storage_api(constants.ISOLATESERVER,
367 dedupe_namespace)
368 for item in iter(dedupe_queue.get, None):
369 with timeout_util.Timeout(DEDUPE_TIMEOUT):
370 storage.push(item, item.content(0))
371 except Exception:
372 sym_file = item.sym_file if (item and item.sym_file) else ''
373 cros_build_lib.Warning('posting %s to dedupe server failed',
374 os.path.basename(sym_file), exc_info=True)
375
376
377def SymbolDeduplicator(storage, sym_paths):
378 """Filter out symbol files that we've already uploaded
379
380 Using the swarming service, ask it to tell us which symbol files we've already
381 uploaded in previous runs and/or by other bots. If the query fails for any
382 reason, we'll just upload all symbols. This is fine as the symbol server will
383 do the right thing and this phase is purely an optimization.
384
385 This code runs in the main thread which is why we can re-use the existing
386 storage object. Saves us from having to recreate one all the time.
387
388 Args:
389 storage: An isolateserver.StorageApi object
390 sym_paths: List of symbol files to check against the dedupe server
391
392 Returns:
393 List of symbol files that have not been uploaded before
394 """
395 if not sym_paths:
396 return sym_paths
397
398 items = [SymbolItem(x) for x in sym_paths]
399 if storage:
400 try:
401 with timeout_util.Timeout(DEDUPE_TIMEOUT):
402 items = storage.contains(items)
403 except Exception:
404 cros_build_lib.Warning('talking to dedupe server failed', exc_info=True)
405
406 return items
407
408
Mike Frysingerd41938e2014-02-10 06:37:55 -0500409def IsTarball(path):
410 """Guess if this is a tarball based on the filename."""
411 parts = path.split('.')
412 if len(parts) <= 1:
413 return False
414
415 if parts[-1] == 'tar':
416 return True
417
418 if parts[-2] == 'tar':
419 return parts[-1] in ('bz2', 'gz', 'xz')
420
421 return parts[-1] in ('tbz2', 'tbz', 'tgz', 'txz')
422
423
424def SymbolFinder(tempdir, paths):
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500425 """Locate symbol files in |paths|
426
427 Args:
Mike Frysingerd41938e2014-02-10 06:37:55 -0500428 tempdir: Path to use for temporary files (caller will clean up).
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500429 paths: A list of input paths to walk. Files are returned w/out any checks.
Mike Frysingerd41938e2014-02-10 06:37:55 -0500430 Dirs are searched for files that end in ".sym". Urls are fetched and then
431 processed. Tarballs are unpacked and walked.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500432
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500433 Returns:
434 Yield every viable sym file.
435 """
436 for p in paths:
Mike Frysingerd41938e2014-02-10 06:37:55 -0500437 o = urlparse.urlparse(p)
438 if o.scheme:
439 # Support globs of filenames.
440 ctx = gs.GSContext()
441 for p in ctx.LS(p):
442 cros_build_lib.Info('processing files inside %s', p)
443 o = urlparse.urlparse(p)
444 cache_dir = commandline.GetCacheDir()
445 common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
446 tar_cache = cache.TarballCache(common_path)
447 key = ('%s%s' % (o.netloc, o.path)).split('/')
448 # The common cache will not be LRU, removing the need to hold a read
449 # lock on the cached gsutil.
450 ref = tar_cache.Lookup(key)
451 try:
452 ref.SetDefault(p)
453 except cros_build_lib.RunCommandError as e:
454 cros_build_lib.Warning('ignoring %s\n%s', p, e)
455 continue
456 for p in SymbolFinder(tempdir, [ref.path]):
457 yield p
458
459 elif os.path.isdir(p):
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500460 for root, _, files in os.walk(p):
461 for f in files:
462 if f.endswith('.sym'):
463 yield os.path.join(root, f)
Mike Frysingerd41938e2014-02-10 06:37:55 -0500464
465 elif IsTarball(p):
466 cros_build_lib.Info('processing files inside %s', p)
467 tardir = tempfile.mkdtemp(dir=tempdir)
468 cache.Untar(os.path.realpath(p), tardir)
469 for p in SymbolFinder(tardir, [tardir]):
470 yield p
471
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500472 else:
473 yield p
474
475
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500476def WriteQueueToFile(listing, queue, relpath=None):
477 """Write all the items in |queue| to the |listing|.
478
Mike Frysinger5e6dd712014-03-07 22:21:17 -0500479 Note: The queue must have a sentinel None appended to the end.
480
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500481 Args:
482 listing: Where to write out the list of files.
483 queue: The queue of paths to drain.
484 relpath: If set, write out paths relative to this one.
485 """
486 if not listing:
487 return
488
489 with cros_build_lib.Open(listing, 'wb+') as f:
Mike Frysinger5e6dd712014-03-07 22:21:17 -0500490 while True:
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500491 path = queue.get()
Mike Frysinger5e6dd712014-03-07 22:21:17 -0500492 if path is None:
493 return
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500494 if relpath:
495 path = os.path.relpath(path, relpath)
496 f.write('%s\n' % path)
497
498
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400499def UploadSymbols(board=None, official=False, breakpad_dir=None,
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400500 file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY,
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500501 upload_limit=None, sym_paths=None, failed_list=None,
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500502 root=None, retry=True, dedupe_namespace=None):
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400503 """Upload all the generated symbols for |board| to the crash server
504
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400505 You can use in a few ways:
506 * pass |board| to locate all of its symbols
507 * pass |breakpad_dir| to upload all the symbols in there
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500508 * pass |sym_paths| to upload specific symbols (or dirs of symbols)
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400509
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400510 Args:
511 board: The board whose symbols we wish to upload
512 official: Use the official symbol server rather than the staging one
513 breakpad_dir: The full path to the breakpad directory where symbols live
514 file_limit: The max file size of a symbol file before we try to strip it
515 sleep: How long to sleep in between uploads
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500516 upload_limit: If set, only upload this many symbols (meant for testing)
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500517 sym_paths: Specific symbol files (or dirs of sym files) to upload,
518 otherwise search |breakpad_dir|
Mike Frysinger7f9be142014-01-15 02:16:42 -0500519 failed_list: Write the names of all sym files we did not upload; can be a
520 filename or file-like object.
Mike Frysinger118d2502013-08-19 03:36:56 -0400521 root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set)
Mike Frysinger02e92402013-11-22 16:22:02 -0500522 retry: Whether we should retry failures.
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500523 dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500524
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400525 Returns:
Mike Frysinger69cb41d2013-08-11 20:08:19 -0400526 The number of errors that were encountered.
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400527 """
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500528 # TODO(build): Delete this assert.
529 assert isolateserver, 'Missing isolateserver import http://crbug.com/341152'
530
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400531 if official:
532 upload_url = OFFICIAL_UPLOAD_URL
533 else:
534 cros_build_lib.Warning('unofficial builds upload to the staging server')
535 upload_url = STAGING_UPLOAD_URL
536
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500537 if sym_paths:
538 cros_build_lib.Info('uploading specified symbols to %s', upload_url)
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400539 else:
540 if breakpad_dir is None:
Mike Frysinger118d2502013-08-19 03:36:56 -0400541 breakpad_dir = os.path.join(
542 root,
543 cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip('/'))
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400544 cros_build_lib.Info('uploading all symbols to %s from %s', upload_url,
545 breakpad_dir)
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500546 sym_paths = [breakpad_dir]
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400547
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500548 # We use storage_query to ask the server about existing symbols. The
549 # storage_notify_proc process is used to post updates to the server. We
550 # cannot safely share the storage object between threads/processes, but
551 # we also want to minimize creating new ones as each object has to init
552 # new state (like server connections).
553 if dedupe_namespace:
554 dedupe_limit = DEDUPE_LIMIT
555 dedupe_queue = multiprocessing.Queue()
556 storage_query = isolateserver.get_storage_api(constants.ISOLATESERVER,
557 dedupe_namespace)
558 else:
559 dedupe_limit = 1
560 dedupe_queue = storage_query = None
561 # Can't use parallel.BackgroundTaskRunner because that'll create multiple
562 # processes and we want only one the whole time (see comment above).
563 storage_notify_proc = multiprocessing.Process(
564 target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue))
565
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400566 bg_errors = multiprocessing.Value('i')
Mike Frysingereb753bf2013-11-22 16:05:35 -0500567 watermark_errors = multiprocessing.Value('f')
Mike Frysinger02e92402013-11-22 16:22:02 -0500568 failed_queue = multiprocessing.Queue()
569 uploader = functools.partial(
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500570 UploadSymbol, upload_url, file_limit=file_limit, sleep=sleep,
571 num_errors=bg_errors, watermark_errors=watermark_errors,
572 failed_queue=failed_queue, passed_queue=dedupe_queue)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400573
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500574 start_time = datetime.datetime.now()
575 Counters = cros_build_lib.Collection(
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500576 'Counters', upload_limit=upload_limit, uploaded_count=0, deduped_count=0)
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500577 counters = Counters()
578
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500579 def _Upload(queue, counters, files):
580 if not files:
581 return
582
583 missing_count = 0
584 for item in SymbolDeduplicator(storage_query, files):
585 if counters.upload_limit == 0:
586 break
587
588 missing_count += 1
589 queue.put((item,))
590 counters.uploaded_count += 1
591 if counters.upload_limit is not None:
592 counters.upload_limit -= 1
593
594 counters.deduped_count += (len(files) - missing_count)
595
Mike Frysingerd41938e2014-02-10 06:37:55 -0500596 with osutils.TempDir(prefix='upload_symbols.') as tempdir:
597 storage_notify_proc.start()
598 # For the first run, we collect the symbols that failed. If the
599 # overall failure rate was low, we'll retry them on the second run.
600 for retry in (retry, False):
601 # We need to limit ourselves to one upload at a time to avoid the server
602 # kicking in DoS protection. See these bugs for more details:
603 # http://crbug.com/209442
604 # http://crbug.com/212496
605 with parallel.BackgroundTaskRunner(uploader, processes=1) as queue:
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500606 dedupe_list = []
Mike Frysingerd41938e2014-02-10 06:37:55 -0500607 for sym_file in SymbolFinder(tempdir, sym_paths):
608 dedupe_list.append(sym_file)
609 dedupe_len = len(dedupe_list)
610 if dedupe_len < dedupe_limit:
611 if (counters.upload_limit is None or
612 dedupe_len < counters.upload_limit):
613 continue
Mike Frysinger02e92402013-11-22 16:22:02 -0500614
Mike Frysingerd41938e2014-02-10 06:37:55 -0500615 _Upload(queue, counters, dedupe_list)
616 dedupe_list = []
617 _Upload(queue, counters, dedupe_list)
Mike Frysinger02e92402013-11-22 16:22:02 -0500618
Mike Frysingerd41938e2014-02-10 06:37:55 -0500619 # See if we need to retry, and if we haven't failed too many times yet.
620 if not retry or ErrorLimitHit(bg_errors, watermark_errors):
621 break
622
623 sym_paths = []
Mike Frysinger5e6dd712014-03-07 22:21:17 -0500624 failed_queue.put(None)
625 while True:
626 sym_path = failed_queue.get()
627 if sym_path is None:
628 break
629 sym_paths.append(sym_path)
630
Mike Frysingerd41938e2014-02-10 06:37:55 -0500631 if sym_paths:
632 cros_build_lib.Warning('retrying %i symbols', len(sym_paths))
633 if counters.upload_limit is not None:
634 counters.upload_limit += len(sym_paths)
635 # Decrement the error count in case we recover in the second pass.
636 assert bg_errors.value >= len(sym_paths), \
637 'more failed files than errors?'
638 bg_errors.value -= len(sym_paths)
639 else:
640 # No failed symbols, so just return now.
641 break
Mike Frysinger02e92402013-11-22 16:22:02 -0500642
Mike Frysinger7f9be142014-01-15 02:16:42 -0500643 # If the user has requested it, save all the symbol files that we failed to
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500644 # upload to a listing file. This should help with recovery efforts later.
Mike Frysinger5e6dd712014-03-07 22:21:17 -0500645 failed_queue.put(None)
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500646 WriteQueueToFile(failed_list, failed_queue, breakpad_dir)
Mike Frysinger7f9be142014-01-15 02:16:42 -0500647
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500648 if dedupe_queue:
649 dedupe_queue.put(None)
650 dedupe_queue.close()
651 storage_notify_proc.join()
652
653 cros_build_lib.Info('uploaded %i symbols (%i were deduped) which took: %s',
654 counters.uploaded_count, counters.deduped_count,
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500655 datetime.datetime.now() - start_time)
656
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500657 return bg_errors.value
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400658
659
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400660def main(argv):
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500661 # TODO(build): Delete this assert.
662 assert isolateserver, 'Missing isolateserver import http://crbug.com/341152'
663
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400664 parser = commandline.ArgumentParser(description=__doc__)
665
Mike Frysingerd41938e2014-02-10 06:37:55 -0500666 parser.add_argument('sym_paths', type='path_or_uri', nargs='*', default=None,
667 help='symbol file or directory or URL or tarball')
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400668 parser.add_argument('--board', default=None,
669 help='board to build packages for')
670 parser.add_argument('--breakpad_root', type='path', default=None,
671 help='root directory for breakpad symbols')
672 parser.add_argument('--official_build', action='store_true', default=False,
673 help='point to official symbol server')
674 parser.add_argument('--regenerate', action='store_true', default=False,
675 help='regenerate all symbols')
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500676 parser.add_argument('--upload-limit', type=int, default=None,
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400677 help='only upload # number of symbols')
678 parser.add_argument('--strip_cfi', type=int,
679 default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
680 help='strip CFI data for files above this size')
Mike Frysinger7f9be142014-01-15 02:16:42 -0500681 parser.add_argument('--failed-list', type='path',
682 help='where to save a list of failed symbols')
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500683 parser.add_argument('--dedupe', action='store_true', default=False,
684 help='use the swarming service to avoid re-uploading')
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400685 parser.add_argument('--testing', action='store_true', default=False,
686 help='run in testing mode')
687 parser.add_argument('--yes', action='store_true', default=False,
688 help='answer yes to all prompts')
689
690 opts = parser.parse_args(argv)
Mike Frysinger90e49ca2014-01-14 14:42:07 -0500691 opts.Freeze()
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400692
Mike Frysinger9b2ff5c2013-11-22 10:01:12 -0500693 if opts.sym_paths:
Mike Frysinger9dcf9ae2013-08-10 15:17:09 -0400694 if opts.regenerate:
695 cros_build_lib.Die('--regenerate may not be used with specific files')
696 else:
697 if opts.board is None:
698 cros_build_lib.Die('--board is required')
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400699
700 if opts.breakpad_root and opts.regenerate:
701 cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
702
703 if opts.testing:
704 # TODO(build): Kill off --testing mode once unittests are up-to-snuff.
705 cros_build_lib.Info('running in testing mode')
706 # pylint: disable=W0601,W0603
707 global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
708 INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
709 SymUpload = TestingSymUpload
710
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500711 dedupe_namespace = None
712 if opts.dedupe:
713 if opts.official_build and not opts.testing:
714 dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE
715 else:
716 dedupe_namespace = STAGING_DEDUPE_NAMESPACE
717
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400718 if not opts.yes:
Mike Frysingerc5de9602014-02-09 02:42:36 -0500719 prolog = '\n'.join(textwrap.wrap(textwrap.dedent("""
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400720 Uploading symbols for an entire Chromium OS build is really only
721 necessary for release builds and in a few cases for developers
722 to debug problems. It will take considerable time to run. For
723 developer debugging purposes, consider instead passing specific
724 files to upload.
Mike Frysingerc5de9602014-02-09 02:42:36 -0500725 """), 80)).strip()
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400726 if not cros_build_lib.BooleanPrompt(
727 prompt='Are you sure you want to upload all build symbols',
Mike Frysingerc5de9602014-02-09 02:42:36 -0500728 default=False, prolog=prolog):
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400729 cros_build_lib.Die('better safe than sorry')
730
731 ret = 0
732 if opts.regenerate:
Mike Frysinger69cb41d2013-08-11 20:08:19 -0400733 ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols(
734 opts.board, breakpad_dir=opts.breakpad_root)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400735
736 ret += UploadSymbols(opts.board, official=opts.official_build,
737 breakpad_dir=opts.breakpad_root,
738 file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY,
Mike Frysinger8ec8c502014-02-10 00:19:13 -0500739 upload_limit=opts.upload_limit, sym_paths=opts.sym_paths,
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500740 failed_list=opts.failed_list,
741 dedupe_namespace=dedupe_namespace)
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400742 if ret:
743 cros_build_lib.Error('encountered %i problem(s)', ret)
744 # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
745 # return 0 in case we are a multiple of the mask.
746 ret = 1
747
748 return ret
Mike Frysinger094a2172013-08-14 12:54:35 -0400749
750
751# We need this to run once per process. Do it at module import time as that
752# will let us avoid doing it inline at function call time (see SymUpload) as
753# that func might be called by the multiprocessing module which means we'll
754# do the opener logic multiple times overall. Plus, if you're importing this
755# module, it's a pretty good chance that you're going to need this.
756poster.streaminghttp.register_openers()