blob: 77dde832b9deeafad32c29f6c6cf90069455199c [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -04002# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Mike Frysingereb753bf2013-11-22 16:05:35 -05006"""Unittests for upload_symbols.py"""
7
Mike Frysinger7f9be142014-01-15 02:16:42 -05008from __future__ import print_function
9
Mike Frysinger0a2fd922014-09-12 20:23:42 -070010import BaseHTTPServer
Mike Frysinger06da5202014-09-26 17:30:33 -050011import errno
Don Garretta28be6d2016-06-16 18:09:35 -070012import itertools
Mike Frysingerea838d12014-12-08 11:55:32 -050013import mock
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040014import os
Mike Frysinger0a2fd922014-09-12 20:23:42 -070015import signal
Mike Frysinger06da5202014-09-26 17:30:33 -050016import socket
Mike Frysinger0a2fd922014-09-12 20:23:42 -070017import SocketServer
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040018import sys
Aviv Keshetd1f04632014-05-09 11:33:46 -070019import time
Mike Frysinger094a2172013-08-14 12:54:35 -040020import urllib2
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040021
Mike Frysinger079863c2014-10-09 23:16:46 -040022# We specifically set up a local server to connect to, so make sure we
23# delete any proxy settings that might screw that up. We also need to
24# do it here because modules that are imported below will implicitly
25# initialize with this proxy setting rather than dynamically pull it
26# on the fly :(.
27os.environ.pop('http_proxy', None)
28
Aviv Keshetb7519e12016-10-04 00:50:00 -070029from chromite.lib import constants
Mike Frysingerbbd1f112016-09-08 18:25:11 -040030
31# The isolateserver includes a bunch of third_party python packages that clash
32# with chromite's bundled third_party python packages (like oauth2client).
33# Since upload_symbols is not imported in to other parts of chromite, and there
34# are no deps in third_party we care about, purge the chromite copy. This way
35# we can use isolateserver for deduping.
36# TODO: If we ever sort out third_party/ handling and make it per-script opt-in,
37# we can purge this logic.
38third_party = os.path.join(constants.CHROMITE_DIR, 'third_party')
39while True:
40 try:
41 sys.path.remove(third_party)
42 except ValueError:
43 break
Mike Frysingerbbd1f112016-09-08 18:25:11 -040044del third_party
45
Ralph Nathan446aee92015-03-23 14:44:56 -070046from chromite.lib import cros_logging as logging
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040047from chromite.lib import cros_test_lib
48from chromite.lib import osutils
49from chromite.lib import parallel
Mike Frysinger0a2fd922014-09-12 20:23:42 -070050from chromite.lib import remote_access
Mike Frysinger5e30a4b2014-02-12 20:23:04 -050051from chromite.scripts import cros_generate_breakpad_symbols
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -040052from chromite.scripts import upload_symbols
53
Mike Frysinger0c0efa22014-02-09 23:32:23 -050054
Don Garretta28be6d2016-06-16 18:09:35 -070055class SymbolsTestBase(cros_test_lib.MockTempDirTestCase):
56 """Base class for most symbols tests."""
57
58 SLIM_CONTENT = """
59some junk
60"""
61
62 FAT_CONTENT = """
63STACK CFI 1234
64some junk
65STACK CFI 1234
66"""
67
68 def setUp(self):
69 # Make certain we don't use the network.
Don Garretta28be6d2016-06-16 18:09:35 -070070 self.urlopen_mock = self.PatchObject(urllib2, 'urlopen')
Mike Nichols90f7c152019-04-09 15:14:08 -060071 self.request_mock = self.PatchObject(upload_symbols, 'ExecRequest',
72 return_value={'uploadUrl':
73 'testurl',
74 'uploadKey':
75 'asdgasgas'})
Don Garretta28be6d2016-06-16 18:09:35 -070076
77 # Make 'uploads' go fast.
78 self.PatchObject(upload_symbols, 'SLEEP_DELAY', 0)
79 self.PatchObject(upload_symbols, 'INITIAL_RETRY_DELAY', 0)
80
81 # So our symbol file content doesn't have to be real.
82 self.PatchObject(cros_generate_breakpad_symbols, 'ReadSymsHeader',
83 return_value=cros_generate_breakpad_symbols.SymbolHeader(
84 os='os', cpu='cpu', id='id', name='name'))
85
86 self.working = os.path.join(self.tempdir, 'expand')
87 osutils.SafeMakedirs(self.working)
88
89 self.data = os.path.join(self.tempdir, 'data')
90 osutils.SafeMakedirs(self.data)
91
92 def createSymbolFile(self, filename, content=FAT_CONTENT, size=0,
93 status=None, dedupe=False):
94 fullname = os.path.join(self.data, filename)
95 osutils.SafeMakedirs(os.path.dirname(fullname))
96
97 # If a file size is given, force that to be the minimum file size. Create
98 # a sparse file so large files are practical.
99 with open(fullname, 'w+b') as f:
100 f.truncate(size)
101 f.seek(0)
102 f.write(content)
103
104 result = upload_symbols.SymbolFile(display_path=filename,
105 file_name=fullname)
106
107 if status:
108 result.status = status
109
110 if dedupe:
111 result.dedupe_item = upload_symbols.DedupeItem(result)
112 result.dedupe_push_state = 'push_state'
113
114 return result
115
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400116
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700117class SymbolServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
118 """HTTP handler for symbol POSTs"""
119
120 RESP_CODE = None
121 RESP_MSG = None
122
123 def do_POST(self):
124 """Handle a POST request"""
125 # Drain the data from the client. If we don't, we might write the response
126 # and close the socket before the client finishes, so they die with EPIPE.
127 clen = int(self.headers.get('Content-Length', '0'))
128 self.rfile.read(clen)
129
130 self.send_response(self.RESP_CODE, self.RESP_MSG)
131 self.end_headers()
132
133 def log_message(self, *args, **kwargs):
134 """Stub the logger as it writes to stderr"""
135 pass
136
137
138class SymbolServer(SocketServer.ThreadingTCPServer, BaseHTTPServer.HTTPServer):
139 """Simple HTTP server that forks each request"""
140
141
142class UploadSymbolsServerTest(cros_test_lib.MockTempDirTestCase):
143 """Tests for UploadSymbols() and a local HTTP server"""
144
145 SYM_CONTENTS = """MODULE Linux arm 123-456 blkid
146PUBLIC 1471 0 main"""
147
148 def SpawnServer(self, RequestHandler):
149 """Spawn a new http server"""
Mike Frysinger06da5202014-09-26 17:30:33 -0500150 while True:
151 try:
152 port = remote_access.GetUnusedPort()
153 address = ('', port)
154 self.httpd = SymbolServer(address, RequestHandler)
155 break
156 except socket.error as e:
157 if e.errno == errno.EADDRINUSE:
158 continue
159 raise
Don Garretta28be6d2016-06-16 18:09:35 -0700160 self.server_url = 'http://localhost:%i/post/path' % port
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700161 self.httpd_pid = os.fork()
162 if self.httpd_pid == 0:
163 self.httpd.serve_forever(poll_interval=0.1)
164 sys.exit(0)
165
166 def setUp(self):
167 self.httpd_pid = None
168 self.httpd = None
Don Garretta28be6d2016-06-16 18:09:35 -0700169 self.server_url = None
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700170 self.sym_file = os.path.join(self.tempdir, 'test.sym')
171 osutils.WriteFile(self.sym_file, self.SYM_CONTENTS)
172
Don Garretta28be6d2016-06-16 18:09:35 -0700173 # Stop sleeps and retries for these tests.
174 self.PatchObject(upload_symbols, 'SLEEP_DELAY', 0)
175 self.PatchObject(upload_symbols, 'INITIAL_RETRY_DELAY', 0)
176 self.PatchObject(upload_symbols, 'MAX_RETRIES', 0)
177
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700178 def tearDown(self):
179 # Only kill the server if we forked one.
180 if self.httpd_pid:
181 os.kill(self.httpd_pid, signal.SIGUSR1)
182
183 def testSuccess(self):
184 """The server returns success for all uploads"""
185 class Handler(SymbolServerRequestHandler):
186 """Always return 200"""
187 RESP_CODE = 200
Mike Nichols90f7c152019-04-09 15:14:08 -0600188 self.PatchObject(upload_symbols, 'ExecRequest',
189 return_value={'uploadUrl': 'testurl',
190 'uploadKey': 'testSuccess'})
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700191 self.SpawnServer(Handler)
Don Garretta28be6d2016-06-16 18:09:35 -0700192 ret = upload_symbols.UploadSymbols(
193 sym_paths=[self.sym_file] * 10,
194 upload_url=self.server_url,
Mike Nichols90f7c152019-04-09 15:14:08 -0600195 api_key='testSuccess')
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700196 self.assertEqual(ret, 0)
197
198 def testError(self):
199 """The server returns errors for all uploads"""
200 class Handler(SymbolServerRequestHandler):
Mike Nichols90f7c152019-04-09 15:14:08 -0600201 """All connections error"""
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700202 RESP_CODE = 500
203 RESP_MSG = 'Internal Server Error'
204
205 self.SpawnServer(Handler)
Don Garretta28be6d2016-06-16 18:09:35 -0700206 ret = upload_symbols.UploadSymbols(
207 sym_paths=[self.sym_file] * 10,
208 upload_url=self.server_url,
Mike Nichols90f7c152019-04-09 15:14:08 -0600209 api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700210 self.assertEqual(ret, 10)
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700211
212 def testHungServer(self):
213 """The server chokes, but we recover"""
214 class Handler(SymbolServerRequestHandler):
215 """All connections choke forever"""
Mike Nichols137e82d2019-05-15 18:40:34 -0600216 self.PatchObject(upload_symbols, 'ExecRequest',
217 return_value={'pairs': []})
218
Mike Frysinger0a2fd922014-09-12 20:23:42 -0700219 def do_POST(self):
220 while True:
221 time.sleep(1000)
222
223 self.SpawnServer(Handler)
224 with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m:
Don Garretta28be6d2016-06-16 18:09:35 -0700225 m.return_value = 0.01
Mike Frysinger58312e92014-03-18 04:18:36 -0400226 ret = upload_symbols.UploadSymbols(
Don Garretta28be6d2016-06-16 18:09:35 -0700227 sym_paths=[self.sym_file] * 10,
228 upload_url=self.server_url,
Mike Nichols137e82d2019-05-15 18:40:34 -0600229 timeout=m.return_value,
Mike Nichols90f7c152019-04-09 15:14:08 -0600230 api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700231 self.assertEqual(ret, 10)
Aviv Keshetd1f04632014-05-09 11:33:46 -0700232
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400233
Don Garretta28be6d2016-06-16 18:09:35 -0700234class UploadSymbolsHelpersTest(cros_test_lib.TestCase):
235 """Test assorted helper functions and classes."""
Don Garretta28be6d2016-06-16 18:09:35 -0700236 def testIsTarball(self):
237 notTar = [
238 '/foo/bar/test.bin',
239 '/foo/bar/test.tar.bin',
240 '/foo/bar/test.faketar.gz',
241 '/foo/bar/test.nottgz',
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500242 ]
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500243
Don Garretta28be6d2016-06-16 18:09:35 -0700244 isTar = [
245 '/foo/bar/test.tar',
246 '/foo/bar/test.bin.tar',
247 '/foo/bar/test.bin.tar.bz2',
248 '/foo/bar/test.bin.tar.gz',
249 '/foo/bar/test.bin.tar.xz',
250 '/foo/bar/test.tbz2',
251 '/foo/bar/test.tbz',
252 '/foo/bar/test.tgz',
253 '/foo/bar/test.txz',
254 ]
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500255
Don Garretta28be6d2016-06-16 18:09:35 -0700256 for p in notTar:
257 self.assertFalse(upload_symbols.IsTarball(p))
258
259 for p in isTar:
260 self.assertTrue(upload_symbols.IsTarball(p))
261
262 def testBatchGenerator(self):
263 result = upload_symbols.BatchGenerator([], 2)
264 self.assertEqual(list(result), [])
265
Mike Frysinger66848312019-07-03 02:12:39 -0400266 # BatchGenerator accepts iterators, so passing it here is safe.
267 # pylint: disable=range-builtin-not-iterating
Mike Frysinger79cca962019-06-13 15:26:53 -0400268 result = upload_symbols.BatchGenerator(range(6), 2)
Don Garretta28be6d2016-06-16 18:09:35 -0700269 self.assertEqual(list(result), [[0, 1], [2, 3], [4, 5]])
270
Mike Frysinger79cca962019-06-13 15:26:53 -0400271 result = upload_symbols.BatchGenerator(range(7), 2)
Don Garretta28be6d2016-06-16 18:09:35 -0700272 self.assertEqual(list(result), [[0, 1], [2, 3], [4, 5], [6]])
273
274 # Prove that we are streaming the results, not generating them all at once.
275 result = upload_symbols.BatchGenerator(itertools.repeat(0), 2)
Mike Frysinger67d90242019-07-03 19:03:58 -0400276 self.assertEqual(next(result), [0, 0])
Don Garretta28be6d2016-06-16 18:09:35 -0700277
278
279class FindSymbolFilesTest(SymbolsTestBase):
280 """Test FindSymbolFiles."""
281 def setUp(self):
282 self.symfile = self.createSymbolFile('root.sym').file_name
283 self.innerfile = self.createSymbolFile(
284 os.path.join('nested', 'inner.sym')).file_name
285
286 # CreateTarball is having issues outside the chroot from open file tests.
287 #
288 # self.tarball = os.path.join(self.tempdir, 'syms.tar.gz')
289 # cros_build_lib.CreateTarball(
290 # 'syms.tar.gz', self.tempdir, inputs=(self.data))
291
292 def testEmpty(self):
293 symbols = list(upload_symbols.FindSymbolFiles(
294 self.working, []))
295 self.assertEqual(symbols, [])
296
297 def testFile(self):
298 symbols = list(upload_symbols.FindSymbolFiles(
299 self.working, [self.symfile]))
300
301 self.assertEqual(len(symbols), 1)
302 sf = symbols[0]
303
304 self.assertEqual(sf.display_name, 'root.sym')
305 self.assertEqual(sf.display_path, self.symfile)
306 self.assertEqual(sf.file_name, self.symfile)
307 self.assertEqual(sf.status, upload_symbols.SymbolFile.INITIAL)
308 self.assertEqual(sf.FileSize(), len(self.FAT_CONTENT))
309
310 def testDir(self):
311 symbols = list(upload_symbols.FindSymbolFiles(
312 self.working, [self.data]))
313
314 self.assertEqual(len(symbols), 2)
315 root = symbols[0]
316 nested = symbols[1]
317
318 self.assertEqual(root.display_name, 'root.sym')
319 self.assertEqual(root.display_path, 'root.sym')
320 self.assertEqual(root.file_name, self.symfile)
321 self.assertEqual(root.status, upload_symbols.SymbolFile.INITIAL)
322 self.assertEqual(root.FileSize(), len(self.FAT_CONTENT))
323
324 self.assertEqual(nested.display_name, 'inner.sym')
325 self.assertEqual(nested.display_path, 'nested/inner.sym')
326 self.assertEqual(nested.file_name, self.innerfile)
327 self.assertEqual(nested.status, upload_symbols.SymbolFile.INITIAL)
328 self.assertEqual(nested.FileSize(), len(self.FAT_CONTENT))
329
330
331class AdjustSymbolFileSizeTest(SymbolsTestBase):
332 """Test AdjustSymbolFileSize."""
333 def setUp(self):
334 self.slim = self.createSymbolFile('slim.sym', self.SLIM_CONTENT)
335 self.fat = self.createSymbolFile('fat.sym', self.FAT_CONTENT)
336
337 self.warn_mock = self.PatchObject(logging, 'PrintBuildbotStepWarnings')
338
339 def _testNotStripped(self, symbol, size=None, content=None):
340 start_file = symbol.file_name
341 after = upload_symbols.AdjustSymbolFileSize(
342 symbol, self.working, size)
343 self.assertIs(after, symbol)
344 self.assertEqual(after.file_name, start_file)
345 if content is not None:
346 self.assertEqual(osutils.ReadFile(after.file_name), content)
347
348 def _testStripped(self, symbol, size=None, content=None):
349 after = upload_symbols.AdjustSymbolFileSize(
350 symbol, self.working, size)
351 self.assertIs(after, symbol)
352 self.assertTrue(after.file_name.startswith(self.working))
353 if content is not None:
354 self.assertEqual(osutils.ReadFile(after.file_name), content)
355
356 def testSmall(self):
357 """Ensure that files smaller than the limit are not modified."""
358 self._testNotStripped(self.slim, 1024, self.SLIM_CONTENT)
359 self._testNotStripped(self.fat, 1024, self.FAT_CONTENT)
360
361 def testLarge(self):
362 """Ensure that files larger than the limit are modified."""
363 self._testStripped(self.slim, 1, self.SLIM_CONTENT)
364 self._testStripped(self.fat, 1, self.SLIM_CONTENT)
365
366 def testMixed(self):
367 """Test mix of large and small."""
368 strip_size = len(self.SLIM_CONTENT) + 1
369
370 self._testNotStripped(self.slim, strip_size, self.SLIM_CONTENT)
371 self._testStripped(self.fat, strip_size, self.SLIM_CONTENT)
372
373 def testSizeWarnings(self):
374 large = self.createSymbolFile(
375 'large.sym', content=self.SLIM_CONTENT,
376 size=upload_symbols.CRASH_SERVER_FILE_LIMIT*2)
377
378 # Would like to Strip as part of this test, but that really copies all
379 # of the sparse file content, which is too expensive for a unittest.
380 self._testNotStripped(large, None, None)
381
382 self.assertEqual(self.warn_mock.call_count, 1)
383
384
Mike Nichols137e82d2019-05-15 18:40:34 -0600385class DeduplicateTest(SymbolsTestBase):
386 """Test server Deduplication."""
387 def setUp(self):
388 self.PatchObject(upload_symbols, 'ExecRequest',
389 return_value={'pairs': [
390 {'status': 'FOUND',
391 'symbolId':
392 {'debugFile': 'sym1_sym',
393 'debugId': 'BEAA9BE'}},
394 {'status': 'FOUND',
395 'symbolId':
396 {'debugFile': 'sym2_sym',
397 'debugId': 'B6B1A36'}},
398 {'status': 'MISSING',
399 'symbolId':
400 {'debugFile': 'sym3_sym',
401 'debugId': 'D4FC0FC'}}]})
402
403 def testFindDuplicates(self):
404 # The first two symbols will be duplicate, the third new.
405 sym1 = self.createSymbolFile('sym1.sym')
406 sym1.header = cros_generate_breakpad_symbols.SymbolHeader('cpu', 'BEAA9BE',
407 'sym1_sym', 'os')
408 sym2 = self.createSymbolFile('sym2.sym')
409 sym2.header = cros_generate_breakpad_symbols.SymbolHeader('cpu', 'B6B1A36',
410 'sym2_sym', 'os')
411 sym3 = self.createSymbolFile('sym3.sym')
412 sym3.header = cros_generate_breakpad_symbols.SymbolHeader('cpu', 'D4FC0FC',
413 'sym3_sym', 'os')
414
415 result = upload_symbols.FindDuplicates((sym1, sym2, sym3), 'fake_url',
416 api_key='testkey')
417 self.assertEqual(list(result), [sym1, sym2, sym3])
418
419 self.assertEqual(sym1.status, upload_symbols.SymbolFile.DUPLICATE)
420 self.assertEqual(sym2.status, upload_symbols.SymbolFile.DUPLICATE)
421 self.assertEqual(sym3.status, upload_symbols.SymbolFile.INITIAL)
422
423
Don Garrettdeb2e032016-07-06 16:44:14 -0700424class PerformSymbolFilesUploadTest(SymbolsTestBase):
Don Garretta28be6d2016-06-16 18:09:35 -0700425 """Test PerformSymbolFile, and it's helper methods."""
426 def setUp(self):
Don Garrettdeb2e032016-07-06 16:44:14 -0700427 self.sym_initial = self.createSymbolFile(
428 'initial.sym')
429 self.sym_error = self.createSymbolFile(
430 'error.sym', status=upload_symbols.SymbolFile.ERROR)
431 self.sym_duplicate = self.createSymbolFile(
432 'duplicate.sym', status=upload_symbols.SymbolFile.DUPLICATE)
433 self.sym_uploaded = self.createSymbolFile(
434 'uploaded.sym', status=upload_symbols.SymbolFile.UPLOADED)
Don Garretta28be6d2016-06-16 18:09:35 -0700435
436 def testGetUploadTimeout(self):
437 """Test GetUploadTimeout helper function."""
438 # Timeout for small file.
Don Garrettdeb2e032016-07-06 16:44:14 -0700439 self.assertEqual(upload_symbols.GetUploadTimeout(self.sym_initial),
Don Garretta28be6d2016-06-16 18:09:35 -0700440 upload_symbols.UPLOAD_MIN_TIMEOUT)
441
442 # Timeout for 300M file.
443 large = self.createSymbolFile('large.sym', size=(300 * 1024 * 1024))
Ryo Hashimotof864c0e2017-02-14 12:46:08 +0900444 self.assertEqual(upload_symbols.GetUploadTimeout(large), 771)
Don Garretta28be6d2016-06-16 18:09:35 -0700445
446 def testUploadSymbolFile(self):
Mike Nichols90f7c152019-04-09 15:14:08 -0600447 upload_symbols.UploadSymbolFile('fake_url', self.sym_initial,
448 api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700449 # TODO: Examine mock in more detail to make sure request is correct.
Mike Nichols137e82d2019-05-15 18:40:34 -0600450 self.assertEqual(self.request_mock.call_count, 3)
Don Garretta28be6d2016-06-16 18:09:35 -0700451
Don Garrettdeb2e032016-07-06 16:44:14 -0700452 def testPerformSymbolsFileUpload(self):
Don Garretta28be6d2016-06-16 18:09:35 -0700453 """We upload on first try."""
Don Garrettdeb2e032016-07-06 16:44:14 -0700454 symbols = [self.sym_initial]
Don Garretta28be6d2016-06-16 18:09:35 -0700455
Don Garrettdeb2e032016-07-06 16:44:14 -0700456 result = upload_symbols.PerformSymbolsFileUpload(
Mike Nichols90f7c152019-04-09 15:14:08 -0600457 symbols, 'fake_url', api_key='testkey')
Don Garrettdeb2e032016-07-06 16:44:14 -0700458
459 self.assertEqual(list(result), symbols)
460 self.assertEqual(self.sym_initial.status,
461 upload_symbols.SymbolFile.UPLOADED)
Mike Nichols137e82d2019-05-15 18:40:34 -0600462 self.assertEqual(self.request_mock.call_count, 3)
Don Garretta28be6d2016-06-16 18:09:35 -0700463
Don Garrettdeb2e032016-07-06 16:44:14 -0700464 def testPerformSymbolsFileUploadFailure(self):
Don Garretta28be6d2016-06-16 18:09:35 -0700465 """All network requests fail."""
Mike Nichols90f7c152019-04-09 15:14:08 -0600466 self.request_mock.side_effect = urllib2.URLError('network failure')
Don Garrettdeb2e032016-07-06 16:44:14 -0700467 symbols = [self.sym_initial]
Don Garretta28be6d2016-06-16 18:09:35 -0700468
Don Garrettdeb2e032016-07-06 16:44:14 -0700469 result = upload_symbols.PerformSymbolsFileUpload(
Mike Nichols90f7c152019-04-09 15:14:08 -0600470 symbols, 'fake_url', api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700471
Don Garrettdeb2e032016-07-06 16:44:14 -0700472 self.assertEqual(list(result), symbols)
473 self.assertEqual(self.sym_initial.status, upload_symbols.SymbolFile.ERROR)
Mike Nichols90f7c152019-04-09 15:14:08 -0600474 self.assertEqual(self.request_mock.call_count, 7)
Don Garretta28be6d2016-06-16 18:09:35 -0700475
Don Garrettdeb2e032016-07-06 16:44:14 -0700476 def testPerformSymbolsFileUploadTransisentFailure(self):
Don Garretta28be6d2016-06-16 18:09:35 -0700477 """We fail once, then succeed."""
478 self.urlopen_mock.side_effect = (urllib2.URLError('network failure'), None)
Don Garrettdeb2e032016-07-06 16:44:14 -0700479 symbols = [self.sym_initial]
Don Garretta28be6d2016-06-16 18:09:35 -0700480
Don Garrettdeb2e032016-07-06 16:44:14 -0700481 result = upload_symbols.PerformSymbolsFileUpload(
Mike Nichols90f7c152019-04-09 15:14:08 -0600482 symbols, 'fake_url', api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700483
Don Garrettdeb2e032016-07-06 16:44:14 -0700484 self.assertEqual(list(result), symbols)
485 self.assertEqual(self.sym_initial.status,
486 upload_symbols.SymbolFile.UPLOADED)
Mike Nichols137e82d2019-05-15 18:40:34 -0600487 self.assertEqual(self.request_mock.call_count, 3)
Don Garrettdeb2e032016-07-06 16:44:14 -0700488
489 def testPerformSymbolsFileUploadMixed(self):
490 """Upload symbols in mixed starting states.
491
492 Demonstrate that INITIAL and ERROR are uploaded, but DUPLICATE/UPLOADED are
493 ignored.
494 """
495 symbols = [self.sym_initial, self.sym_error,
496 self.sym_duplicate, self.sym_uploaded]
497
498 result = upload_symbols.PerformSymbolsFileUpload(
Mike Nichols90f7c152019-04-09 15:14:08 -0600499 symbols, 'fake_url', api_key='testkey')
Don Garrettdeb2e032016-07-06 16:44:14 -0700500
501 #
502 self.assertEqual(list(result), symbols)
503 self.assertEqual(self.sym_initial.status,
504 upload_symbols.SymbolFile.UPLOADED)
505 self.assertEqual(self.sym_error.status,
506 upload_symbols.SymbolFile.UPLOADED)
507 self.assertEqual(self.sym_duplicate.status,
508 upload_symbols.SymbolFile.DUPLICATE)
509 self.assertEqual(self.sym_uploaded.status,
510 upload_symbols.SymbolFile.UPLOADED)
Mike Nichols137e82d2019-05-15 18:40:34 -0600511 self.assertEqual(self.request_mock.call_count, 6)
Don Garrettdeb2e032016-07-06 16:44:14 -0700512
513
514 def testPerformSymbolsFileUploadErrorOut(self):
515 """Demonstate we exit only after X errors."""
516
517 symbol_count = upload_symbols.MAX_TOTAL_ERRORS_FOR_RETRY + 10
518 symbols = []
519 fail_file = None
520
521 # potentially twice as many errors as we should attempt.
Mike Frysinger79cca962019-06-13 15:26:53 -0400522 for _ in range(symbol_count):
Don Garrettdeb2e032016-07-06 16:44:14 -0700523 # Each loop will get unique SymbolFile instances that use the same files.
524 fail = self.createSymbolFile('fail.sym')
525 fail_file = fail.file_name
526 symbols.append(self.createSymbolFile('pass.sym'))
527 symbols.append(fail)
528
529 # Mock out UploadSymbolFile and fail for fail.sym files.
Mike Nichols90f7c152019-04-09 15:14:08 -0600530 def failSome(_url, symbol, _api_key):
Don Garrettdeb2e032016-07-06 16:44:14 -0700531 if symbol.file_name == fail_file:
532 raise urllib2.URLError('network failure')
533
Luigi Semenzato5104f3c2016-10-12 12:37:42 -0700534 upload_mock = self.PatchObject(upload_symbols, 'UploadSymbolFile',
535 side_effect=failSome)
536 upload_mock.__name__ = 'UploadSymbolFileMock2'
Don Garrettdeb2e032016-07-06 16:44:14 -0700537
538 result = upload_symbols.PerformSymbolsFileUpload(
Mike Nichols90f7c152019-04-09 15:14:08 -0600539 symbols, 'fake_url', api_key='testkey')
Don Garrettdeb2e032016-07-06 16:44:14 -0700540
541 self.assertEqual(list(result), symbols)
542
543 passed = sum(s.status == upload_symbols.SymbolFile.UPLOADED
544 for s in symbols)
545 failed = sum(s.status == upload_symbols.SymbolFile.ERROR
546 for s in symbols)
547 skipped = sum(s.status == upload_symbols.SymbolFile.INITIAL
548 for s in symbols)
549
550 # Shows we all pass.sym files worked until limit hit.
551 self.assertEqual(passed, upload_symbols.MAX_TOTAL_ERRORS_FOR_RETRY)
552
553 # Shows we all fail.sym files failed until limit hit.
554 self.assertEqual(failed, upload_symbols.MAX_TOTAL_ERRORS_FOR_RETRY)
555
556 # Shows both pass/fail were skipped after limit hit.
557 self.assertEqual(skipped, 10 * 2)
Don Garretta28be6d2016-06-16 18:09:35 -0700558
559
560class UploadSymbolsTest(SymbolsTestBase):
561 """Test UploadSymbols, along with most helper methods."""
562 def setUp(self):
563 # Results gathering.
564 self.failure_file = os.path.join(self.tempdir, 'failures.txt')
565
566 def testUploadSymbolsEmpty(self):
567 """Upload dir is empty."""
Mike Nichols137e82d2019-05-15 18:40:34 -0600568 result = upload_symbols.UploadSymbols([self.data], 'fake_url')
Don Garretta28be6d2016-06-16 18:09:35 -0700569
570 self.assertEquals(result, 0)
571 self.assertEqual(self.urlopen_mock.call_count, 0)
572
573 def testUploadSymbols(self):
574 """Upload a few files."""
575 self.createSymbolFile('slim.sym', self.SLIM_CONTENT)
576 self.createSymbolFile(os.path.join('nested', 'inner.sym'))
577 self.createSymbolFile('fat.sym', self.FAT_CONTENT)
578
579 result = upload_symbols.UploadSymbols(
Mike Nichols90f7c152019-04-09 15:14:08 -0600580 [self.data], 'fake_url',
581 failed_list=self.failure_file, strip_cfi=len(self.SLIM_CONTENT)+1,
582 api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700583
584 self.assertEquals(result, 0)
Mike Nichols137e82d2019-05-15 18:40:34 -0600585 self.assertEqual(self.request_mock.call_count, 10)
Don Garretta28be6d2016-06-16 18:09:35 -0700586 self.assertEquals(osutils.ReadFile(self.failure_file), '')
587
588 def testUploadSymbolsLimited(self):
589 """Upload a few files."""
590 self.createSymbolFile('slim.sym', self.SLIM_CONTENT)
591 self.createSymbolFile(os.path.join('nested', 'inner.sym'))
592 self.createSymbolFile('fat.sym', self.FAT_CONTENT)
593
594 result = upload_symbols.UploadSymbols(
Mike Nichols90f7c152019-04-09 15:14:08 -0600595 [self.data], 'fake_url', upload_limit=2,
596 api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700597
598 self.assertEquals(result, 0)
Mike Nichols137e82d2019-05-15 18:40:34 -0600599 self.assertEqual(self.request_mock.call_count, 7)
Mike Frysingerf2fa7d62017-12-14 18:33:59 -0500600 self.assertNotExists(self.failure_file)
Don Garretta28be6d2016-06-16 18:09:35 -0700601
602 def testUploadSymbolsFailures(self):
603 """Upload a few files."""
604 self.createSymbolFile('pass.sym')
605 fail = self.createSymbolFile('fail.sym')
606
Mike Nichols90f7c152019-04-09 15:14:08 -0600607 def failSome(_url, symbol, _api_key):
Don Garretta28be6d2016-06-16 18:09:35 -0700608 if symbol.file_name == fail.file_name:
609 raise urllib2.URLError('network failure')
610
611 # Mock out UploadSymbolFile so it's easy to see which file to fail for.
612 upload_mock = self.PatchObject(upload_symbols, 'UploadSymbolFile',
613 side_effect=failSome)
Luigi Semenzato5104f3c2016-10-12 12:37:42 -0700614 # Mock __name__ for logging.
615 upload_mock.__name__ = 'UploadSymbolFileMock'
Don Garretta28be6d2016-06-16 18:09:35 -0700616
617 result = upload_symbols.UploadSymbols(
Mike Nichols90f7c152019-04-09 15:14:08 -0600618 [self.data], 'fake_url',
619 failed_list=self.failure_file, api_key='testkey')
Don Garretta28be6d2016-06-16 18:09:35 -0700620
621 self.assertEquals(result, 1)
622 self.assertEqual(upload_mock.call_count, 8)
623 self.assertEquals(osutils.ReadFile(self.failure_file), 'fail.sym\n')
624
Don Garretta28be6d2016-06-16 18:09:35 -0700625# TODO: We removed --network integration tests.
Mike Frysinger0c0efa22014-02-09 23:32:23 -0500626
627
Mike Frysingerea838d12014-12-08 11:55:32 -0500628def main(_argv):
Mike Frysinger27e21b72018-07-12 14:20:21 -0400629 # pylint: disable=protected-access
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400630 # Set timeouts small so that if the unit test hangs, it won't hang for long.
631 parallel._BackgroundTask.STARTUP_TIMEOUT = 5
632 parallel._BackgroundTask.EXIT_TIMEOUT = 5
633
Mike Frysingerd5fcb3a2013-05-30 21:10:50 -0400634 # Run the tests.
Mike Frysingerba167372015-01-21 10:37:03 -0500635 cros_test_lib.main(level='info', module=__name__)