Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Mike Frysinger | eb753bf | 2013-11-22 16:05:35 -0500 | [diff] [blame] | 5 | """Unittests for upload_symbols.py""" |
| 6 | |
Mike Frysinger | 7f9be14 | 2014-01-15 02:16:42 -0500 | [diff] [blame] | 7 | from __future__ import print_function |
| 8 | |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 9 | import BaseHTTPServer |
Mike Frysinger | 06da520 | 2014-09-26 17:30:33 -0500 | [diff] [blame] | 10 | import errno |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 11 | import itertools |
Mike Frysinger | ea838d1 | 2014-12-08 11:55:32 -0500 | [diff] [blame] | 12 | import mock |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 13 | import os |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 14 | import signal |
Mike Frysinger | 06da520 | 2014-09-26 17:30:33 -0500 | [diff] [blame] | 15 | import socket |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 16 | import SocketServer |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 17 | import sys |
Aviv Keshet | d1f0463 | 2014-05-09 11:33:46 -0700 | [diff] [blame] | 18 | import time |
Mike Frysinger | 094a217 | 2013-08-14 12:54:35 -0400 | [diff] [blame] | 19 | import urllib2 |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 20 | |
Mike Frysinger | 079863c | 2014-10-09 23:16:46 -0400 | [diff] [blame] | 21 | # We specifically set up a local server to connect to, so make sure we |
| 22 | # delete any proxy settings that might screw that up. We also need to |
| 23 | # do it here because modules that are imported below will implicitly |
| 24 | # initialize with this proxy setting rather than dynamically pull it |
| 25 | # on the fly :(. |
| 26 | os.environ.pop('http_proxy', None) |
| 27 | |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 28 | from chromite.lib import cros_logging as logging |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 29 | from chromite.lib import cros_test_lib |
| 30 | from chromite.lib import osutils |
| 31 | from chromite.lib import parallel |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 32 | from chromite.lib import remote_access |
Mike Frysinger | 5e30a4b | 2014-02-12 20:23:04 -0500 | [diff] [blame] | 33 | from chromite.scripts import cros_generate_breakpad_symbols |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 34 | from chromite.scripts import upload_symbols |
| 35 | |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 36 | import isolateserver |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 37 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 38 | class SymbolsTestBase(cros_test_lib.MockTempDirTestCase): |
| 39 | """Base class for most symbols tests.""" |
| 40 | |
| 41 | SLIM_CONTENT = """ |
| 42 | some junk |
| 43 | """ |
| 44 | |
| 45 | FAT_CONTENT = """ |
| 46 | STACK CFI 1234 |
| 47 | some junk |
| 48 | STACK CFI 1234 |
| 49 | """ |
| 50 | |
| 51 | def setUp(self): |
| 52 | # Make certain we don't use the network. |
| 53 | self.isolate_mock = self.PatchObject(isolateserver, 'get_storage_api') |
| 54 | self.urlopen_mock = self.PatchObject(urllib2, 'urlopen') |
| 55 | |
| 56 | # Make 'uploads' go fast. |
| 57 | self.PatchObject(upload_symbols, 'SLEEP_DELAY', 0) |
| 58 | self.PatchObject(upload_symbols, 'INITIAL_RETRY_DELAY', 0) |
| 59 | |
| 60 | # So our symbol file content doesn't have to be real. |
| 61 | self.PatchObject(cros_generate_breakpad_symbols, 'ReadSymsHeader', |
| 62 | return_value=cros_generate_breakpad_symbols.SymbolHeader( |
| 63 | os='os', cpu='cpu', id='id', name='name')) |
| 64 | |
| 65 | self.working = os.path.join(self.tempdir, 'expand') |
| 66 | osutils.SafeMakedirs(self.working) |
| 67 | |
| 68 | self.data = os.path.join(self.tempdir, 'data') |
| 69 | osutils.SafeMakedirs(self.data) |
| 70 | |
| 71 | def createSymbolFile(self, filename, content=FAT_CONTENT, size=0, |
| 72 | status=None, dedupe=False): |
| 73 | fullname = os.path.join(self.data, filename) |
| 74 | osutils.SafeMakedirs(os.path.dirname(fullname)) |
| 75 | |
| 76 | # If a file size is given, force that to be the minimum file size. Create |
| 77 | # a sparse file so large files are practical. |
| 78 | with open(fullname, 'w+b') as f: |
| 79 | f.truncate(size) |
| 80 | f.seek(0) |
| 81 | f.write(content) |
| 82 | |
| 83 | result = upload_symbols.SymbolFile(display_path=filename, |
| 84 | file_name=fullname) |
| 85 | |
| 86 | if status: |
| 87 | result.status = status |
| 88 | |
| 89 | if dedupe: |
| 90 | result.dedupe_item = upload_symbols.DedupeItem(result) |
| 91 | result.dedupe_push_state = 'push_state' |
| 92 | |
| 93 | return result |
| 94 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 95 | |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 96 | class SymbolServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): |
| 97 | """HTTP handler for symbol POSTs""" |
| 98 | |
| 99 | RESP_CODE = None |
| 100 | RESP_MSG = None |
| 101 | |
| 102 | def do_POST(self): |
| 103 | """Handle a POST request""" |
| 104 | # Drain the data from the client. If we don't, we might write the response |
| 105 | # and close the socket before the client finishes, so they die with EPIPE. |
| 106 | clen = int(self.headers.get('Content-Length', '0')) |
| 107 | self.rfile.read(clen) |
| 108 | |
| 109 | self.send_response(self.RESP_CODE, self.RESP_MSG) |
| 110 | self.end_headers() |
| 111 | |
| 112 | def log_message(self, *args, **kwargs): |
| 113 | """Stub the logger as it writes to stderr""" |
| 114 | pass |
| 115 | |
| 116 | |
| 117 | class SymbolServer(SocketServer.ThreadingTCPServer, BaseHTTPServer.HTTPServer): |
| 118 | """Simple HTTP server that forks each request""" |
| 119 | |
| 120 | |
| 121 | class UploadSymbolsServerTest(cros_test_lib.MockTempDirTestCase): |
| 122 | """Tests for UploadSymbols() and a local HTTP server""" |
| 123 | |
| 124 | SYM_CONTENTS = """MODULE Linux arm 123-456 blkid |
| 125 | PUBLIC 1471 0 main""" |
| 126 | |
| 127 | def SpawnServer(self, RequestHandler): |
| 128 | """Spawn a new http server""" |
Mike Frysinger | 06da520 | 2014-09-26 17:30:33 -0500 | [diff] [blame] | 129 | while True: |
| 130 | try: |
| 131 | port = remote_access.GetUnusedPort() |
| 132 | address = ('', port) |
| 133 | self.httpd = SymbolServer(address, RequestHandler) |
| 134 | break |
| 135 | except socket.error as e: |
| 136 | if e.errno == errno.EADDRINUSE: |
| 137 | continue |
| 138 | raise |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 139 | self.server_url = 'http://localhost:%i/post/path' % port |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 140 | self.httpd_pid = os.fork() |
| 141 | if self.httpd_pid == 0: |
| 142 | self.httpd.serve_forever(poll_interval=0.1) |
| 143 | sys.exit(0) |
| 144 | |
| 145 | def setUp(self): |
| 146 | self.httpd_pid = None |
| 147 | self.httpd = None |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 148 | self.server_url = None |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 149 | self.sym_file = os.path.join(self.tempdir, 'test.sym') |
| 150 | osutils.WriteFile(self.sym_file, self.SYM_CONTENTS) |
| 151 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 152 | # Stop sleeps and retries for these tests. |
| 153 | self.PatchObject(upload_symbols, 'SLEEP_DELAY', 0) |
| 154 | self.PatchObject(upload_symbols, 'INITIAL_RETRY_DELAY', 0) |
| 155 | self.PatchObject(upload_symbols, 'MAX_RETRIES', 0) |
| 156 | |
| 157 | # Just to be certain we don't really contact it. |
| 158 | self.PatchObject(isolateserver, 'get_storage_api') |
| 159 | |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 160 | def tearDown(self): |
| 161 | # Only kill the server if we forked one. |
| 162 | if self.httpd_pid: |
| 163 | os.kill(self.httpd_pid, signal.SIGUSR1) |
| 164 | |
| 165 | def testSuccess(self): |
| 166 | """The server returns success for all uploads""" |
| 167 | class Handler(SymbolServerRequestHandler): |
| 168 | """Always return 200""" |
| 169 | RESP_CODE = 200 |
| 170 | |
| 171 | self.SpawnServer(Handler) |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 172 | ret = upload_symbols.UploadSymbols( |
| 173 | sym_paths=[self.sym_file] * 10, |
| 174 | upload_url=self.server_url, |
| 175 | product_name='upload_symbols_test') |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 176 | self.assertEqual(ret, 0) |
| 177 | |
| 178 | def testError(self): |
| 179 | """The server returns errors for all uploads""" |
| 180 | class Handler(SymbolServerRequestHandler): |
| 181 | """Always return 500""" |
| 182 | RESP_CODE = 500 |
| 183 | RESP_MSG = 'Internal Server Error' |
| 184 | |
| 185 | self.SpawnServer(Handler) |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 186 | ret = upload_symbols.UploadSymbols( |
| 187 | sym_paths=[self.sym_file] * 10, |
| 188 | upload_url=self.server_url, |
| 189 | product_name='upload_symbols_test') |
| 190 | self.assertEqual(ret, 10) |
Mike Frysinger | 0a2fd92 | 2014-09-12 20:23:42 -0700 | [diff] [blame] | 191 | |
| 192 | def testHungServer(self): |
| 193 | """The server chokes, but we recover""" |
| 194 | class Handler(SymbolServerRequestHandler): |
| 195 | """All connections choke forever""" |
| 196 | def do_POST(self): |
| 197 | while True: |
| 198 | time.sleep(1000) |
| 199 | |
| 200 | self.SpawnServer(Handler) |
| 201 | with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m: |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 202 | m.return_value = 0.01 |
Mike Frysinger | 58312e9 | 2014-03-18 04:18:36 -0400 | [diff] [blame] | 203 | ret = upload_symbols.UploadSymbols( |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 204 | sym_paths=[self.sym_file] * 10, |
| 205 | upload_url=self.server_url, |
| 206 | product_name='upload_symbols_test') |
| 207 | self.assertEqual(ret, 10) |
Aviv Keshet | d1f0463 | 2014-05-09 11:33:46 -0700 | [diff] [blame] | 208 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 209 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 210 | class UploadSymbolsHelpersTest(cros_test_lib.TestCase): |
| 211 | """Test assorted helper functions and classes.""" |
| 212 | def testCounter(self): |
| 213 | counter = upload_symbols.Counter() |
| 214 | self.assertEquals(counter.value, 0) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 215 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 216 | counter.increment() |
| 217 | self.assertEquals(counter.value, 1) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 218 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 219 | for _ in xrange(1000): |
| 220 | counter.increment() |
| 221 | self.assertEquals(counter.value, 1001) |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 222 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 223 | def testIsTarball(self): |
| 224 | notTar = [ |
| 225 | '/foo/bar/test.bin', |
| 226 | '/foo/bar/test.tar.bin', |
| 227 | '/foo/bar/test.faketar.gz', |
| 228 | '/foo/bar/test.nottgz', |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 229 | ] |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 230 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 231 | isTar = [ |
| 232 | '/foo/bar/test.tar', |
| 233 | '/foo/bar/test.bin.tar', |
| 234 | '/foo/bar/test.bin.tar.bz2', |
| 235 | '/foo/bar/test.bin.tar.gz', |
| 236 | '/foo/bar/test.bin.tar.xz', |
| 237 | '/foo/bar/test.tbz2', |
| 238 | '/foo/bar/test.tbz', |
| 239 | '/foo/bar/test.tgz', |
| 240 | '/foo/bar/test.txz', |
| 241 | ] |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 242 | |
Don Garrett | a28be6d | 2016-06-16 18:09:35 -0700 | [diff] [blame] | 243 | for p in notTar: |
| 244 | self.assertFalse(upload_symbols.IsTarball(p)) |
| 245 | |
| 246 | for p in isTar: |
| 247 | self.assertTrue(upload_symbols.IsTarball(p)) |
| 248 | |
| 249 | def testBatchGenerator(self): |
| 250 | result = upload_symbols.BatchGenerator([], 2) |
| 251 | self.assertEqual(list(result), []) |
| 252 | |
| 253 | result = upload_symbols.BatchGenerator(xrange(6), 2) |
| 254 | self.assertEqual(list(result), [[0, 1], [2, 3], [4, 5]]) |
| 255 | |
| 256 | result = upload_symbols.BatchGenerator(xrange(7), 2) |
| 257 | self.assertEqual(list(result), [[0, 1], [2, 3], [4, 5], [6]]) |
| 258 | |
| 259 | # Prove that we are streaming the results, not generating them all at once. |
| 260 | result = upload_symbols.BatchGenerator(itertools.repeat(0), 2) |
| 261 | self.assertEqual(result.next(), [0, 0]) |
| 262 | |
| 263 | |
| 264 | class FindSymbolFilesTest(SymbolsTestBase): |
| 265 | """Test FindSymbolFiles.""" |
| 266 | def setUp(self): |
| 267 | self.symfile = self.createSymbolFile('root.sym').file_name |
| 268 | self.innerfile = self.createSymbolFile( |
| 269 | os.path.join('nested', 'inner.sym')).file_name |
| 270 | |
| 271 | # CreateTarball is having issues outside the chroot from open file tests. |
| 272 | # |
| 273 | # self.tarball = os.path.join(self.tempdir, 'syms.tar.gz') |
| 274 | # cros_build_lib.CreateTarball( |
| 275 | # 'syms.tar.gz', self.tempdir, inputs=(self.data)) |
| 276 | |
| 277 | def testEmpty(self): |
| 278 | symbols = list(upload_symbols.FindSymbolFiles( |
| 279 | self.working, [])) |
| 280 | self.assertEqual(symbols, []) |
| 281 | |
| 282 | def testFile(self): |
| 283 | symbols = list(upload_symbols.FindSymbolFiles( |
| 284 | self.working, [self.symfile])) |
| 285 | |
| 286 | self.assertEqual(len(symbols), 1) |
| 287 | sf = symbols[0] |
| 288 | |
| 289 | self.assertEqual(sf.display_name, 'root.sym') |
| 290 | self.assertEqual(sf.display_path, self.symfile) |
| 291 | self.assertEqual(sf.file_name, self.symfile) |
| 292 | self.assertEqual(sf.status, upload_symbols.SymbolFile.INITIAL) |
| 293 | self.assertEqual(sf.FileSize(), len(self.FAT_CONTENT)) |
| 294 | |
| 295 | def testDir(self): |
| 296 | symbols = list(upload_symbols.FindSymbolFiles( |
| 297 | self.working, [self.data])) |
| 298 | |
| 299 | self.assertEqual(len(symbols), 2) |
| 300 | root = symbols[0] |
| 301 | nested = symbols[1] |
| 302 | |
| 303 | self.assertEqual(root.display_name, 'root.sym') |
| 304 | self.assertEqual(root.display_path, 'root.sym') |
| 305 | self.assertEqual(root.file_name, self.symfile) |
| 306 | self.assertEqual(root.status, upload_symbols.SymbolFile.INITIAL) |
| 307 | self.assertEqual(root.FileSize(), len(self.FAT_CONTENT)) |
| 308 | |
| 309 | self.assertEqual(nested.display_name, 'inner.sym') |
| 310 | self.assertEqual(nested.display_path, 'nested/inner.sym') |
| 311 | self.assertEqual(nested.file_name, self.innerfile) |
| 312 | self.assertEqual(nested.status, upload_symbols.SymbolFile.INITIAL) |
| 313 | self.assertEqual(nested.FileSize(), len(self.FAT_CONTENT)) |
| 314 | |
| 315 | |
| 316 | class AdjustSymbolFileSizeTest(SymbolsTestBase): |
| 317 | """Test AdjustSymbolFileSize.""" |
| 318 | def setUp(self): |
| 319 | self.slim = self.createSymbolFile('slim.sym', self.SLIM_CONTENT) |
| 320 | self.fat = self.createSymbolFile('fat.sym', self.FAT_CONTENT) |
| 321 | |
| 322 | self.warn_mock = self.PatchObject(logging, 'PrintBuildbotStepWarnings') |
| 323 | |
| 324 | def _testNotStripped(self, symbol, size=None, content=None): |
| 325 | start_file = symbol.file_name |
| 326 | after = upload_symbols.AdjustSymbolFileSize( |
| 327 | symbol, self.working, size) |
| 328 | self.assertIs(after, symbol) |
| 329 | self.assertEqual(after.file_name, start_file) |
| 330 | if content is not None: |
| 331 | self.assertEqual(osutils.ReadFile(after.file_name), content) |
| 332 | |
| 333 | def _testStripped(self, symbol, size=None, content=None): |
| 334 | after = upload_symbols.AdjustSymbolFileSize( |
| 335 | symbol, self.working, size) |
| 336 | self.assertIs(after, symbol) |
| 337 | self.assertTrue(after.file_name.startswith(self.working)) |
| 338 | if content is not None: |
| 339 | self.assertEqual(osutils.ReadFile(after.file_name), content) |
| 340 | |
| 341 | def testSmall(self): |
| 342 | """Ensure that files smaller than the limit are not modified.""" |
| 343 | self._testNotStripped(self.slim, 1024, self.SLIM_CONTENT) |
| 344 | self._testNotStripped(self.fat, 1024, self.FAT_CONTENT) |
| 345 | |
| 346 | def testLarge(self): |
| 347 | """Ensure that files larger than the limit are modified.""" |
| 348 | self._testStripped(self.slim, 1, self.SLIM_CONTENT) |
| 349 | self._testStripped(self.fat, 1, self.SLIM_CONTENT) |
| 350 | |
| 351 | def testMixed(self): |
| 352 | """Test mix of large and small.""" |
| 353 | strip_size = len(self.SLIM_CONTENT) + 1 |
| 354 | |
| 355 | self._testNotStripped(self.slim, strip_size, self.SLIM_CONTENT) |
| 356 | self._testStripped(self.fat, strip_size, self.SLIM_CONTENT) |
| 357 | |
| 358 | def testSizeWarnings(self): |
| 359 | large = self.createSymbolFile( |
| 360 | 'large.sym', content=self.SLIM_CONTENT, |
| 361 | size=upload_symbols.CRASH_SERVER_FILE_LIMIT*2) |
| 362 | |
| 363 | # Would like to Strip as part of this test, but that really copies all |
| 364 | # of the sparse file content, which is too expensive for a unittest. |
| 365 | self._testNotStripped(large, None, None) |
| 366 | |
| 367 | self.assertEqual(self.warn_mock.call_count, 1) |
| 368 | |
| 369 | |
| 370 | class DeduplicateTest(SymbolsTestBase): |
| 371 | """Test Isolate server Deduplication.""" |
| 372 | def setUp(self): |
| 373 | self.connection_mock = self.isolate_mock.return_value |
| 374 | self.contains_mock = self.connection_mock.contains |
| 375 | self.push_mock = self.connection_mock.push |
| 376 | |
| 377 | def testConnection(self): |
| 378 | result = upload_symbols.OpenDeduplicateConnection('namespace') |
| 379 | self.assertTrue(result is not None) |
| 380 | self.assertEqual( |
| 381 | self.isolate_mock.mock_calls, |
| 382 | [mock.call('https://isolateserver.appspot.com', 'namespace')]) |
| 383 | |
| 384 | def testConnectionFail(self): |
| 385 | self.isolate_mock.side_effect = Exception |
| 386 | result = upload_symbols.OpenDeduplicateConnection('namespace') |
| 387 | self.assertEqual(result, None) |
| 388 | |
| 389 | def testFindDuplicates(self): |
| 390 | # The first symbol will new new, the second a duplicate. |
| 391 | def containsOneNew(items): |
| 392 | return {items[0]: 'state'} |
| 393 | self.contains_mock.side_effect = containsOneNew |
| 394 | |
| 395 | sym1 = self.createSymbolFile('sym1.sym') |
| 396 | sym2 = self.createSymbolFile('sym2.sym') |
| 397 | |
| 398 | result = upload_symbols.FindDuplicates((sym1, sym2), 'namespace') |
| 399 | self.assertEqual(list(result), [sym1, sym2]) |
| 400 | |
| 401 | self.assertEqual(sym1.status, upload_symbols.SymbolFile.INITIAL) |
| 402 | self.assertIsInstance(sym1.dedupe_item, upload_symbols.DedupeItem) |
| 403 | self.assertEqual(sym1.dedupe_push_state, 'state') |
| 404 | |
| 405 | self.assertEqual(sym2.status, upload_symbols.SymbolFile.DUPLICATE) |
| 406 | self.assertIsNone(sym2.dedupe_item) |
| 407 | self.assertIsNone(sym2.dedupe_push_state) |
| 408 | |
| 409 | def testFindDuplicatesConnectFail(self): |
| 410 | self.isolate_mock.side_effect = Exception |
| 411 | |
| 412 | sym1 = self.createSymbolFile('sym1.sym') |
| 413 | sym2 = self.createSymbolFile('sym2.sym') |
| 414 | |
| 415 | result = upload_symbols.FindDuplicates((sym1, sym2), 'namespace') |
| 416 | self.assertEqual(list(result), [sym1, sym2]) |
| 417 | |
| 418 | self.assertFalse(self.contains_mock.called) |
| 419 | |
| 420 | self.assertEqual(sym1.status, upload_symbols.SymbolFile.INITIAL) |
| 421 | self.assertIsNone(sym1.dedupe_item) |
| 422 | self.assertIsNone(sym1.dedupe_push_state) |
| 423 | |
| 424 | self.assertEqual(sym2.status, upload_symbols.SymbolFile.INITIAL) |
| 425 | self.assertIsNone(sym2.dedupe_item) |
| 426 | self.assertIsNone(sym2.dedupe_push_state) |
| 427 | |
| 428 | def testFindDuplicatesContainsFail(self): |
| 429 | self.contains_mock.side_effect = Exception |
| 430 | |
| 431 | sym1 = self.createSymbolFile('sym1.sym') |
| 432 | sym2 = self.createSymbolFile('sym2.sym') |
| 433 | |
| 434 | result = upload_symbols.FindDuplicates((sym1, sym2), 'namespace') |
| 435 | self.assertEqual(list(result), [sym1, sym2]) |
| 436 | |
| 437 | self.assertEqual(sym1.status, upload_symbols.SymbolFile.INITIAL) |
| 438 | self.assertIsNone(sym1.dedupe_item) |
| 439 | self.assertIsNone(sym1.dedupe_push_state) |
| 440 | |
| 441 | self.assertEqual(sym2.status, upload_symbols.SymbolFile.INITIAL) |
| 442 | self.assertIsNone(sym2.dedupe_item) |
| 443 | self.assertIsNone(sym2.dedupe_push_state) |
| 444 | |
| 445 | def testPostForDeduplication(self): |
| 446 | symInitial = self.createSymbolFile( |
| 447 | 'initial.sym') |
| 448 | symDup = self.createSymbolFile( |
| 449 | 'dup.sym', status=upload_symbols.SymbolFile.DUPLICATE) |
| 450 | symUploaded = self.createSymbolFile( |
| 451 | 'uploaded.sym', status=upload_symbols.SymbolFile.UPLOADED) |
| 452 | |
| 453 | symInitialItem = self.createSymbolFile( |
| 454 | 'initialItem.sym', dedupe=True) |
| 455 | |
| 456 | symDupItem = self.createSymbolFile( |
| 457 | 'dupItem.sym', status=upload_symbols.SymbolFile.DUPLICATE, dedupe=True) |
| 458 | # This is the only symbol file which should be pushed to isolate. |
| 459 | symUploadedItem = self.createSymbolFile( |
| 460 | 'upItem.sym', status=upload_symbols.SymbolFile.UPLOADED, dedupe=True) |
| 461 | |
| 462 | symbols = (symInitial, symDup, symUploaded, |
| 463 | symInitialItem, symDupItem, symUploadedItem) |
| 464 | |
| 465 | result = upload_symbols.PostForDeduplication(symbols, 'namespace') |
| 466 | self.assertEqual(list(result), list(symbols)) |
| 467 | self.assertEqual( |
| 468 | self.push_mock.mock_calls, |
| 469 | [ |
| 470 | mock.call( |
| 471 | symUploadedItem.dedupe_item, 'push_state', |
| 472 | ["SymbolHeader(cpu='cpu', id='id', name='name', os='os')"]) |
| 473 | ]) |
| 474 | |
| 475 | def testPostForDeduplicationConnectFail(self): |
| 476 | self.isolate_mock.side_effect = Exception |
| 477 | |
| 478 | sym1 = self.createSymbolFile( |
| 479 | 'sym1.sym', status=upload_symbols.SymbolFile.UPLOADED, dedupe=True) |
| 480 | sym2 = self.createSymbolFile( |
| 481 | 'sym2.sym', status=upload_symbols.SymbolFile.UPLOADED, dedupe=True) |
| 482 | |
| 483 | result = upload_symbols.PostForDeduplication((sym1, sym2), 'namespace') |
| 484 | self.assertEqual(list(result), [sym1, sym2]) |
| 485 | self.assertEqual(self.push_mock.call_count, 0) |
| 486 | |
| 487 | def testPostForDeduplicationPushFail(self): |
| 488 | self.push_mock.side_effect = Exception |
| 489 | |
| 490 | sym1 = self.createSymbolFile( |
| 491 | 'sym1.sym', status=upload_symbols.SymbolFile.UPLOADED, dedupe=True) |
| 492 | sym2 = self.createSymbolFile( |
| 493 | 'sym2.sym', status=upload_symbols.SymbolFile.UPLOADED, dedupe=True) |
| 494 | |
| 495 | result = upload_symbols.PostForDeduplication((sym1, sym2), 'namespace') |
| 496 | self.assertEqual(list(result), [sym1, sym2]) |
| 497 | self.assertEqual(self.push_mock.call_count, 1) |
| 498 | |
| 499 | |
| 500 | class PerformSymbolFileUploadTest(SymbolsTestBase): |
| 501 | """Test PerformSymbolFile, and it's helper methods.""" |
| 502 | def setUp(self): |
| 503 | self.symfile = self.createSymbolFile('symfile.sym') |
| 504 | self.failures = upload_symbols.Counter() |
| 505 | |
| 506 | def testGetUploadTimeout(self): |
| 507 | """Test GetUploadTimeout helper function.""" |
| 508 | # Timeout for small file. |
| 509 | self.assertEqual(upload_symbols.GetUploadTimeout(self.symfile), |
| 510 | upload_symbols.UPLOAD_MIN_TIMEOUT) |
| 511 | |
| 512 | # Timeout for 300M file. |
| 513 | large = self.createSymbolFile('large.sym', size=(300 * 1024 * 1024)) |
| 514 | self.assertEqual(upload_symbols.GetUploadTimeout(large), 1542) |
| 515 | |
| 516 | def testUploadSymbolFile(self): |
| 517 | upload_symbols.UploadSymbolFile('fake_url', self.symfile, 'product') |
| 518 | # TODO: Examine mock in more detail to make sure request is correct. |
| 519 | self.assertEqual(self.urlopen_mock.call_count, 1) |
| 520 | |
| 521 | def testPerformSymbolFileUpload(self): |
| 522 | """We upload on first try.""" |
| 523 | result = upload_symbols.PerformSymbolFileUpload( |
| 524 | self.symfile, 'fake_url', self.failures, product_name='product') |
| 525 | |
| 526 | self.assertIs(result, self.symfile) |
| 527 | self.assertEqual(result.status, upload_symbols.SymbolFile.UPLOADED) |
| 528 | self.assertEqual(self.urlopen_mock.call_count, 1) |
| 529 | self.assertEqual(self.failures.value, 0) |
| 530 | |
| 531 | def testPerformSymbolFileUploadFailure(self): |
| 532 | """All network requests fail.""" |
| 533 | self.urlopen_mock.side_effect = urllib2.URLError('network failure') |
| 534 | |
| 535 | result = upload_symbols.PerformSymbolFileUpload( |
| 536 | self.symfile, 'fake_url', self.failures, product_name='product') |
| 537 | |
| 538 | self.assertIs(result, self.symfile) |
| 539 | self.assertEqual(result.status, upload_symbols.SymbolFile.INITIAL) |
| 540 | self.assertEqual(self.urlopen_mock.call_count, 7) |
| 541 | self.assertEqual(self.failures.value, 1) |
| 542 | |
| 543 | def testPerformSymbolFileUploadTransientFailure(self): |
| 544 | """We fail once, then succeed.""" |
| 545 | self.urlopen_mock.side_effect = (urllib2.URLError('network failure'), None) |
| 546 | |
| 547 | result = upload_symbols.PerformSymbolFileUpload( |
| 548 | self.symfile, 'fake_url', self.failures, product_name='product') |
| 549 | |
| 550 | self.assertIs(result, self.symfile) |
| 551 | self.assertEqual(result.status, upload_symbols.SymbolFile.UPLOADED) |
| 552 | self.assertEqual(self.urlopen_mock.call_count, 2) |
| 553 | self.assertEqual(self.failures.value, 0) |
| 554 | |
| 555 | |
| 556 | class UploadSymbolsTest(SymbolsTestBase): |
| 557 | """Test UploadSymbols, along with most helper methods.""" |
| 558 | def setUp(self): |
| 559 | # Results gathering. |
| 560 | self.failure_file = os.path.join(self.tempdir, 'failures.txt') |
| 561 | |
| 562 | def testUploadSymbolsEmpty(self): |
| 563 | """Upload dir is empty.""" |
| 564 | result = upload_symbols.UploadSymbols( |
| 565 | [self.data], 'fake_url', 'product') |
| 566 | |
| 567 | self.assertEquals(result, 0) |
| 568 | self.assertEqual(self.urlopen_mock.call_count, 0) |
| 569 | |
| 570 | def testUploadSymbols(self): |
| 571 | """Upload a few files.""" |
| 572 | self.createSymbolFile('slim.sym', self.SLIM_CONTENT) |
| 573 | self.createSymbolFile(os.path.join('nested', 'inner.sym')) |
| 574 | self.createSymbolFile('fat.sym', self.FAT_CONTENT) |
| 575 | |
| 576 | result = upload_symbols.UploadSymbols( |
| 577 | [self.data], 'fake_url', 'product', |
| 578 | failed_list=self.failure_file, strip_cfi=len(self.SLIM_CONTENT)+1) |
| 579 | |
| 580 | self.assertEquals(result, 0) |
| 581 | self.assertEqual(self.urlopen_mock.call_count, 3) |
| 582 | self.assertEquals(osutils.ReadFile(self.failure_file), '') |
| 583 | |
| 584 | def testUploadSymbolsLimited(self): |
| 585 | """Upload a few files.""" |
| 586 | self.createSymbolFile('slim.sym', self.SLIM_CONTENT) |
| 587 | self.createSymbolFile(os.path.join('nested', 'inner.sym')) |
| 588 | self.createSymbolFile('fat.sym', self.FAT_CONTENT) |
| 589 | |
| 590 | result = upload_symbols.UploadSymbols( |
| 591 | [self.data], 'fake_url', 'product', upload_limit=2) |
| 592 | |
| 593 | self.assertEquals(result, 0) |
| 594 | self.assertEqual(self.urlopen_mock.call_count, 2) |
| 595 | self.assertFalse(os.path.exists(self.failure_file)) |
| 596 | |
| 597 | def testUploadSymbolsFailures(self): |
| 598 | """Upload a few files.""" |
| 599 | self.createSymbolFile('pass.sym') |
| 600 | fail = self.createSymbolFile('fail.sym') |
| 601 | |
| 602 | def failSome(_url, symbol, _product): |
| 603 | if symbol.file_name == fail.file_name: |
| 604 | raise urllib2.URLError('network failure') |
| 605 | |
| 606 | # Mock out UploadSymbolFile so it's easy to see which file to fail for. |
| 607 | upload_mock = self.PatchObject(upload_symbols, 'UploadSymbolFile', |
| 608 | side_effect=failSome) |
| 609 | |
| 610 | result = upload_symbols.UploadSymbols( |
| 611 | [self.data], 'fake_url', 'product', |
| 612 | failed_list=self.failure_file) |
| 613 | |
| 614 | self.assertEquals(result, 1) |
| 615 | self.assertEqual(upload_mock.call_count, 8) |
| 616 | self.assertEquals(osutils.ReadFile(self.failure_file), 'fail.sym\n') |
| 617 | |
| 618 | |
| 619 | # TODO: We removed --network integration tests. |
Mike Frysinger | 0c0efa2 | 2014-02-09 23:32:23 -0500 | [diff] [blame] | 620 | |
| 621 | |
Mike Frysinger | ea838d1 | 2014-12-08 11:55:32 -0500 | [diff] [blame] | 622 | def main(_argv): |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 623 | # pylint: disable=W0212 |
| 624 | # Set timeouts small so that if the unit test hangs, it won't hang for long. |
| 625 | parallel._BackgroundTask.STARTUP_TIMEOUT = 5 |
| 626 | parallel._BackgroundTask.EXIT_TIMEOUT = 5 |
| 627 | |
Mike Frysinger | d5fcb3a | 2013-05-30 21:10:50 -0400 | [diff] [blame] | 628 | # Run the tests. |
Mike Frysinger | ba16737 | 2015-01-21 10:37:03 -0500 | [diff] [blame] | 629 | cros_test_lib.main(level='info', module=__name__) |