blob: f2305797c5296d8915bbf1a5ce03e292fe717b24 [file] [log] [blame]
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00006"""Reads a .isolated, creates a tree of hardlinks and runs the test.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00007
8Keeps a local cache.
9"""
10
vadimsh@chromium.org87d63262013-04-04 19:34:21 +000011import cookielib
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000012import ctypes
vadimsh@chromium.org80f73002013-07-12 14:52:44 +000013import functools
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000014import hashlib
csharp@chromium.orga110d792013-01-07 16:16:16 +000015import httplib
maruel@chromium.org2b2139a2013-04-30 20:14:58 +000016import itertools
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000017import json
csharp@chromium.orgbfb98742013-03-26 20:28:36 +000018import locale
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000019import logging
csharp@chromium.orgff2a4662012-11-21 20:49:32 +000020import logging.handlers
csharp@chromium.orgf13eec02013-03-11 18:22:56 +000021import math
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000022import optparse
23import os
24import Queue
csharp@chromium.orgf13eec02013-03-11 18:22:56 +000025import random
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000026import re
27import shutil
vadimsh@chromium.org80f73002013-07-12 14:52:44 +000028import socket
29import ssl
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000030import stat
31import subprocess
32import sys
33import tempfile
34import threading
35import time
36import urllib
csharp@chromium.orga92403f2012-11-20 15:13:59 +000037import urllib2
csharp@chromium.orgf13eec02013-03-11 18:22:56 +000038import urlparse
csharp@chromium.orga92403f2012-11-20 15:13:59 +000039import zlib
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000040
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +000041from utils import lru
vadimsh@chromium.orgb074b162013-08-22 17:55:46 +000042from utils import threading_utils
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +000043from utils import zip_package
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000044
vadimsh@chromium.org87d63262013-04-04 19:34:21 +000045# Try to import 'upload' module used by AppEngineService for authentication.
46# If it is not there, app engine authentication support will be disabled.
47try:
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +000048 from third_party.rietveld import upload
vadimsh@chromium.org87d63262013-04-04 19:34:21 +000049 # Hack out upload logging.info()
50 upload.logging = logging.getLogger('upload')
51 # Mac pylint choke on this line.
52 upload.logging.setLevel(logging.WARNING) # pylint: disable=E1103
53except ImportError:
54 upload = None
55
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000056
vadimsh@chromium.org85071062013-08-21 23:37:45 +000057# Absolute path to this file (can be None if running from zip on Mac).
58THIS_FILE_PATH = os.path.abspath(__file__) if __file__ else None
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000059
60# Directory that contains this file (might be inside zip package).
vadimsh@chromium.org85071062013-08-21 23:37:45 +000061BASE_DIR = os.path.dirname(THIS_FILE_PATH) if __file__ else None
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000062
63# Directory that contains currently running script file.
64MAIN_DIR = os.path.dirname(os.path.abspath(zip_package.get_main_script_path()))
65
maruel@chromium.org6b365dc2012-10-18 19:17:56 +000066# Types of action accepted by link_file().
maruel@chromium.orgba6489b2013-07-11 20:23:33 +000067HARDLINK, HARDLINK_WITH_FALLBACK, SYMLINK, COPY = range(1, 5)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000068
69RE_IS_SHA1 = re.compile(r'^[a-fA-F0-9]{40}$')
70
csharp@chromium.org8dc52542012-11-08 20:29:55 +000071# The file size to be used when we don't know the correct file size,
72# generally used for .isolated files.
73UNKNOWN_FILE_SIZE = None
74
csharp@chromium.orga92403f2012-11-20 15:13:59 +000075# The size of each chunk to read when downloading and unzipping files.
76ZIPPED_FILE_CHUNK = 16 * 1024
77
csharp@chromium.orgff2a4662012-11-21 20:49:32 +000078# The name of the log file to use.
79RUN_ISOLATED_LOG_FILE = 'run_isolated.log'
80
csharp@chromium.orge217f302012-11-22 16:51:53 +000081# The name of the log to use for the run_test_cases.py command
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000082RUN_TEST_CASES_LOG = 'run_test_cases.log'
csharp@chromium.orge217f302012-11-22 16:51:53 +000083
csharp@chromium.org9c59ff12012-12-12 02:32:29 +000084# The delay (in seconds) to wait between logging statements when retrieving
85# the required files. This is intended to let the user (or buildbot) know that
86# the program is still running.
87DELAY_BETWEEN_UPDATES_IN_SECS = 30
88
vadimsh@chromium.org5db0f4f2013-07-04 13:57:02 +000089# Maximum expected delay (in seconds) between successive file fetches
90# in run_tha_test. If it takes longer than that, a deadlock might be happening
91# and all stack frames for all threads are dumped to log.
92DEADLOCK_TIMEOUT = 5 * 60
93
csharp@chromium.orgf13eec02013-03-11 18:22:56 +000094# The name of the key to store the count of url attempts.
95COUNT_KEY = 'UrlOpenAttempt'
96
maruel@chromium.org2b2139a2013-04-30 20:14:58 +000097# Default maximum number of attempts to trying opening a url before aborting.
98URL_OPEN_MAX_ATTEMPTS = 30
99# Default timeout when retrying.
100URL_OPEN_TIMEOUT = 6*60.
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000101
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000102# Read timeout in seconds for downloads from isolate storage. If there's no
103# response from the server within this timeout whole download will be aborted.
104DOWNLOAD_READ_TIMEOUT = 60
105
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000106# Global (for now) map: server URL (http://example.com) -> HttpService instance.
107# Used by get_http_service to cache HttpService instances.
108_http_services = {}
109_http_services_lock = threading.Lock()
110
maruel@chromium.org9e9ceaa2013-04-05 15:42:42 +0000111# Used by get_flavor().
112FLAVOR_MAPPING = {
113 'cygwin': 'win',
114 'win32': 'win',
115 'darwin': 'mac',
116 'sunos5': 'solaris',
117 'freebsd7': 'freebsd',
118 'freebsd8': 'freebsd',
119}
120
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000121
122class ConfigError(ValueError):
maruel@chromium.org0cd0b182012-10-22 13:34:15 +0000123 """Generic failure to load a .isolated file."""
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000124 pass
125
126
127class MappingError(OSError):
128 """Failed to recreate the tree."""
129 pass
130
131
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000132class TimeoutError(IOError):
133 """Timeout while reading HTTP response."""
134
135 def __init__(self, inner_exc=None):
136 super(TimeoutError, self).__init__(str(inner_exc or 'Timeout'))
137 self.inner_exc = inner_exc
138
139
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +0000140def get_as_zip_package(executable=True):
141 """Returns ZipPackage with this module and all its dependencies.
142
143 If |executable| is True will store run_isolated.py as __main__.py so that
144 zip package is directly executable be python.
145 """
146 # Building a zip package when running from another zip package is
147 # unsupported and probably unneeded.
148 assert not zip_package.is_zipped_module(sys.modules[__name__])
vadimsh@chromium.org85071062013-08-21 23:37:45 +0000149 assert THIS_FILE_PATH
150 assert BASE_DIR
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +0000151 package = zip_package.ZipPackage(root=BASE_DIR)
152 package.add_python_file(THIS_FILE_PATH, '__main__.py' if executable else None)
153 package.add_directory(os.path.join(BASE_DIR, 'third_party'))
154 package.add_directory(os.path.join(BASE_DIR, 'utils'))
155 return package
156
157
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000158def get_flavor():
159 """Returns the system default flavor. Copied from gyp/pylib/gyp/common.py."""
maruel@chromium.org9e9ceaa2013-04-05 15:42:42 +0000160 return FLAVOR_MAPPING.get(sys.platform, 'linux')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000161
162
csharp@chromium.orgbfb98742013-03-26 20:28:36 +0000163def fix_default_encoding():
164 """Forces utf8 solidly on all platforms.
165
166 By default python execution environment is lazy and defaults to ascii
167 encoding.
168
169 http://uucode.com/blog/2007/03/23/shut-up-you-dummy-7-bit-python/
170 """
171 if sys.getdefaultencoding() == 'utf-8':
172 return False
173
174 # Regenerate setdefaultencoding.
175 reload(sys)
176 # Module 'sys' has no 'setdefaultencoding' member
177 # pylint: disable=E1101
178 sys.setdefaultencoding('utf-8')
179 for attr in dir(locale):
180 if attr[0:3] != 'LC_':
181 continue
182 aref = getattr(locale, attr)
183 try:
184 locale.setlocale(aref, '')
185 except locale.Error:
186 continue
187 try:
188 lang = locale.getlocale(aref)[0]
189 except (TypeError, ValueError):
190 continue
191 if lang:
192 try:
193 locale.setlocale(aref, (lang, 'UTF-8'))
194 except locale.Error:
195 os.environ[attr] = lang + '.UTF-8'
196 try:
197 locale.setlocale(locale.LC_ALL, '')
198 except locale.Error:
199 pass
200 return True
201
202
maruel@chromium.org46e61cc2013-03-25 19:55:34 +0000203class Unbuffered(object):
204 """Disable buffering on a file object."""
205 def __init__(self, stream):
206 self.stream = stream
207
208 def write(self, data):
209 self.stream.write(data)
210 if '\n' in data:
211 self.stream.flush()
212
213 def __getattr__(self, attr):
214 return getattr(self.stream, attr)
215
216
217def disable_buffering():
218 """Makes this process and child processes stdout unbuffered."""
219 if not os.environ.get('PYTHONUNBUFFERED'):
220 # Since sys.stdout is a C++ object, it's impossible to do
221 # sys.stdout.write = lambda...
222 sys.stdout = Unbuffered(sys.stdout)
223 os.environ['PYTHONUNBUFFERED'] = 'x'
224
225
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000226def os_link(source, link_name):
227 """Add support for os.link() on Windows."""
228 if sys.platform == 'win32':
229 if not ctypes.windll.kernel32.CreateHardLinkW(
230 unicode(link_name), unicode(source), 0):
231 raise OSError()
232 else:
233 os.link(source, link_name)
234
235
236def readable_copy(outfile, infile):
237 """Makes a copy of the file that is readable by everyone."""
csharp@chromium.org59d116d2013-07-05 18:04:08 +0000238 shutil.copy2(infile, outfile)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000239 read_enabled_mode = (os.stat(outfile).st_mode | stat.S_IRUSR |
240 stat.S_IRGRP | stat.S_IROTH)
241 os.chmod(outfile, read_enabled_mode)
242
243
244def link_file(outfile, infile, action):
245 """Links a file. The type of link depends on |action|."""
246 logging.debug('Mapping %s to %s' % (infile, outfile))
maruel@chromium.orgba6489b2013-07-11 20:23:33 +0000247 if action not in (HARDLINK, HARDLINK_WITH_FALLBACK, SYMLINK, COPY):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000248 raise ValueError('Unknown mapping action %s' % action)
249 if not os.path.isfile(infile):
250 raise MappingError('%s is missing' % infile)
251 if os.path.isfile(outfile):
252 raise MappingError(
253 '%s already exist; insize:%d; outsize:%d' %
254 (outfile, os.stat(infile).st_size, os.stat(outfile).st_size))
255
256 if action == COPY:
257 readable_copy(outfile, infile)
258 elif action == SYMLINK and sys.platform != 'win32':
259 # On windows, symlink are converted to hardlink and fails over to copy.
maruel@chromium.orgf43e68b2012-10-15 20:23:10 +0000260 os.symlink(infile, outfile) # pylint: disable=E1101
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000261 else:
262 try:
263 os_link(infile, outfile)
maruel@chromium.orgba6489b2013-07-11 20:23:33 +0000264 except OSError as e:
265 if action == HARDLINK:
266 raise MappingError(
267 'Failed to hardlink %s to %s: %s' % (infile, outfile, e))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000268 # Probably a different file system.
maruel@chromium.org9e98e432013-05-31 17:06:51 +0000269 logging.warning(
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000270 'Failed to hardlink, failing back to copy %s to %s' % (
271 infile, outfile))
272 readable_copy(outfile, infile)
273
274
275def _set_write_bit(path, read_only):
276 """Sets or resets the executable bit on a file or directory."""
277 mode = os.lstat(path).st_mode
278 if read_only:
279 mode = mode & 0500
280 else:
281 mode = mode | 0200
282 if hasattr(os, 'lchmod'):
283 os.lchmod(path, mode) # pylint: disable=E1101
284 else:
285 if stat.S_ISLNK(mode):
286 # Skip symlink without lchmod() support.
287 logging.debug('Can\'t change +w bit on symlink %s' % path)
288 return
289
290 # TODO(maruel): Implement proper DACL modification on Windows.
291 os.chmod(path, mode)
292
293
294def make_writable(root, read_only):
295 """Toggle the writable bit on a directory tree."""
csharp@chromium.org837352f2013-01-17 21:17:03 +0000296 assert os.path.isabs(root), root
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000297 for dirpath, dirnames, filenames in os.walk(root, topdown=True):
298 for filename in filenames:
299 _set_write_bit(os.path.join(dirpath, filename), read_only)
300
301 for dirname in dirnames:
302 _set_write_bit(os.path.join(dirpath, dirname), read_only)
303
304
305def rmtree(root):
306 """Wrapper around shutil.rmtree() to retry automatically on Windows."""
307 make_writable(root, False)
308 if sys.platform == 'win32':
309 for i in range(3):
310 try:
311 shutil.rmtree(root)
312 break
313 except WindowsError: # pylint: disable=E0602
314 delay = (i+1)*2
315 print >> sys.stderr, (
316 'The test has subprocess outliving it. Sleep %d seconds.' % delay)
317 time.sleep(delay)
318 else:
319 shutil.rmtree(root)
320
321
322def is_same_filesystem(path1, path2):
323 """Returns True if both paths are on the same filesystem.
324
325 This is required to enable the use of hardlinks.
326 """
327 assert os.path.isabs(path1), path1
328 assert os.path.isabs(path2), path2
329 if sys.platform == 'win32':
330 # If the drive letter mismatches, assume it's a separate partition.
331 # TODO(maruel): It should look at the underlying drive, a drive letter could
332 # be a mount point to a directory on another drive.
333 assert re.match(r'^[a-zA-Z]\:\\.*', path1), path1
334 assert re.match(r'^[a-zA-Z]\:\\.*', path2), path2
335 if path1[0].lower() != path2[0].lower():
336 return False
337 return os.stat(path1).st_dev == os.stat(path2).st_dev
338
339
340def get_free_space(path):
341 """Returns the number of free bytes."""
342 if sys.platform == 'win32':
343 free_bytes = ctypes.c_ulonglong(0)
344 ctypes.windll.kernel32.GetDiskFreeSpaceExW(
345 ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
346 return free_bytes.value
maruel@chromium.orgf43e68b2012-10-15 20:23:10 +0000347 # For OSes other than Windows.
348 f = os.statvfs(path) # pylint: disable=E1101
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000349 return f.f_bfree * f.f_frsize
350
351
352def make_temp_dir(prefix, root_dir):
353 """Returns a temporary directory on the same file system as root_dir."""
354 base_temp_dir = None
355 if not is_same_filesystem(root_dir, tempfile.gettempdir()):
356 base_temp_dir = os.path.dirname(root_dir)
357 return tempfile.mkdtemp(prefix=prefix, dir=base_temp_dir)
358
359
frankf@chromium.org3348ee02013-06-27 14:53:17 +0000360def load_isolated(content, os_flavor=None):
maruel@chromium.org0cd0b182012-10-22 13:34:15 +0000361 """Verifies the .isolated file is valid and loads this object with the json
362 data.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000363 """
364 try:
365 data = json.loads(content)
366 except ValueError:
367 raise ConfigError('Failed to parse: %s...' % content[:100])
368
369 if not isinstance(data, dict):
370 raise ConfigError('Expected dict, got %r' % data)
371
372 for key, value in data.iteritems():
373 if key == 'command':
374 if not isinstance(value, list):
375 raise ConfigError('Expected list, got %r' % value)
maruel@chromium.org89ad2db2012-12-12 14:29:22 +0000376 if not value:
377 raise ConfigError('Expected non-empty command')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000378 for subvalue in value:
379 if not isinstance(subvalue, basestring):
380 raise ConfigError('Expected string, got %r' % subvalue)
381
382 elif key == 'files':
383 if not isinstance(value, dict):
384 raise ConfigError('Expected dict, got %r' % value)
385 for subkey, subvalue in value.iteritems():
386 if not isinstance(subkey, basestring):
387 raise ConfigError('Expected string, got %r' % subkey)
388 if not isinstance(subvalue, dict):
389 raise ConfigError('Expected dict, got %r' % subvalue)
390 for subsubkey, subsubvalue in subvalue.iteritems():
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000391 if subsubkey == 'l':
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000392 if not isinstance(subsubvalue, basestring):
393 raise ConfigError('Expected string, got %r' % subsubvalue)
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000394 elif subsubkey == 'm':
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000395 if not isinstance(subsubvalue, int):
396 raise ConfigError('Expected int, got %r' % subsubvalue)
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000397 elif subsubkey == 'h':
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000398 if not RE_IS_SHA1.match(subsubvalue):
399 raise ConfigError('Expected sha-1, got %r' % subsubvalue)
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000400 elif subsubkey == 's':
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000401 if not isinstance(subsubvalue, int):
402 raise ConfigError('Expected int, got %r' % subsubvalue)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000403 else:
404 raise ConfigError('Unknown subsubkey %s' % subsubkey)
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000405 if bool('h' in subvalue) and bool('l' in subvalue):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000406 raise ConfigError(
maruel@chromium.orge5c17132012-11-21 18:18:46 +0000407 'Did not expect both \'h\' (sha-1) and \'l\' (link), got: %r' %
408 subvalue)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000409
410 elif key == 'includes':
411 if not isinstance(value, list):
412 raise ConfigError('Expected list, got %r' % value)
maruel@chromium.org89ad2db2012-12-12 14:29:22 +0000413 if not value:
414 raise ConfigError('Expected non-empty includes list')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000415 for subvalue in value:
416 if not RE_IS_SHA1.match(subvalue):
417 raise ConfigError('Expected sha-1, got %r' % subvalue)
418
419 elif key == 'read_only':
420 if not isinstance(value, bool):
421 raise ConfigError('Expected bool, got %r' % value)
422
423 elif key == 'relative_cwd':
424 if not isinstance(value, basestring):
425 raise ConfigError('Expected string, got %r' % value)
426
427 elif key == 'os':
frankf@chromium.org3348ee02013-06-27 14:53:17 +0000428 expected_value = os_flavor or get_flavor()
429 if value != expected_value:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000430 raise ConfigError(
431 'Expected \'os\' to be \'%s\' but got \'%s\'' %
frankf@chromium.org3348ee02013-06-27 14:53:17 +0000432 (expected_value, value))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000433
434 else:
435 raise ConfigError('Unknown key %s' % key)
436
437 return data
438
439
440def fix_python_path(cmd):
441 """Returns the fixed command line to call the right python executable."""
442 out = cmd[:]
443 if out[0] == 'python':
444 out[0] = sys.executable
445 elif out[0].endswith('.py'):
446 out.insert(0, sys.executable)
447 return out
448
449
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000450def url_open(url, **kwargs):
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000451 """Attempts to open the given url multiple times.
452
453 |data| can be either:
454 -None for a GET request
455 -str for pre-encoded data
456 -list for data to be encoded
457 -dict for data to be encoded (COUNT_KEY will be added in this case)
458
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000459 Returns HttpResponse object, where the response may be read from, or None
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000460 if it was unable to connect.
461 """
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000462 urlhost, urlpath = split_server_request_url(url)
463 service = get_http_service(urlhost)
464 return service.request(urlpath, **kwargs)
465
466
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000467def url_read(url, **kwargs):
468 """Attempts to open the given url multiple times and read all data from it.
469
470 Accepts same arguments as url_open function.
471
472 Returns all data read or None if it was unable to connect or read the data.
473 """
474 response = url_open(url, **kwargs)
475 if not response:
476 return None
477 try:
478 return response.read()
479 except TimeoutError:
480 return None
481
482
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000483def split_server_request_url(url):
484 """Splits the url into scheme+netloc and path+params+query+fragment."""
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000485 url_parts = list(urlparse.urlparse(url))
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000486 urlhost = '%s://%s' % (url_parts[0], url_parts[1])
487 urlpath = urlparse.urlunparse(['', ''] + url_parts[2:])
488 return urlhost, urlpath
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000489
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000490
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000491def get_http_service(urlhost):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000492 """Returns existing or creates new instance of HttpService that can send
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000493 requests to given base urlhost.
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000494 """
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000495 # Ensure consistency.
496 urlhost = str(urlhost).lower().rstrip('/')
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000497 with _http_services_lock:
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000498 service = _http_services.get(urlhost)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000499 if not service:
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000500 service = AppEngineService(urlhost)
501 _http_services[urlhost] = service
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000502 return service
503
504
505class HttpService(object):
506 """Base class for a class that provides an API to HTTP based service:
507 - Provides 'request' method.
508 - Supports automatic request retries.
509 - Supports persistent cookies.
510 - Thread safe.
511 """
512
513 # File to use to store all auth cookies.
maruel@chromium.orgbf2a02a2013-07-11 13:27:16 +0000514 COOKIE_FILE = os.path.join(os.path.expanduser('~'), '.isolated_cookies')
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000515
516 # CookieJar reused by all services + lock that protects its instantiation.
517 _cookie_jar = None
518 _cookie_jar_lock = threading.Lock()
519
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000520 def __init__(self, urlhost):
521 self.urlhost = urlhost
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000522 self.cookie_jar = self.load_cookie_jar()
523 self.opener = self.create_url_opener()
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000524
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000525 def authenticate(self): # pylint: disable=R0201
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000526 """Called when HTTP server asks client to authenticate.
527 Can be implemented in subclasses.
528 """
529 return False
530
531 @staticmethod
532 def load_cookie_jar():
533 """Returns global CoookieJar object that stores cookies in the file."""
534 with HttpService._cookie_jar_lock:
535 if HttpService._cookie_jar is not None:
536 return HttpService._cookie_jar
maruel@chromium.orgbf2a02a2013-07-11 13:27:16 +0000537 jar = ThreadSafeCookieJar(HttpService.COOKIE_FILE)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000538 jar.load()
539 HttpService._cookie_jar = jar
540 return jar
541
542 @staticmethod
543 def save_cookie_jar():
544 """Called when cookie jar needs to be flushed to disk."""
545 with HttpService._cookie_jar_lock:
546 if HttpService._cookie_jar is not None:
547 HttpService._cookie_jar.save()
548
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000549 def create_url_opener(self): # pylint: disable=R0201
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000550 """Returns OpenerDirector that will be used when sending requests.
551 Can be reimplemented in subclasses."""
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000552 return urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_jar))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000553
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000554 def request(self, urlpath, data=None, content_type=None, **kwargs):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000555 """Attempts to open the given url multiple times.
556
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000557 |urlpath| is relative to the server root, i.e. '/some/request?param=1'.
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000558
559 |data| can be either:
560 -None for a GET request
561 -str for pre-encoded data
562 -list for data to be encoded
563 -dict for data to be encoded (COUNT_KEY will be added in this case)
564
565 Returns a file-like object, where the response may be read from, or None
566 if it was unable to connect.
567 """
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000568 assert urlpath and urlpath[0] == '/'
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000569
570 if isinstance(data, dict) and COUNT_KEY in data:
571 logging.error('%s already existed in the data passed into UlrOpen. It '
572 'would be overwritten. Aborting UrlOpen', COUNT_KEY)
573 return None
574
575 method = 'GET' if data is None else 'POST'
576 assert not ((method != 'POST') and content_type), (
577 'Can\'t use content_type on GET')
578
579 def make_request(extra):
580 """Returns a urllib2.Request instance for this specific retry."""
581 if isinstance(data, str) or data is None:
582 payload = data
583 else:
584 if isinstance(data, dict):
585 payload = data.items()
586 else:
587 payload = data[:]
588 payload.extend(extra.iteritems())
589 payload = urllib.urlencode(payload)
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000590 new_url = urlparse.urljoin(self.urlhost, urlpath[1:])
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000591 if isinstance(data, str) or data is None:
592 # In these cases, add the extra parameter to the query part of the url.
593 url_parts = list(urlparse.urlparse(new_url))
594 # Append the query parameter.
595 if url_parts[4] and extra:
596 url_parts[4] += '&'
597 url_parts[4] += urllib.urlencode(extra)
598 new_url = urlparse.urlunparse(url_parts)
599 request = urllib2.Request(new_url, data=payload)
600 if payload is not None:
601 if content_type:
602 request.add_header('Content-Type', content_type)
603 request.add_header('Content-Length', len(payload))
604 return request
605
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000606 return self._retry_loop(make_request, **kwargs)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000607
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000608 def _retry_loop(
609 self,
610 make_request,
611 max_attempts=URL_OPEN_MAX_ATTEMPTS,
612 retry_404=False,
613 retry_50x=True,
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000614 timeout=URL_OPEN_TIMEOUT,
615 read_timeout=None):
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000616 """Runs internal request-retry loop.
617
618 - Optionally retries HTTP 404 and 50x.
619 - Retries up to |max_attempts| times. If None or 0, there's no limit in the
620 number of retries.
621 - Retries up to |timeout| duration in seconds. If None or 0, there's no
622 limit in the time taken to do retries.
623 - If both |max_attempts| and |timeout| are None or 0, this functions retries
624 indefinitely.
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000625
626 If |read_timeout| is not None will configure underlying socket to
627 raise TimeoutError exception whenever there's no response from the server
628 for more than |read_timeout| seconds. It can happen during any read
629 operation so once you pass non-None |read_timeout| be prepared to handle
630 these exceptions in subsequent reads from the stream.
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000631 """
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000632 authenticated = False
633 last_error = None
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000634 attempt = 0
635 start = self._now()
636 for attempt in itertools.count():
637 if max_attempts and attempt >= max_attempts:
638 # Too many attempts.
639 break
640 if timeout and (self._now() - start) >= timeout:
641 # Retried for too long.
642 break
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000643 extra = {COUNT_KEY: attempt} if attempt else {}
644 request = make_request(extra)
645 try:
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000646 url_response = self._url_open(request, timeout=read_timeout)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000647 logging.debug('url_open(%s) succeeded', request.get_full_url())
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000648 return HttpResponse(url_response, request.get_full_url())
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000649 except urllib2.HTTPError as e:
650 # Unauthorized. Ask to authenticate and then try again.
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000651 if e.code in (401, 403):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000652 # Try to authenticate only once. If it doesn't help, then server does
653 # not support app engine authentication.
vadimsh@chromium.orga1697342013-04-10 22:57:09 +0000654 logging.error(
vadimsh@chromium.orgdde2d732013-04-10 21:12:52 +0000655 'Authentication is required for %s on attempt %d.\n%s',
656 request.get_full_url(), attempt,
657 self._format_exception(e, verbose=True))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000658 if not authenticated and self.authenticate():
659 authenticated = True
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000660 # Do not sleep.
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000661 continue
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000662 # If authentication failed, return.
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000663 logging.error(
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000664 'Unable to authenticate to %s.\n%s',
665 request.get_full_url(), self._format_exception(e, verbose=True))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000666 return None
667
maruel@chromium.orgd58bf5b2013-04-26 17:57:42 +0000668 if ((e.code < 500 and not (retry_404 and e.code == 404)) or
669 (e.code >= 500 and not retry_50x)):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000670 # This HTTPError means we reached the server and there was a problem
671 # with the request, so don't retry.
672 logging.error(
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000673 'Able to connect to %s but an exception was thrown.\n%s',
674 request.get_full_url(), self._format_exception(e, verbose=True))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000675 return None
676
677 # The HTTPError was due to a server error, so retry the attempt.
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000678 logging.warning('Able to connect to %s on attempt %d.\n%s',
679 request.get_full_url(), attempt,
680 self._format_exception(e))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000681 last_error = e
682
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000683 except (urllib2.URLError, httplib.HTTPException,
684 socket.timeout, ssl.SSLError) as e:
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000685 logging.warning('Unable to open url %s on attempt %d.\n%s',
686 request.get_full_url(), attempt,
687 self._format_exception(e))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000688 last_error = e
689
690 # Only sleep if we are going to try again.
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000691 if max_attempts and attempt != max_attempts:
692 remaining = None
693 if timeout:
694 remaining = timeout - (self._now() - start)
695 if remaining <= 0:
696 break
697 self.sleep_before_retry(attempt, remaining)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000698
699 logging.error('Unable to open given url, %s, after %d attempts.\n%s',
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000700 request.get_full_url(), max_attempts,
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000701 self._format_exception(last_error, verbose=True))
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000702 return None
703
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000704 def _url_open(self, request, timeout=None):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000705 """Low level method to execute urllib2.Request's.
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000706
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000707 To be mocked in tests.
708 """
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000709 if timeout is not None:
710 return self.opener.open(request, timeout=timeout)
711 else:
712 # Leave original default value for |timeout|. It's nontrivial.
713 return self.opener.open(request)
maruel@chromium.orgef333122013-03-12 20:36:40 +0000714
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000715 @staticmethod
716 def _now():
717 """To be mocked in tests."""
718 return time.time()
719
720 @staticmethod
721 def calculate_sleep_before_retry(attempt, max_duration):
722 # Maximum sleeping time. We're hammering a cloud-distributed service, it'll
723 # survive.
724 MAX_SLEEP = 10.
725 # random.random() returns [0.0, 1.0). Starts with relatively short waiting
726 # time by starting with 1.5/2+1.5^-1 median offset.
727 duration = (random.random() * 1.5) + math.pow(1.5, (attempt - 1))
728 assert duration > 0.1
729 duration = min(MAX_SLEEP, duration)
730 if max_duration:
731 duration = min(max_duration, duration)
732 return duration
733
734 @classmethod
735 def sleep_before_retry(cls, attempt, max_duration):
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000736 """Sleeps for some amount of time when retrying the request.
maruel@chromium.org2b2139a2013-04-30 20:14:58 +0000737
738 To be mocked in tests.
739 """
740 time.sleep(cls.calculate_sleep_before_retry(attempt, max_duration))
maruel@chromium.orgef333122013-03-12 20:36:40 +0000741
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000742 @staticmethod
743 def _format_exception(exc, verbose=False):
744 """Given an instance of some exception raised by urlopen returns human
745 readable piece of text with detailed information about the error.
746 """
747 out = ['Exception: %s' % (exc,)]
748 if verbose:
749 if isinstance(exc, urllib2.HTTPError):
750 out.append('-' * 10)
751 if exc.hdrs:
752 for header, value in exc.hdrs.items():
753 if not header.startswith('x-'):
754 out.append('%s: %s' % (header.capitalize(), value))
755 out.append('')
756 out.append(exc.read() or '<empty body>')
757 out.append('-' * 10)
758 return '\n'.join(out)
759
maruel@chromium.orgef333122013-03-12 20:36:40 +0000760
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000761class HttpResponse(object):
762 """Response from HttpService."""
763
764 def __init__(self, url_response, url):
765 self._url_response = url_response
766 self._url = url
767 self._read = 0
768
769 @property
770 def content_length(self):
771 """Total length to the response or None if not known in advance."""
772 length = self._url_response.headers.get('Content-Length')
773 return int(length) if length is not None else None
774
775 def read(self, size=None):
776 """Reads up to |size| bytes from the stream and returns them.
777
778 If |size| is None reads all available bytes.
779
780 Raises TimeoutError on read timeout.
781 """
782 try:
783 data = self._url_response.read(size)
784 self._read += len(data)
785 return data
786 except (socket.timeout, ssl.SSLError) as e:
787 logging.error('Timeout while reading from %s, read %d of %s: %s',
788 self._url, self._read, self.content_length, e)
789 raise TimeoutError(e)
790
791
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000792class AppEngineService(HttpService):
793 """This class implements authentication support for
794 an app engine based services.
maruel@chromium.orgef333122013-03-12 20:36:40 +0000795 """
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000796
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000797 # This lock ensures that user won't be confused with multiple concurrent
798 # login prompts.
799 _auth_lock = threading.Lock()
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000800
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000801 def __init__(self, urlhost, email=None, password=None):
802 super(AppEngineService, self).__init__(urlhost)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000803 self.email = email
804 self.password = password
805 self._keyring = None
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000806
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000807 def authenticate(self):
808 """Authenticates in the app engine application.
809 Returns True on success.
810 """
811 if not upload:
vadimsh@chromium.orga1697342013-04-10 22:57:09 +0000812 logging.error('\'upload\' module is missing, '
813 'app engine authentication is disabled.')
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000814 return False
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000815 cookie_jar = self.cookie_jar
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000816 save_cookie_jar = self.save_cookie_jar
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000817 # RPC server that uses AuthenticationSupport's cookie jar.
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000818 class AuthServer(upload.AbstractRpcServer):
819 def _GetOpener(self):
vadimsh@chromium.org2edbe3f2013-04-05 19:44:54 +0000820 # Authentication code needs to know about 302 response.
821 # So make OpenerDirector without HTTPRedirectHandler.
822 opener = urllib2.OpenerDirector()
823 opener.add_handler(urllib2.ProxyHandler())
824 opener.add_handler(urllib2.UnknownHandler())
825 opener.add_handler(urllib2.HTTPHandler())
826 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
827 opener.add_handler(urllib2.HTTPSHandler())
828 opener.add_handler(urllib2.HTTPErrorProcessor())
829 opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar))
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000830 return opener
831 def PerformAuthentication(self):
832 self._Authenticate()
833 save_cookie_jar()
834 return self.authenticated
835 with AppEngineService._auth_lock:
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000836 rpc_server = AuthServer(self.urlhost, self.get_credentials)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000837 return rpc_server.PerformAuthentication()
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000838
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000839 def get_credentials(self):
840 """Called during authentication process to get the credentials.
841 May be called mutliple times if authentication fails.
842 Returns tuple (email, password).
843 """
844 # 'authenticate' calls this only if 'upload' is present.
845 # Ensure other callers (if any) fail non-cryptically if 'upload' is missing.
846 assert upload, '\'upload\' module is required for this to work'
847 if self.email and self.password:
848 return (self.email, self.password)
849 if not self._keyring:
maruel@chromium.org000bb4d2013-04-26 17:53:27 +0000850 self._keyring = upload.KeyringCreds(self.urlhost,
851 self.urlhost,
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000852 self.email)
853 return self._keyring.GetUserCredentials()
854
855
856class ThreadSafeCookieJar(cookielib.MozillaCookieJar):
857 """MozillaCookieJar with thread safe load and save."""
858
859 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
860 """Loads cookies from the file if it exists."""
maruel@chromium.org4e2676d2013-06-06 18:39:48 +0000861 filename = os.path.expanduser(filename or self.filename)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000862 with self._cookies_lock:
863 if os.path.exists(filename):
864 try:
865 cookielib.MozillaCookieJar.load(self, filename,
866 ignore_discard,
867 ignore_expires)
868 logging.debug('Loaded cookies from %s', filename)
869 except (cookielib.LoadError, IOError):
870 pass
871 else:
maruel@chromium.org16452a32013-04-05 00:18:44 +0000872 try:
873 fd = os.open(filename, os.O_CREAT, 0600)
874 os.close(fd)
875 except OSError:
876 logging.error('Failed to create %s', filename)
877 try:
878 os.chmod(filename, 0600)
879 except OSError:
880 logging.error('Failed to fix mode for %s', filename)
vadimsh@chromium.org87d63262013-04-04 19:34:21 +0000881
882 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
883 """Saves cookies to the file, completely overwriting it."""
884 logging.debug('Saving cookies to %s', filename or self.filename)
885 with self._cookies_lock:
maruel@chromium.org16452a32013-04-05 00:18:44 +0000886 try:
887 cookielib.MozillaCookieJar.save(self, filename,
888 ignore_discard,
889 ignore_expires)
890 except OSError:
891 logging.error('Failed to save %s', filename)
csharp@chromium.orgf13eec02013-03-11 18:22:56 +0000892
893
csharp@chromium.org8dc52542012-11-08 20:29:55 +0000894def valid_file(filepath, size):
895 """Determines if the given files appears valid (currently it just checks
896 the file's size)."""
maruel@chromium.org770993b2012-12-11 17:16:48 +0000897 if size == UNKNOWN_FILE_SIZE:
898 return True
899 actual_size = os.stat(filepath).st_size
900 if size != actual_size:
901 logging.warning(
902 'Found invalid item %s; %d != %d',
903 os.path.basename(filepath), actual_size, size)
904 return False
905 return True
csharp@chromium.org8dc52542012-11-08 20:29:55 +0000906
907
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000908class Profiler(object):
909 def __init__(self, name):
910 self.name = name
911 self.start_time = None
912
913 def __enter__(self):
914 self.start_time = time.time()
915 return self
916
917 def __exit__(self, _exc_type, _exec_value, _traceback):
918 time_taken = time.time() - self.start_time
919 logging.info('Profiling: Section %s took %3.3f seconds',
920 self.name, time_taken)
921
922
923class Remote(object):
maruel@chromium.orgfb155e92012-09-28 20:36:54 +0000924 """Priority based worker queue to fetch or upload files from a
925 content-address server. Any function may be given as the fetcher/upload,
926 as long as it takes two inputs (the item contents, and their relative
927 destination).
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000928
929 Supports local file system, CIFS or http remotes.
930
931 When the priority of items is equals, works in strict FIFO mode.
932 """
933 # Initial and maximum number of worker threads.
934 INITIAL_WORKERS = 2
935 MAX_WORKERS = 16
936 # Priorities.
937 LOW, MED, HIGH = (1<<8, 2<<8, 3<<8)
938 INTERNAL_PRIORITY_BITS = (1<<8) - 1
939 RETRIES = 5
940
maruel@chromium.orgfb155e92012-09-28 20:36:54 +0000941 def __init__(self, destination_root):
942 # Function to fetch a remote object or upload to a remote location..
943 self._do_item = self.get_file_handler(destination_root)
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000944 # Contains tuple(priority, obj).
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000945 self._done = Queue.PriorityQueue()
vadimsh@chromium.orgb074b162013-08-22 17:55:46 +0000946 self._pool = threading_utils.ThreadPool(
947 self.INITIAL_WORKERS, self.MAX_WORKERS, 0, 'remote')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000948
maruel@chromium.orgfb155e92012-09-28 20:36:54 +0000949 def join(self):
950 """Blocks until the queue is empty."""
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000951 return self._pool.join()
maruel@chromium.orgfb155e92012-09-28 20:36:54 +0000952
vadimsh@chromium.org53f8d5a2013-06-19 13:03:55 +0000953 def close(self):
954 """Terminates all worker threads."""
955 self._pool.close()
956
csharp@chromium.orgdf2968f2012-11-16 20:25:37 +0000957 def add_item(self, priority, obj, dest, size):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000958 """Retrieves an object from the remote data store.
959
960 The smaller |priority| gets fetched first.
961
962 Thread-safe.
963 """
964 assert (priority & self.INTERNAL_PRIORITY_BITS) == 0
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000965 return self._add_item(priority, obj, dest, size)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000966
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000967 def _add_item(self, priority, obj, dest, size):
968 assert isinstance(obj, basestring), obj
969 assert isinstance(dest, basestring), dest
970 assert size is None or isinstance(size, int), size
971 return self._pool.add_task(
972 priority, self._task_executer, priority, obj, dest, size)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000973
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000974 def get_one_result(self):
975 return self._pool.get_one_result()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000976
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000977 def _task_executer(self, priority, obj, dest, size):
978 """Wraps self._do_item to trap and retry on IOError exceptions."""
979 try:
980 self._do_item(obj, dest)
981 if size and not valid_file(dest, size):
982 download_size = os.stat(dest).st_size
983 os.remove(dest)
984 raise IOError('File incorrect size after download of %s. Got %s and '
985 'expected %s' % (obj, download_size, size))
986 # TODO(maruel): Technically, we'd want to have an output queue to be a
987 # PriorityQueue.
988 return obj
989 except IOError as e:
990 logging.debug('Caught IOError: %s', e)
vadimsh@chromium.org80f73002013-07-12 14:52:44 +0000991 # Remove unfinished download.
992 if os.path.exists(dest):
993 os.remove(dest)
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000994 # Retry a few times, lowering the priority.
995 if (priority & self.INTERNAL_PRIORITY_BITS) < self.RETRIES:
996 self._add_item(priority + 1, obj, dest, size)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000997 return
maruel@chromium.org13eca0b2013-01-22 16:42:21 +0000998 raise
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000999
csharp@chromium.org59c7bcf2012-11-21 21:13:18 +00001000 def get_file_handler(self, file_or_url): # pylint: disable=R0201
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001001 """Returns a object to retrieve objects from a remote."""
1002 if re.match(r'^https?://.+$', file_or_url):
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001003 return functools.partial(self._download_file, file_or_url)
1004 else:
1005 return functools.partial(self._copy_file, file_or_url)
csharp@chromium.orge9c8d942013-03-11 20:48:36 +00001006
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001007 @staticmethod
1008 def _download_file(base_url, item, dest):
1009 # TODO(maruel): Reuse HTTP connections. The stdlib doesn't make this
1010 # easy.
1011 try:
1012 zipped_source = base_url + item
1013 logging.debug('download_file(%s)', zipped_source)
csharp@chromium.orgec477752013-05-24 20:48:48 +00001014
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001015 # Because the app engine DB is only eventually consistent, retry
1016 # 404 errors because the file might just not be visible yet (even
1017 # though it has been uploaded).
1018 connection = url_open(zipped_source, retry_404=True,
1019 read_timeout=DOWNLOAD_READ_TIMEOUT)
1020 if not connection:
1021 raise IOError('Unable to open connection to %s' % zipped_source)
csharp@chromium.orgec477752013-05-24 20:48:48 +00001022
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001023 content_length = connection.content_length
1024 decompressor = zlib.decompressobj()
1025 size = 0
1026 with open(dest, 'wb') as f:
1027 while True:
1028 chunk = connection.read(ZIPPED_FILE_CHUNK)
1029 if not chunk:
1030 break
1031 size += len(chunk)
1032 f.write(decompressor.decompress(chunk))
1033 # Ensure that all the data was properly decompressed.
1034 uncompressed_data = decompressor.flush()
1035 assert not uncompressed_data
1036 except IOError as e:
1037 logging.error('Failed to download %s at %s.\n%s', item, dest, e)
1038 raise
1039 except httplib.HTTPException as e:
1040 msg = 'HTTPException while retrieving %s at %s.\n%s' % (item, dest, e)
1041 logging.error(msg)
1042 raise IOError(msg)
1043 except zlib.error as e:
1044 msg = 'Corrupted zlib for item %s. Processed %d of %s bytes.\n%s' % (
1045 item, size, content_length, e)
1046 logging.error(msg)
csharp@chromium.orge3413b42013-05-24 17:56:56 +00001047
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001048 # Testing seems to show that if a few machines are trying to download
1049 # the same blob, they can cause each other to fail. So if we hit a
1050 # zip error, this is the most likely cause (it only downloads some of
1051 # the data). Randomly sleep for between 5 and 25 seconds to try and
1052 # spread out the downloads.
1053 # TODO(csharp): Switch from blobstorage to cloud storage and see if
1054 # that solves the issue.
1055 sleep_duration = (random.random() * 20) + 5
1056 time.sleep(sleep_duration)
csharp@chromium.orga92403f2012-11-20 15:13:59 +00001057
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001058 raise IOError(msg)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001059
vadimsh@chromium.org80f73002013-07-12 14:52:44 +00001060 @staticmethod
1061 def _copy_file(base_path, item, dest):
1062 source = os.path.join(base_path, item)
1063 if source == dest:
1064 logging.info('Source and destination are the same, no action required')
1065 return
1066 logging.debug('copy_file(%s, %s)', source, dest)
1067 shutil.copy(source, dest)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001068
1069
1070class CachePolicies(object):
1071 def __init__(self, max_cache_size, min_free_space, max_items):
1072 """
1073 Arguments:
1074 - max_cache_size: Trim if the cache gets larger than this value. If 0, the
1075 cache is effectively a leak.
1076 - min_free_space: Trim if disk free space becomes lower than this value. If
1077 0, it unconditionally fill the disk.
1078 - max_items: Maximum number of items to keep in the cache. If 0, do not
1079 enforce a limit.
1080 """
1081 self.max_cache_size = max_cache_size
1082 self.min_free_space = min_free_space
1083 self.max_items = max_items
1084
1085
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001086class NoCache(object):
1087 """This class is intended to be usable everywhere the Cache class is.
1088 Instead of downloading to a cache, all files are downloaded to the target
1089 directory and then moved to where they are needed.
1090 """
1091
1092 def __init__(self, target_directory, remote):
1093 self.target_directory = target_directory
1094 self.remote = remote
1095
1096 def retrieve(self, priority, item, size):
1097 """Get the request file."""
1098 self.remote.add_item(priority, item, self.path(item), size)
maruel@chromium.org13eca0b2013-01-22 16:42:21 +00001099 self.remote.get_one_result()
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001100
1101 def wait_for(self, items):
1102 """Download the first item of the given list if it is missing."""
1103 item = items.iterkeys().next()
1104
1105 if not os.path.exists(self.path(item)):
1106 self.remote.add_item(Remote.MED, item, self.path(item), UNKNOWN_FILE_SIZE)
maruel@chromium.org13eca0b2013-01-22 16:42:21 +00001107 downloaded = self.remote.get_one_result()
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001108 assert downloaded == item
1109
1110 return item
1111
1112 def path(self, item):
1113 return os.path.join(self.target_directory, item)
1114
1115
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001116class Cache(object):
1117 """Stateful LRU cache.
1118
1119 Saves its state as json file.
1120 """
1121 STATE_FILE = 'state.json'
1122
1123 def __init__(self, cache_dir, remote, policies):
1124 """
1125 Arguments:
1126 - cache_dir: Directory where to place the cache.
1127 - remote: Remote where to fetch items from.
1128 - policies: cache retention policies.
1129 """
1130 self.cache_dir = cache_dir
1131 self.remote = remote
1132 self.policies = policies
1133 self.state_file = os.path.join(cache_dir, self.STATE_FILE)
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001134 self.lru = lru.LRUDict()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001135
1136 # Items currently being fetched. Keep it local to reduce lock contention.
1137 self._pending_queue = set()
1138
1139 # Profiling values.
1140 self._added = []
1141 self._removed = []
1142 self._free_disk = 0
1143
maruel@chromium.org770993b2012-12-11 17:16:48 +00001144 with Profiler('Setup'):
1145 if not os.path.isdir(self.cache_dir):
1146 os.makedirs(self.cache_dir)
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001147
1148 # Load state of the cache.
vadimsh@chromium.orga40428e2013-07-04 15:43:14 +00001149 if os.path.isfile(self.state_file):
1150 try:
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001151 self.lru = lru.LRUDict.load(self.state_file)
1152 except ValueError as err:
1153 logging.error('Failed to load cache state: %s' % (err,))
1154 # Don't want to keep broken state file.
1155 os.remove(self.state_file)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001156
maruel@chromium.org770993b2012-12-11 17:16:48 +00001157 # Ensure that all files listed in the state still exist and add new ones.
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001158 previous = self.lru.keys_set()
1159 unknown = []
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001160 for filename in os.listdir(self.cache_dir):
1161 if filename == self.STATE_FILE:
1162 continue
1163 if filename in previous:
1164 previous.remove(filename)
1165 continue
1166 # An untracked file.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001167 if not RE_IS_SHA1.match(filename):
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001168 logging.warning('Removing unknown file %s from cache', filename)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001169 os.remove(self.path(filename))
maruel@chromium.org770993b2012-12-11 17:16:48 +00001170 continue
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001171 # File that's not referenced in 'state.json'.
1172 # TODO(vadimsh): Verify its SHA1 matches file name.
1173 logging.warning('Adding unknown file %s to cache', filename)
1174 unknown.append(filename)
maruel@chromium.org770993b2012-12-11 17:16:48 +00001175
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001176 if unknown:
1177 # Add as oldest files. They will be deleted eventually if not accessed.
1178 self._add_oldest_list(unknown)
1179 logging.warning('Added back %d unknown files', len(unknown))
1180
maruel@chromium.org770993b2012-12-11 17:16:48 +00001181 if previous:
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001182 # Filter out entries that were not found.
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001183 logging.warning('Removed %d lost files', len(previous))
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001184 for filename in previous:
1185 self.lru.pop(filename)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001186 self.trim()
1187
1188 def __enter__(self):
1189 return self
1190
1191 def __exit__(self, _exc_type, _exec_value, _traceback):
1192 with Profiler('CleanupTrimming'):
1193 self.trim()
1194
1195 logging.info(
maruel@chromium.org5fd6f472012-12-11 00:26:08 +00001196 '%5d (%8dkb) added', len(self._added), sum(self._added) / 1024)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001197 logging.info(
maruel@chromium.org5fd6f472012-12-11 00:26:08 +00001198 '%5d (%8dkb) current',
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001199 len(self.lru),
1200 sum(self.lru.itervalues()) / 1024)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001201 logging.info(
maruel@chromium.org5fd6f472012-12-11 00:26:08 +00001202 '%5d (%8dkb) removed', len(self._removed), sum(self._removed) / 1024)
1203 logging.info(' %8dkb free', self._free_disk / 1024)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001204
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001205 def remove_lru_file(self):
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001206 """Removes the last recently used file and returns its size."""
1207 item, size = self.lru.pop_oldest()
1208 self._delete_file(item, size)
1209 return size
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001210
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001211 def trim(self):
1212 """Trims anything we don't know, make sure enough free space exists."""
1213 # Ensure maximum cache size.
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001214 if self.policies.max_cache_size:
1215 total_size = sum(self.lru.itervalues())
1216 while total_size > self.policies.max_cache_size:
1217 total_size -= self.remove_lru_file()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001218
1219 # Ensure maximum number of items in the cache.
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001220 if self.policies.max_items and len(self.lru) > self.policies.max_items:
1221 for _ in xrange(len(self.lru) - self.policies.max_items):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001222 self.remove_lru_file()
1223
1224 # Ensure enough free space.
1225 self._free_disk = get_free_space(self.cache_dir)
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001226 trimmed_due_to_space = False
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001227 while (
1228 self.policies.min_free_space and
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001229 self.lru and
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001230 self._free_disk < self.policies.min_free_space):
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001231 trimmed_due_to_space = True
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001232 self.remove_lru_file()
1233 self._free_disk = get_free_space(self.cache_dir)
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001234 if trimmed_due_to_space:
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001235 total = sum(self.lru.itervalues())
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001236 logging.warning(
1237 'Trimmed due to not enough free disk space: %.1fkb free, %.1fkb '
1238 'cache (%.1f%% of its maximum capacity)',
1239 self._free_disk / 1024.,
1240 total / 1024.,
1241 100. * self.policies.max_cache_size / float(total),
1242 )
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001243 self.save()
1244
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001245 def retrieve(self, priority, item, size):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001246 """Retrieves a file from the remote, if not already cached, and adds it to
1247 the cache.
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001248
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001249 If the file is in the cache, verify that the file is valid (i.e. it is
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001250 the correct size), retrieving it again if it isn't.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001251 """
1252 assert not '/' in item
1253 path = self.path(item)
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001254 found = False
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001255
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001256 if item in self.lru:
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001257 if not valid_file(self.path(item), size):
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001258 self.lru.pop(item)
1259 self._delete_file(item, size)
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001260 else:
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001261 # Was already in cache. Update it's LRU value by putting it at the end.
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001262 self.lru.touch(item)
1263 found = True
csharp@chromium.org8dc52542012-11-08 20:29:55 +00001264
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001265 if not found:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001266 if item in self._pending_queue:
1267 # Already pending. The same object could be referenced multiple times.
1268 return
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001269 # TODO(maruel): It should look at the free disk space, the current cache
1270 # size and the size of the new item on every new item:
1271 # - Trim the cache as more entries are listed when free disk space is low,
1272 # otherwise if the amount of data downloaded during the run > free disk
1273 # space, it'll crash.
1274 # - Make sure there's enough free disk space to fit all dependencies of
1275 # this run! If not, abort early.
csharp@chromium.orgdf2968f2012-11-16 20:25:37 +00001276 self.remote.add_item(priority, item, path, size)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001277 self._pending_queue.add(item)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001278
1279 def add(self, filepath, obj):
1280 """Forcibly adds a file to the cache."""
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001281 if obj not in self.lru:
1282 link_file(self.path(obj), filepath, HARDLINK)
1283 self._add(obj)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001284
1285 def path(self, item):
1286 """Returns the path to one item."""
1287 return os.path.join(self.cache_dir, item)
1288
1289 def save(self):
1290 """Saves the LRU ordering."""
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001291 self.lru.save(self.state_file)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001292
1293 def wait_for(self, items):
1294 """Starts a loop that waits for at least one of |items| to be retrieved.
1295
1296 Returns the first item retrieved.
1297 """
1298 # Flush items already present.
1299 for item in items:
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001300 if item in self.lru:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001301 return item
1302
1303 assert all(i in self._pending_queue for i in items), (
1304 items, self._pending_queue)
1305 # Note that:
1306 # len(self._pending_queue) ==
1307 # ( len(self.remote._workers) - self.remote._ready +
1308 # len(self._remote._queue) + len(self._remote.done))
1309 # There is no lock-free way to verify that.
1310 while self._pending_queue:
maruel@chromium.org13eca0b2013-01-22 16:42:21 +00001311 item = self.remote.get_one_result()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001312 self._pending_queue.remove(item)
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001313 self._add(item)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001314 if item in items:
1315 return item
1316
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001317 def _add(self, item):
1318 """Adds an item into LRU cache marking it as a newest one."""
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001319 size = os.stat(self.path(item)).st_size
1320 self._added.append(size)
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001321 self.lru.add(item, size)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001322
vadimsh@chromium.org3e97deb2013-08-24 00:56:44 +00001323 def _add_oldest_list(self, items):
1324 """Adds a bunch of items into LRU cache marking them as oldest ones."""
1325 pairs = []
1326 for item in items:
1327 size = os.stat(self.path(item)).st_size
1328 self._added.append(size)
1329 pairs.append((item, size))
1330 self.lru.batch_insert_oldest(pairs)
1331
1332 def _delete_file(self, item, size):
1333 """Deletes cache file from the file system."""
1334 self._removed.append(size)
1335 try:
1336 os.remove(self.path(item))
1337 except OSError as e:
1338 logging.error('Error attempting to delete a file\n%s' % e)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001339
1340
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001341class IsolatedFile(object):
1342 """Represents a single parsed .isolated file."""
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001343 def __init__(self, obj_hash):
1344 """|obj_hash| is really the sha-1 of the file."""
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001345 logging.debug('IsolatedFile(%s)' % obj_hash)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001346 self.obj_hash = obj_hash
1347 # Set once all the left-side of the tree is parsed. 'Tree' here means the
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001348 # .isolate and all the .isolated files recursively included by it with
1349 # 'includes' key. The order of each sha-1 in 'includes', each representing a
1350 # .isolated file in the hash table, is important, as the later ones are not
1351 # processed until the firsts are retrieved and read.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001352 self.can_fetch = False
1353
1354 # Raw data.
1355 self.data = {}
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001356 # A IsolatedFile instance, one per object in self.includes.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001357 self.children = []
1358
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001359 # Set once the .isolated file is loaded.
1360 self._is_parsed = False
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001361 # Set once the files are fetched.
1362 self.files_fetched = False
1363
1364 def load(self, content):
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001365 """Verifies the .isolated file is valid and loads this object with the json
1366 data.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001367 """
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001368 logging.debug('IsolatedFile.load(%s)' % self.obj_hash)
1369 assert not self._is_parsed
1370 self.data = load_isolated(content)
1371 self.children = [IsolatedFile(i) for i in self.data.get('includes', [])]
1372 self._is_parsed = True
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001373
1374 def fetch_files(self, cache, files):
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001375 """Adds files in this .isolated file not present in |files| dictionary.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001376
1377 Preemptively request files.
1378
1379 Note that |files| is modified by this function.
1380 """
1381 assert self.can_fetch
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001382 if not self._is_parsed or self.files_fetched:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001383 return
1384 logging.debug('fetch_files(%s)' % self.obj_hash)
1385 for filepath, properties in self.data.get('files', {}).iteritems():
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001386 # Root isolated has priority on the files being mapped. In particular,
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001387 # overriden files must not be fetched.
1388 if filepath not in files:
1389 files[filepath] = properties
maruel@chromium.orge5c17132012-11-21 18:18:46 +00001390 if 'h' in properties:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001391 # Preemptively request files.
1392 logging.debug('fetching %s' % filepath)
maruel@chromium.orge5c17132012-11-21 18:18:46 +00001393 cache.retrieve(Remote.MED, properties['h'], properties['s'])
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001394 self.files_fetched = True
1395
1396
1397class Settings(object):
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001398 """Results of a completely parsed .isolated file."""
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001399 def __init__(self):
1400 self.command = []
1401 self.files = {}
1402 self.read_only = None
1403 self.relative_cwd = None
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001404 # The main .isolated file, a IsolatedFile instance.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001405 self.root = None
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001406
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001407 def load(self, cache, root_isolated_hash):
1408 """Loads the .isolated and all the included .isolated asynchronously.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001409
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001410 It enables support for "included" .isolated files. They are processed in
1411 strict order but fetched asynchronously from the cache. This is important so
1412 that a file in an included .isolated file that is overridden by an embedding
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001413 .isolated file is not fetched needlessly. The includes are fetched in one
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001414 pass and the files are fetched as soon as all the ones on the left-side
1415 of the tree were fetched.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001416
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001417 The prioritization is very important here for nested .isolated files.
1418 'includes' have the highest priority and the algorithm is optimized for both
1419 deep and wide trees. A deep one is a long link of .isolated files referenced
1420 one at a time by one item in 'includes'. A wide one has a large number of
1421 'includes' in a single .isolated file. 'left' is defined as an included
1422 .isolated file earlier in the 'includes' list. So the order of the elements
1423 in 'includes' is important.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001424 """
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001425 self.root = IsolatedFile(root_isolated_hash)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001426
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001427 # Isolated files being retrieved now: hash -> IsolatedFile instance.
1428 pending = {}
1429 # Set of hashes of already retrieved items to refuse recursive includes.
1430 seen = set()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001431
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001432 def retrieve(isolated_file):
1433 h = isolated_file.obj_hash
1434 if h in seen:
1435 raise ConfigError('IsolatedFile %s is retrieved recursively' % h)
1436 assert h not in pending
1437 seen.add(h)
1438 pending[h] = isolated_file
1439 cache.retrieve(Remote.HIGH, h, UNKNOWN_FILE_SIZE)
1440
1441 retrieve(self.root)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001442
1443 while pending:
1444 item_hash = cache.wait_for(pending)
1445 item = pending.pop(item_hash)
1446 item.load(open(cache.path(item_hash), 'r').read())
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001447 if item_hash == root_isolated_hash:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001448 # It's the root item.
1449 item.can_fetch = True
1450
1451 for new_child in item.children:
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001452 retrieve(new_child)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001453
1454 # Traverse the whole tree to see if files can now be fetched.
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001455 self._traverse_tree(cache, self.root)
1456
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001457 def check(n):
1458 return all(check(x) for x in n.children) and n.files_fetched
1459 assert check(self.root)
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001460
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001461 self.relative_cwd = self.relative_cwd or ''
1462 self.read_only = self.read_only or False
1463
vadimsh@chromium.orgf4c063e2013-07-04 14:23:31 +00001464 def _traverse_tree(self, cache, node):
1465 if node.can_fetch:
1466 if not node.files_fetched:
1467 self._update_self(cache, node)
1468 will_break = False
1469 for i in node.children:
1470 if not i.can_fetch:
1471 if will_break:
1472 break
1473 # Automatically mark the first one as fetcheable.
1474 i.can_fetch = True
1475 will_break = True
1476 self._traverse_tree(cache, i)
1477
1478 def _update_self(self, cache, node):
1479 node.fetch_files(cache, self.files)
1480 # Grabs properties.
1481 if not self.command and node.data.get('command'):
1482 self.command = node.data['command']
1483 if self.read_only is None and node.data.get('read_only') is not None:
1484 self.read_only = node.data['read_only']
1485 if (self.relative_cwd is None and
1486 node.data.get('relative_cwd') is not None):
1487 self.relative_cwd = node.data['relative_cwd']
1488
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001489
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001490def create_directories(base_directory, files):
1491 """Creates the directory structure needed by the given list of files."""
1492 logging.debug('create_directories(%s, %d)', base_directory, len(files))
1493 # Creates the tree of directories to create.
1494 directories = set(os.path.dirname(f) for f in files)
1495 for item in list(directories):
1496 while item:
1497 directories.add(item)
1498 item = os.path.dirname(item)
1499 for d in sorted(directories):
1500 if d:
1501 os.mkdir(os.path.join(base_directory, d))
1502
1503
1504def create_links(base_directory, files):
1505 """Creates any links needed by the given set of files."""
1506 for filepath, properties in files:
csharp@chromium.org89eaf082013-03-26 18:56:21 +00001507 if 'l' not in properties:
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001508 continue
maruel@chromium.org3320ee12013-03-28 13:23:31 +00001509 if sys.platform == 'win32':
1510 # TODO(maruel): Create junctions or empty text files similar to what
1511 # cygwin do?
1512 logging.warning('Ignoring symlink %s', filepath)
1513 continue
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001514 outfile = os.path.join(base_directory, filepath)
1515 # symlink doesn't exist on Windows. So the 'link' property should
1516 # never be specified for windows .isolated file.
1517 os.symlink(properties['l'], outfile) # pylint: disable=E1101
1518 if 'm' in properties:
1519 lchmod = getattr(os, 'lchmod', None)
1520 if lchmod:
1521 lchmod(outfile, properties['m'])
1522
1523
1524def setup_commands(base_directory, cwd, cmd):
1525 """Correctly adjusts and then returns the required working directory
1526 and command needed to run the test.
1527 """
1528 assert not os.path.isabs(cwd), 'The cwd must be a relative path, got %s' % cwd
1529 cwd = os.path.join(base_directory, cwd)
1530 if not os.path.isdir(cwd):
1531 os.makedirs(cwd)
1532
1533 # Ensure paths are correctly separated on windows.
1534 cmd[0] = cmd[0].replace('/', os.path.sep)
1535 cmd = fix_python_path(cmd)
1536
1537 return cwd, cmd
1538
1539
1540def generate_remaining_files(files):
1541 """Generates a dictionary of all the remaining files to be downloaded."""
1542 remaining = {}
1543 for filepath, props in files:
1544 if 'h' in props:
1545 remaining.setdefault(props['h'], []).append((filepath, props))
1546
1547 return remaining
1548
1549
1550def download_test_data(isolated_hash, target_directory, remote):
1551 """Downloads the dependencies to the given directory."""
1552 if not os.path.exists(target_directory):
1553 os.makedirs(target_directory)
1554
1555 settings = Settings()
1556 no_cache = NoCache(target_directory, Remote(remote))
1557
1558 # Download all the isolated files.
1559 with Profiler('GetIsolateds') as _prof:
1560 settings.load(no_cache, isolated_hash)
1561
1562 if not settings.command:
1563 print >> sys.stderr, 'No command to run'
1564 return 1
1565
1566 with Profiler('GetRest') as _prof:
1567 create_directories(target_directory, settings.files)
1568 create_links(target_directory, settings.files.iteritems())
1569
1570 cwd, cmd = setup_commands(target_directory, settings.relative_cwd,
1571 settings.command[:])
1572
1573 remaining = generate_remaining_files(settings.files.iteritems())
1574
1575 # Now block on the remaining files to be downloaded and mapped.
1576 logging.info('Retrieving remaining files')
1577 last_update = time.time()
1578 while remaining:
1579 obj = no_cache.wait_for(remaining)
1580 files = remaining.pop(obj)
1581
1582 for i, (filepath, properties) in enumerate(files):
1583 outfile = os.path.join(target_directory, filepath)
1584 logging.info(no_cache.path(obj))
1585
1586 if i + 1 == len(files):
1587 os.rename(no_cache.path(obj), outfile)
1588 else:
1589 shutil.copyfile(no_cache.path(obj), outfile)
1590
maruel@chromium.orgbaa108d2013-03-28 13:24:51 +00001591 if 'm' in properties and not sys.platform == 'win32':
1592 # It's not set on Windows. It could be set only in the case of
1593 # downloading content generated from another OS. Do not crash in that
1594 # case.
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001595 os.chmod(outfile, properties['m'])
1596
1597 if time.time() - last_update > DELAY_BETWEEN_UPDATES_IN_SECS:
csharp@chromium.org5daba352013-07-03 17:29:27 +00001598 msg = '%d files remaining...' % len(remaining)
1599 print msg
1600 logging.info(msg)
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001601 last_update = time.time()
1602
1603 print('.isolated files successfully downloaded and setup in %s' %
1604 target_directory)
1605 print('To run this test please run the command %s from the directory %s' %
1606 (cmd, cwd))
1607
1608 return 0
1609
1610
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001611def run_tha_test(isolated_hash, cache_dir, remote, policies):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001612 """Downloads the dependencies in the cache, hardlinks them into a temporary
1613 directory and runs the executable.
1614 """
1615 settings = Settings()
1616 with Cache(cache_dir, Remote(remote), policies) as cache:
1617 outdir = make_temp_dir('run_tha_test', cache_dir)
1618 try:
1619 # Initiate all the files download.
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001620 with Profiler('GetIsolateds') as _prof:
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001621 # Optionally support local files.
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001622 if not RE_IS_SHA1.match(isolated_hash):
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001623 # Adds it in the cache. While not strictly necessary, this simplifies
1624 # the rest.
maruel@chromium.orgcb3c3d52013-03-14 18:55:30 +00001625 h = hashlib.sha1(open(isolated_hash, 'rb').read()).hexdigest()
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001626 cache.add(isolated_hash, h)
1627 isolated_hash = h
1628 settings.load(cache, isolated_hash)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001629
1630 if not settings.command:
1631 print >> sys.stderr, 'No command to run'
1632 return 1
1633
1634 with Profiler('GetRest') as _prof:
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001635 create_directories(outdir, settings.files)
1636 create_links(outdir, settings.files.iteritems())
1637 remaining = generate_remaining_files(settings.files.iteritems())
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001638
1639 # Do bookkeeping while files are being downloaded in the background.
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001640 cwd, cmd = setup_commands(outdir, settings.relative_cwd,
1641 settings.command[:])
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001642
1643 # Now block on the remaining files to be downloaded and mapped.
csharp@chromium.org9c59ff12012-12-12 02:32:29 +00001644 logging.info('Retrieving remaining files')
1645 last_update = time.time()
vadimsh@chromium.orgb074b162013-08-22 17:55:46 +00001646 with threading_utils.DeadlockDetector(DEADLOCK_TIMEOUT) as detector:
vadimsh@chromium.org5db0f4f2013-07-04 13:57:02 +00001647 while remaining:
1648 detector.ping()
1649 obj = cache.wait_for(remaining)
1650 for filepath, properties in remaining.pop(obj):
1651 outfile = os.path.join(outdir, filepath)
maruel@chromium.orgb7c003d2013-07-24 13:04:30 +00001652 link_file(outfile, cache.path(obj), HARDLINK)
vadimsh@chromium.org5db0f4f2013-07-04 13:57:02 +00001653 if 'm' in properties:
1654 # It's not set on Windows.
1655 os.chmod(outfile, properties['m'])
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001656
vadimsh@chromium.org5db0f4f2013-07-04 13:57:02 +00001657 if time.time() - last_update > DELAY_BETWEEN_UPDATES_IN_SECS:
1658 msg = '%d files remaining...' % len(remaining)
1659 print msg
1660 logging.info(msg)
1661 last_update = time.time()
csharp@chromium.org9c59ff12012-12-12 02:32:29 +00001662
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001663 if settings.read_only:
vadimsh@chromium.org5db0f4f2013-07-04 13:57:02 +00001664 logging.info('Making files read only')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001665 make_writable(outdir, True)
1666 logging.info('Running %s, cwd=%s' % (cmd, cwd))
csharp@chromium.orge217f302012-11-22 16:51:53 +00001667
1668 # TODO(csharp): This should be specified somewhere else.
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +00001669 # TODO(vadimsh): Pass it via 'env_vars' in manifest.
csharp@chromium.orge217f302012-11-22 16:51:53 +00001670 # Add a rotating log file if one doesn't already exist.
1671 env = os.environ.copy()
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +00001672 env.setdefault('RUN_TEST_CASES_LOG_FILE',
1673 os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001674 try:
1675 with Profiler('RunTest') as _prof:
csharp@chromium.orge217f302012-11-22 16:51:53 +00001676 return subprocess.call(cmd, cwd=cwd, env=env)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001677 except OSError:
1678 print >> sys.stderr, 'Failed to run %s; cwd=%s' % (cmd, cwd)
1679 raise
1680 finally:
1681 rmtree(outdir)
1682
1683
maruel@chromium.orgea101982013-07-24 15:54:29 +00001684class OptionParserWithLogging(optparse.OptionParser):
1685 """Adds --verbose option."""
1686 def __init__(self, verbose=0, log_file=None, **kwargs):
1687 kwargs.setdefault('description', sys.modules['__main__'].__doc__)
1688 optparse.OptionParser.__init__(self, **kwargs)
1689 self.add_option(
1690 '-v', '--verbose',
1691 action='count',
1692 default=verbose,
1693 help='Use multiple times to increase verbosity')
1694 self.add_option(
1695 '-l', '--log_file',
1696 default=log_file,
1697 help='The name of the file to store rotating log details.')
1698
1699 def parse_args(self, *args, **kwargs):
1700 options, args = optparse.OptionParser.parse_args(self, *args, **kwargs)
1701 levels = [logging.ERROR, logging.INFO, logging.DEBUG]
1702 level = levels[min(len(levels) - 1, options.verbose)]
1703
1704 logging_console = logging.StreamHandler()
1705 logging_console.setFormatter(logging.Formatter(
1706 '%(levelname)5s %(module)15s(%(lineno)3d): %(message)s'))
1707 logging_console.setLevel(level)
1708 logging.getLogger().setLevel(level)
1709 logging.getLogger().addHandler(logging_console)
1710
1711 if options.log_file:
1712 # This is necessary otherwise attached handler will miss the messages.
1713 logging.getLogger().setLevel(logging.DEBUG)
1714
1715 logging_rotating_file = logging.handlers.RotatingFileHandler(
1716 options.log_file,
1717 maxBytes=10 * 1024 * 1024,
1718 backupCount=5,
1719 encoding='utf-8')
1720 # log files are always at DEBUG level.
1721 logging_rotating_file.setLevel(logging.DEBUG)
1722 logging_rotating_file.setFormatter(logging.Formatter(
1723 '%(asctime)s %(levelname)-8s %(module)15s(%(lineno)3d): %(message)s'))
1724 logging.getLogger().addHandler(logging_rotating_file)
1725
1726 return options, args
1727
1728
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001729def main():
maruel@chromium.org46e61cc2013-03-25 19:55:34 +00001730 disable_buffering()
maruel@chromium.orgea101982013-07-24 15:54:29 +00001731 parser = OptionParserWithLogging(
1732 usage='%prog <options>', log_file=RUN_ISOLATED_LOG_FILE)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001733
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001734 group = optparse.OptionGroup(parser, 'Download')
1735 group.add_option(
1736 '--download', metavar='DEST',
1737 help='Downloads files to DEST and returns without running, instead of '
1738 'downloading and then running from a temporary directory.')
1739 parser.add_option_group(group)
1740
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001741 group = optparse.OptionGroup(parser, 'Data source')
1742 group.add_option(
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001743 '-s', '--isolated',
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001744 metavar='FILE',
1745 help='File/url describing what to map or run')
1746 group.add_option(
1747 '-H', '--hash',
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001748 help='Hash of the .isolated to grab from the hash table')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001749 parser.add_option_group(group)
1750
1751 group.add_option(
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001752 '-r', '--remote', metavar='URL',
1753 default=
1754 'https://isolateserver.appspot.com/content/retrieve/default-gzip/',
1755 help='Remote where to get the items. Defaults to %default')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001756 group = optparse.OptionGroup(parser, 'Cache management')
1757 group.add_option(
1758 '--cache',
1759 default='cache',
1760 metavar='DIR',
1761 help='Cache directory, default=%default')
1762 group.add_option(
1763 '--max-cache-size',
1764 type='int',
1765 metavar='NNN',
1766 default=20*1024*1024*1024,
1767 help='Trim if the cache gets larger than this value, default=%default')
1768 group.add_option(
1769 '--min-free-space',
1770 type='int',
1771 metavar='NNN',
maruel@chromium.org9e98e432013-05-31 17:06:51 +00001772 default=2*1024*1024*1024,
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001773 help='Trim if disk free space becomes lower than this value, '
1774 'default=%default')
1775 group.add_option(
1776 '--max-items',
1777 type='int',
1778 metavar='NNN',
1779 default=100000,
1780 help='Trim if more than this number of items are in the cache '
1781 'default=%default')
1782 parser.add_option_group(group)
1783
1784 options, args = parser.parse_args()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001785
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001786 if bool(options.isolated) == bool(options.hash):
maruel@chromium.org5dd75dd2012-12-03 15:11:32 +00001787 logging.debug('One and only one of --isolated or --hash is required.')
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001788 parser.error('One and only one of --isolated or --hash is required.')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001789 if args:
maruel@chromium.org5dd75dd2012-12-03 15:11:32 +00001790 logging.debug('Unsupported args %s' % ' '.join(args))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001791 parser.error('Unsupported args %s' % ' '.join(args))
1792
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001793 options.cache = os.path.abspath(options.cache)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001794 policies = CachePolicies(
1795 options.max_cache_size, options.min_free_space, options.max_items)
csharp@chromium.orgffd8cf02013-01-09 21:57:38 +00001796
1797 if options.download:
1798 return download_test_data(options.isolated or options.hash,
1799 options.download, options.remote)
1800 else:
1801 try:
1802 return run_tha_test(
1803 options.isolated or options.hash,
1804 options.cache,
1805 options.remote,
1806 policies)
1807 except Exception, e:
1808 # Make sure any exception is logged.
1809 logging.exception(e)
1810 return 1
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001811
1812
1813if __name__ == '__main__':
csharp@chromium.orgbfb98742013-03-26 20:28:36 +00001814 # Ensure that we are always running with the correct encoding.
1815 fix_default_encoding()
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001816 sys.exit(main())