maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
Marc-Antoine Ruel | 8add124 | 2013-11-05 17:28:27 -0500 | [diff] [blame] | 2 | # Copyright 2013 The Swarming Authors. All rights reserved. |
Marc-Antoine Ruel | e98b112 | 2013-11-05 20:27:57 -0500 | [diff] [blame] | 3 | # Use of this source code is governed under the Apache License, Version 2.0 that |
| 4 | # can be found in the LICENSE file. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 5 | |
| 6 | """Client tool to trigger tasks or retrieve results from a Swarming server.""" |
| 7 | |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 8 | __version__ = '0.2' |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 9 | |
| 10 | import hashlib |
| 11 | import json |
| 12 | import logging |
| 13 | import os |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 14 | import shutil |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 15 | import subprocess |
| 16 | import sys |
| 17 | import time |
| 18 | import urllib |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 19 | |
| 20 | from third_party import colorama |
| 21 | from third_party.depot_tools import fix_encoding |
| 22 | from third_party.depot_tools import subcommand |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 23 | |
| 24 | from utils import net |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 25 | from utils import threading_utils |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 26 | from utils import tools |
| 27 | from utils import zip_package |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 28 | |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 29 | import isolateserver |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 30 | import run_isolated |
| 31 | |
| 32 | |
| 33 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 34 | TOOLS_PATH = os.path.join(ROOT_DIR, 'tools') |
| 35 | |
| 36 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 37 | # The default time to wait for a shard to finish running. |
csharp@chromium.org | 2475849 | 2013-08-28 19:10:54 +0000 | [diff] [blame] | 38 | DEFAULT_SHARD_WAIT_TIME = 80 * 60. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 39 | |
| 40 | |
| 41 | NO_OUTPUT_FOUND = ( |
| 42 | 'No output produced by the test, it may have failed to run.\n' |
| 43 | '\n') |
| 44 | |
| 45 | |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 46 | # TODO(maruel): cygwin != Windows. If a swarm_bot is running in cygwin, it's |
| 47 | # different from running in native python. |
| 48 | PLATFORM_MAPPING_SWARMING = { |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 49 | 'cygwin': 'Windows', |
| 50 | 'darwin': 'Mac', |
| 51 | 'linux2': 'Linux', |
| 52 | 'win32': 'Windows', |
| 53 | } |
| 54 | |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 55 | PLATFORM_MAPPING_ISOLATE = { |
| 56 | 'linux2': 'linux', |
| 57 | 'darwin': 'mac', |
| 58 | 'win32': 'win', |
| 59 | } |
| 60 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 61 | |
| 62 | class Failure(Exception): |
| 63 | """Generic failure.""" |
| 64 | pass |
| 65 | |
| 66 | |
| 67 | class Manifest(object): |
| 68 | """Represents a Swarming task manifest. |
| 69 | |
| 70 | Also includes code to zip code and upload itself. |
| 71 | """ |
| 72 | def __init__( |
Marc-Antoine Ruel | 05dab5e | 2013-11-06 15:06:47 -0500 | [diff] [blame] | 73 | self, isolate_server, isolated_hash, test_name, shards, env, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 74 | dimensions, working_dir, verbose, profile, priority, algo): |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 75 | """Populates a manifest object. |
| 76 | Args: |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 77 | isolate_server - isolate server url. |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 78 | isolated_hash - The manifest's sha-1 that the slave is going to fetch. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 79 | test_name - The name to give the test request. |
| 80 | shards - The number of swarm shards to request. |
Marc-Antoine Ruel | 05dab5e | 2013-11-06 15:06:47 -0500 | [diff] [blame] | 81 | env - environment variables to set. |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 82 | dimensions - dimensions to filter the task on. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 83 | working_dir - Relative working directory to start the script. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 84 | verbose - if True, have the slave print more details. |
| 85 | profile - if True, have the slave print more timing data. |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 86 | priority - int between 0 and 1000, lower the higher priority. |
| 87 | algo - hashing algorithm used. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 88 | """ |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 89 | self.isolate_server = isolate_server |
| 90 | self.storage = isolateserver.get_storage(isolate_server, 'default') |
| 91 | |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 92 | self.isolated_hash = isolated_hash |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 93 | self.bundle = zip_package.ZipPackage(ROOT_DIR) |
| 94 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 95 | self._test_name = test_name |
| 96 | self._shards = shards |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 97 | self._env = env.copy() |
| 98 | self._dimensions = dimensions.copy() |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 99 | self._working_dir = working_dir |
| 100 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 101 | self.verbose = bool(verbose) |
| 102 | self.profile = bool(profile) |
| 103 | self.priority = priority |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 104 | self._algo = algo |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 105 | |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 106 | self._isolate_item = None |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 107 | self._tasks = [] |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 108 | |
| 109 | def add_task(self, task_name, actions, time_out=600): |
| 110 | """Appends a new task to the swarm manifest file.""" |
| 111 | # See swarming/src/common/test_request_message.py TestObject constructor for |
| 112 | # the valid flags. |
| 113 | self._tasks.append( |
| 114 | { |
| 115 | 'action': actions, |
| 116 | 'decorate_output': self.verbose, |
| 117 | 'test_name': task_name, |
| 118 | 'time_out': time_out, |
| 119 | }) |
| 120 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 121 | def zip_and_upload(self): |
| 122 | """Zips up all the files necessary to run a shard and uploads to Swarming |
| 123 | master. |
| 124 | """ |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 125 | assert not self._isolate_item |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 126 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 127 | start_time = time.time() |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 128 | self._isolate_item = isolateserver.BufferItem( |
| 129 | self.bundle.zip_into_buffer(), self._algo, is_isolated=True) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 130 | print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) |
| 131 | |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 132 | try: |
| 133 | start_time = time.time() |
| 134 | uploaded = self.storage.upload_items([self._isolate_item]) |
| 135 | elapsed = time.time() - start_time |
| 136 | except (IOError, OSError) as exc: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 137 | tools.report_error('Failed to upload the zip file: %s' % exc) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 138 | return False |
| 139 | |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 140 | if self._isolate_item in uploaded: |
| 141 | print 'Upload complete, time elapsed: %f' % elapsed |
| 142 | else: |
| 143 | print 'Zip file already on server, time elapsed: %f' % elapsed |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 144 | |
| 145 | return True |
| 146 | |
| 147 | def to_json(self): |
| 148 | """Exports the current configuration into a swarm-readable manifest file. |
| 149 | |
| 150 | This function doesn't mutate the object. |
| 151 | """ |
| 152 | test_case = { |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 153 | 'cleanup': 'root', |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 154 | 'configurations': [ |
| 155 | { |
Marc-Antoine Ruel | 5d79919 | 2013-11-06 15:20:39 -0500 | [diff] [blame] | 156 | 'config_name': 'isolated', |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 157 | 'dimensions': self._dimensions, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 158 | 'min_instances': self._shards, |
| 159 | 'priority': self.priority, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 160 | }, |
| 161 | ], |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 162 | 'data': [], |
| 163 | # TODO: Let the encoding get set from the command line. |
| 164 | 'encoding': 'UTF-8', |
Marc-Antoine Ruel | 05dab5e | 2013-11-06 15:06:47 -0500 | [diff] [blame] | 165 | 'env_vars': self._env, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 166 | 'restart_on_failure': True, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 167 | 'test_case_name': self._test_name, |
| 168 | 'tests': self._tasks, |
| 169 | 'working_dir': self._working_dir, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 170 | } |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 171 | if self._isolate_item: |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 172 | test_case['data'].append( |
| 173 | [ |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 174 | self.storage.get_fetch_url(self._isolate_item.digest), |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 175 | 'swarm_data.zip', |
| 176 | ]) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 177 | return json.dumps(test_case, separators=(',',':')) |
| 178 | |
| 179 | |
| 180 | def now(): |
| 181 | """Exists so it can be mocked easily.""" |
| 182 | return time.time() |
| 183 | |
| 184 | |
| 185 | def get_test_keys(swarm_base_url, test_name): |
| 186 | """Returns the Swarm test key for each shards of test_name.""" |
| 187 | key_data = urllib.urlencode([('name', test_name)]) |
| 188 | url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) |
| 189 | |
vadimsh@chromium.org | 043b76d | 2013-09-12 16:15:13 +0000 | [diff] [blame] | 190 | for _ in net.retry_loop(max_attempts=net.URL_OPEN_MAX_ATTEMPTS): |
| 191 | result = net.url_read(url, retry_404=True) |
| 192 | if result is None: |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 193 | raise Failure( |
| 194 | 'Error: Unable to find any tests with the name, %s, on swarm server' |
| 195 | % test_name) |
| 196 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 197 | # TODO(maruel): Compare exact string. |
| 198 | if 'No matching' in result: |
| 199 | logging.warning('Unable to find any tests with the name, %s, on swarm ' |
| 200 | 'server' % test_name) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 201 | continue |
| 202 | return json.loads(result) |
| 203 | |
| 204 | raise Failure( |
| 205 | 'Error: Unable to find any tests with the name, %s, on swarm server' |
| 206 | % test_name) |
| 207 | |
| 208 | |
| 209 | def retrieve_results(base_url, test_key, timeout, should_stop): |
| 210 | """Retrieves results for a single test_key.""" |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 211 | assert isinstance(timeout, float), timeout |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 212 | params = [('r', test_key)] |
| 213 | result_url = '%s/get_result?%s' % (base_url, urllib.urlencode(params)) |
| 214 | start = now() |
| 215 | while True: |
| 216 | if timeout and (now() - start) >= timeout: |
| 217 | logging.error('retrieve_results(%s) timed out', base_url) |
| 218 | return {} |
| 219 | # Do retries ourselves. |
vadimsh@chromium.org | 043b76d | 2013-09-12 16:15:13 +0000 | [diff] [blame] | 220 | response = net.url_read(result_url, retry_404=False, retry_50x=False) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 221 | if response is None: |
| 222 | # Aggressively poll for results. Do not use retry_404 so |
| 223 | # should_stop is polled more often. |
| 224 | remaining = min(5, timeout - (now() - start)) if timeout else 5 |
| 225 | if remaining > 0: |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 226 | if should_stop.get(): |
| 227 | return {} |
vadimsh@chromium.org | 043b76d | 2013-09-12 16:15:13 +0000 | [diff] [blame] | 228 | net.sleep_before_retry(1, remaining) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 229 | else: |
| 230 | try: |
vadimsh@chromium.org | 043b76d | 2013-09-12 16:15:13 +0000 | [diff] [blame] | 231 | data = json.loads(response) or {} |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 232 | except (ValueError, TypeError): |
| 233 | logging.warning( |
| 234 | 'Received corrupted data for test_key %s. Retrying.', test_key) |
| 235 | else: |
| 236 | if data['output']: |
| 237 | return data |
| 238 | if should_stop.get(): |
| 239 | return {} |
| 240 | |
| 241 | |
| 242 | def yield_results(swarm_base_url, test_keys, timeout, max_threads): |
| 243 | """Yields swarm test results from the swarm server as (index, result). |
| 244 | |
| 245 | Duplicate shards are ignored, the first one to complete is returned. |
| 246 | |
| 247 | max_threads is optional and is used to limit the number of parallel fetches |
| 248 | done. Since in general the number of test_keys is in the range <=10, it's not |
| 249 | worth normally to limit the number threads. Mostly used for testing purposes. |
| 250 | """ |
| 251 | shards_remaining = range(len(test_keys)) |
| 252 | number_threads = ( |
| 253 | min(max_threads, len(test_keys)) if max_threads else len(test_keys)) |
| 254 | should_stop = threading_utils.Bit() |
| 255 | results_remaining = len(test_keys) |
| 256 | with threading_utils.ThreadPool(number_threads, number_threads, 0) as pool: |
| 257 | try: |
| 258 | for test_key in test_keys: |
| 259 | pool.add_task( |
| 260 | 0, retrieve_results, swarm_base_url, test_key, timeout, should_stop) |
| 261 | while shards_remaining and results_remaining: |
| 262 | result = pool.get_one_result() |
| 263 | results_remaining -= 1 |
| 264 | if not result: |
| 265 | # Failed to retrieve one key. |
| 266 | logging.error('Failed to retrieve the results for a swarm key') |
| 267 | continue |
| 268 | shard_index = result['config_instance_index'] |
| 269 | if shard_index in shards_remaining: |
| 270 | shards_remaining.remove(shard_index) |
| 271 | yield shard_index, result |
| 272 | else: |
| 273 | logging.warning('Ignoring duplicate shard index %d', shard_index) |
| 274 | # Pop the last entry, there's no such shard. |
| 275 | shards_remaining.pop() |
| 276 | finally: |
| 277 | # Done, kill the remaining threads. |
| 278 | should_stop.set() |
| 279 | |
| 280 | |
| 281 | def chromium_setup(manifest): |
| 282 | """Sets up the commands to run. |
| 283 | |
| 284 | Highly chromium specific. |
| 285 | """ |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 286 | # Add uncompressed zip here. It'll be compressed as part of the package sent |
| 287 | # to Swarming server. |
| 288 | run_test_name = 'run_isolated.zip' |
| 289 | manifest.bundle.add_buffer(run_test_name, |
| 290 | run_isolated.get_as_zip_package().zip_into_buffer(compress=False)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 291 | |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 292 | cleanup_script_name = 'swarm_cleanup.py' |
| 293 | manifest.bundle.add_file(os.path.join(TOOLS_PATH, cleanup_script_name), |
| 294 | cleanup_script_name) |
| 295 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 296 | run_cmd = [ |
| 297 | 'python', run_test_name, |
maruel@chromium.org | 814d23f | 2013-10-01 19:08:00 +0000 | [diff] [blame] | 298 | '--hash', manifest.isolated_hash, |
maruel@chromium.org | b7e79a2 | 2013-09-13 01:24:56 +0000 | [diff] [blame] | 299 | '--isolate-server', manifest.isolate_server, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 300 | ] |
| 301 | if manifest.verbose or manifest.profile: |
| 302 | # Have it print the profiling section. |
| 303 | run_cmd.append('--verbose') |
| 304 | manifest.add_task('Run Test', run_cmd) |
| 305 | |
| 306 | # Clean up |
| 307 | manifest.add_task('Clean Up', ['python', cleanup_script_name]) |
| 308 | |
| 309 | |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 310 | def archive(isolated, isolate_server, algo, verbose): |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 311 | """Archives a .isolated and all the dependencies on the CAC.""" |
| 312 | tempdir = None |
| 313 | try: |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 314 | logging.info('archive(%s, %s)', isolated, isolate_server) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 315 | cmd = [ |
| 316 | sys.executable, |
| 317 | os.path.join(ROOT_DIR, 'isolate.py'), |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 318 | 'archive', |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 319 | '--outdir', isolate_server, |
| 320 | '--isolated', isolated, |
| 321 | ] |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 322 | cmd.extend(['--verbose'] * verbose) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 323 | logging.info(' '.join(cmd)) |
| 324 | if subprocess.call(cmd, verbose): |
| 325 | return |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 326 | return isolateserver.hash_file(isolated, algo) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 327 | finally: |
| 328 | if tempdir: |
| 329 | shutil.rmtree(tempdir) |
| 330 | |
| 331 | |
| 332 | def process_manifest( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 333 | swarming, isolate_server, file_hash_or_isolated, test_name, shards, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 334 | dimensions, env, working_dir, verbose, profile, priority, algo): |
| 335 | """Processes the manifest file and send off the swarming test request. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 336 | |
| 337 | Optionally archives an .isolated file. |
| 338 | """ |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 339 | if file_hash_or_isolated.endswith('.isolated'): |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 340 | file_hash = archive( |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 341 | file_hash_or_isolated, isolate_server, algo, verbose) |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 342 | if not file_hash: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 343 | tools.report_error('Archival failure %s' % file_hash_or_isolated) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 344 | return 1 |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 345 | elif isolateserver.is_valid_hash(file_hash_or_isolated, algo): |
| 346 | file_hash = file_hash_or_isolated |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 347 | else: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 348 | tools.report_error('Invalid hash %s' % file_hash_or_isolated) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 349 | return 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 350 | try: |
| 351 | manifest = Manifest( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 352 | isolate_server=isolate_server, |
| 353 | isolated_hash=file_hash, |
| 354 | test_name=test_name, |
| 355 | shards=shards, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 356 | dimensions=dimensions, |
Marc-Antoine Ruel | 05dab5e | 2013-11-06 15:06:47 -0500 | [diff] [blame] | 357 | env=env, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 358 | working_dir=working_dir, |
| 359 | verbose=verbose, |
| 360 | profile=profile, |
| 361 | priority=priority, |
| 362 | algo=algo) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 363 | except ValueError as e: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 364 | tools.report_error('Unable to process %s: %s' % (test_name, e)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 365 | return 1 |
| 366 | |
| 367 | chromium_setup(manifest) |
| 368 | |
| 369 | # Zip up relevant files. |
| 370 | print('Zipping up files...') |
| 371 | if not manifest.zip_and_upload(): |
| 372 | return 1 |
| 373 | |
| 374 | # Send test requests off to swarm. |
| 375 | print('Sending test requests to swarm.') |
| 376 | print('Server: %s' % swarming) |
| 377 | print('Job name: %s' % test_name) |
| 378 | test_url = swarming + '/test' |
| 379 | manifest_text = manifest.to_json() |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 380 | result = net.url_read(test_url, data={'request': manifest_text}) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 381 | if not result: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 382 | tools.report_error( |
| 383 | 'Failed to send test for %s\n%s' % (test_name, test_url)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 384 | return 1 |
| 385 | try: |
vadimsh@chromium.org | f24e5c3 | 2013-10-11 21:16:21 +0000 | [diff] [blame] | 386 | json.loads(result) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 387 | except (ValueError, TypeError) as e: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 388 | msg = '\n'.join(( |
| 389 | 'Failed to send test for %s' % test_name, |
| 390 | 'Manifest: %s' % manifest_text, |
| 391 | 'Bad response: %s' % result, |
| 392 | str(e))) |
| 393 | tools.report_error(msg) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 394 | return 1 |
| 395 | return 0 |
| 396 | |
| 397 | |
| 398 | def trigger( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 399 | swarming, |
| 400 | isolate_server, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 401 | tasks, |
| 402 | task_prefix, |
| 403 | working_dir, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 404 | dimensions, |
| 405 | env, |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 406 | verbose, |
| 407 | profile, |
| 408 | priority): |
| 409 | """Sends off the hash swarming test requests.""" |
| 410 | highest_exit_code = 0 |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 411 | for (file_hash, test_name, shards, test_filter) in tasks: |
| 412 | task_env = env.copy() |
| 413 | # These flags are googletest specific. |
| 414 | if test_filter and test_filter != '*': |
| 415 | task_env['GTEST_FILTER'] = test_filter |
| 416 | if int(shards) > 1: |
| 417 | task_env['GTEST_SHARD_INDEX'] = '%(instance_index)s' |
| 418 | task_env['GTEST_TOTAL_SHARDS'] = '%(num_instances)s' |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 419 | # TODO(maruel): It should first create a request manifest object, then pass |
| 420 | # it to a function to zip, archive and trigger. |
| 421 | exit_code = process_manifest( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 422 | swarming=swarming, |
| 423 | isolate_server=isolate_server, |
| 424 | file_hash_or_isolated=file_hash, |
| 425 | test_name=task_prefix + test_name, |
| 426 | shards=int(shards), |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 427 | dimensions=dimensions, |
| 428 | env=task_env, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 429 | working_dir=working_dir, |
| 430 | verbose=verbose, |
| 431 | profile=profile, |
| 432 | priority=priority, |
| 433 | algo=hashlib.sha1) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 434 | highest_exit_code = max(highest_exit_code, exit_code) |
| 435 | return highest_exit_code |
| 436 | |
| 437 | |
| 438 | def decorate_shard_output(result, shard_exit_code): |
| 439 | """Returns wrapped output for swarming task shard.""" |
| 440 | tag = 'index %s (machine tag: %s, id: %s)' % ( |
| 441 | result['config_instance_index'], |
| 442 | result['machine_id'], |
| 443 | result.get('machine_tag', 'unknown')) |
| 444 | return ( |
| 445 | '\n' |
| 446 | '================================================================\n' |
| 447 | 'Begin output from shard %s\n' |
| 448 | '================================================================\n' |
| 449 | '\n' |
| 450 | '%s' |
| 451 | '================================================================\n' |
| 452 | 'End output from shard %s. Return %d\n' |
| 453 | '================================================================\n' |
| 454 | ) % (tag, result['output'] or NO_OUTPUT_FOUND, tag, shard_exit_code) |
| 455 | |
| 456 | |
| 457 | def collect(url, test_name, timeout, decorate): |
| 458 | """Retrieves results of a Swarming job.""" |
| 459 | test_keys = get_test_keys(url, test_name) |
| 460 | if not test_keys: |
| 461 | raise Failure('No test keys to get results with.') |
| 462 | |
maruel@chromium.org | 9c1c7b5 | 2013-08-28 19:04:36 +0000 | [diff] [blame] | 463 | exit_code = None |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 464 | for _index, output in yield_results(url, test_keys, timeout, None): |
| 465 | shard_exit_codes = (output['exit_codes'] or '1').split(',') |
| 466 | shard_exit_code = max(int(i) for i in shard_exit_codes) |
| 467 | if decorate: |
| 468 | print decorate_shard_output(output, shard_exit_code) |
| 469 | else: |
| 470 | print( |
| 471 | '%s/%s: %s' % ( |
| 472 | output['machine_id'], |
| 473 | output['machine_tag'], |
| 474 | output['exit_codes'])) |
| 475 | print(''.join(' %s\n' % l for l in output['output'].splitlines())) |
maruel@chromium.org | 9c1c7b5 | 2013-08-28 19:04:36 +0000 | [diff] [blame] | 476 | exit_code = exit_code or shard_exit_code |
| 477 | return exit_code if exit_code is not None else 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 478 | |
| 479 | |
| 480 | def add_trigger_options(parser): |
| 481 | """Adds all options to trigger a task on Swarming.""" |
| 482 | parser.add_option( |
| 483 | '-I', '--isolate-server', |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 484 | metavar='URL', default='', |
| 485 | help='Isolate server to use') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 486 | parser.add_option( |
| 487 | '-w', '--working_dir', default='swarm_tests', |
| 488 | help='Working directory on the swarm slave side. default: %default.') |
| 489 | parser.add_option( |
| 490 | '-o', '--os', default=sys.platform, |
| 491 | help='Swarm OS image to request. Should be one of the valid sys.platform ' |
| 492 | 'values like darwin, linux2 or win32 default: %default.') |
| 493 | parser.add_option( |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 494 | '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar', |
| 495 | help='environment variables to set') |
| 496 | parser.add_option( |
| 497 | '-d', '--dimensions', default=[], action='append', nargs=2, |
| 498 | dest='dimensions', metavar='FOO bar', |
| 499 | help='dimension to filter on') |
| 500 | parser.add_option( |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 501 | '-T', '--task-prefix', default='', |
| 502 | help='Prefix to give the swarm test request. default: %default') |
| 503 | parser.add_option( |
| 504 | '--profile', action='store_true', |
| 505 | default=bool(os.environ.get('ISOLATE_DEBUG')), |
| 506 | help='Have run_isolated.py print profiling info') |
| 507 | parser.add_option( |
| 508 | '--priority', type='int', default=100, |
| 509 | help='The lower value, the more important the task is') |
| 510 | |
| 511 | |
| 512 | def process_trigger_options(parser, options): |
| 513 | options.isolate_server = options.isolate_server.rstrip('/') |
| 514 | if not options.isolate_server: |
| 515 | parser.error('--isolate-server is required.') |
| 516 | if options.os in ('', 'None'): |
| 517 | # Use the current OS. |
| 518 | options.os = sys.platform |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 519 | if not options.os in PLATFORM_MAPPING_SWARMING: |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 520 | parser.error('Invalid --os option.') |
| 521 | |
| 522 | |
| 523 | def add_collect_options(parser): |
| 524 | parser.add_option( |
| 525 | '-t', '--timeout', |
| 526 | type='float', |
| 527 | default=DEFAULT_SHARD_WAIT_TIME, |
| 528 | help='Timeout to wait for result, set to 0 for no timeout; default: ' |
| 529 | '%default s') |
| 530 | parser.add_option('--decorate', action='store_true', help='Decorate output') |
| 531 | |
| 532 | |
| 533 | @subcommand.usage('test_name') |
| 534 | def CMDcollect(parser, args): |
| 535 | """Retrieves results of a Swarming job. |
| 536 | |
| 537 | The result can be in multiple part if the execution was sharded. It can |
| 538 | potentially have retries. |
| 539 | """ |
| 540 | add_collect_options(parser) |
| 541 | (options, args) = parser.parse_args(args) |
| 542 | if not args: |
| 543 | parser.error('Must specify one test name.') |
| 544 | elif len(args) > 1: |
| 545 | parser.error('Must specify only one test name.') |
| 546 | |
| 547 | try: |
| 548 | return collect(options.swarming, args[0], options.timeout, options.decorate) |
| 549 | except Failure as e: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 550 | tools.report_error(e) |
| 551 | return 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 552 | |
| 553 | |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 554 | @subcommand.usage('[hash|isolated ...]') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 555 | def CMDrun(parser, args): |
| 556 | """Triggers a job and wait for the results. |
| 557 | |
| 558 | Basically, does everything to run command(s) remotely. |
| 559 | """ |
| 560 | add_trigger_options(parser) |
| 561 | add_collect_options(parser) |
| 562 | options, args = parser.parse_args(args) |
| 563 | |
| 564 | if not args: |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 565 | parser.error('Must pass at least one .isolated file or its hash (sha1).') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 566 | process_trigger_options(parser, options) |
| 567 | |
| 568 | success = [] |
| 569 | for arg in args: |
| 570 | logging.info('Triggering %s', arg) |
| 571 | try: |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 572 | dimensions = { |
| 573 | 'os': PLATFORM_MAPPING_SWARMING[options.os], |
| 574 | } |
| 575 | dimensions.update(dict(options.dimensions)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 576 | result = trigger( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 577 | swarming=options.swarming, |
| 578 | isolate_server=options.isolate_server, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 579 | tasks=[(arg, os.path.basename(arg), '1', '')], |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 580 | dimensions=dimensions, |
| 581 | env=dict(options.env), |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 582 | task_prefix=options.task_prefix, |
| 583 | working_dir=options.working_dir, |
| 584 | verbose=options.verbose, |
| 585 | profile=options.profile, |
| 586 | priority=options.priority) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 587 | except Failure as e: |
| 588 | result = e.args[0] |
| 589 | if result: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 590 | tools.report_error('Failed to trigger %s: %s' % (arg, result)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 591 | else: |
| 592 | success.append(os.path.basename(arg)) |
| 593 | |
| 594 | if not success: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 595 | tools.report_error('Failed to trigger any job.') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 596 | return result |
| 597 | |
| 598 | code = 0 |
| 599 | for arg in success: |
| 600 | logging.info('Collecting %s', arg) |
| 601 | try: |
| 602 | new_code = collect( |
| 603 | options.swarming, |
| 604 | options.task_prefix + arg, |
| 605 | options.timeout, |
| 606 | options.decorate) |
| 607 | code = max(code, new_code) |
| 608 | except Failure as e: |
| 609 | code = max(code, 1) |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 610 | tools.report_error(e) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 611 | return code |
| 612 | |
| 613 | |
| 614 | def CMDtrigger(parser, args): |
| 615 | """Triggers Swarm request(s). |
| 616 | |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 617 | Accepts one or multiple --task requests, with either the hash (sha1) of a |
| 618 | .isolated file already uploaded or the path to an .isolated file to archive, |
| 619 | packages it if needed and sends a Swarm manifest file to the Swarm server. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 620 | """ |
| 621 | add_trigger_options(parser) |
| 622 | parser.add_option( |
| 623 | '--task', nargs=4, action='append', default=[], dest='tasks', |
| 624 | help='Task to trigger. The format is ' |
| 625 | '(hash|isolated, test_name, shards, test_filter). This may be ' |
| 626 | 'used multiple times to send multiple hashes jobs. If an isolated ' |
| 627 | 'file is specified instead of an hash, it is first archived.') |
| 628 | (options, args) = parser.parse_args(args) |
| 629 | |
| 630 | if args: |
| 631 | parser.error('Unknown args: %s' % args) |
| 632 | process_trigger_options(parser, options) |
| 633 | if not options.tasks: |
| 634 | parser.error('At least one --task is required.') |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 635 | dimensions = { |
| 636 | 'os': PLATFORM_MAPPING_SWARMING[options.os], |
| 637 | } |
| 638 | dimensions.update(dict(options.dimensions)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 639 | |
| 640 | try: |
| 641 | return trigger( |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 642 | swarming=options.swarming, |
| 643 | isolate_server=options.isolate_server, |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 644 | tasks=options.tasks, |
| 645 | task_prefix=options.task_prefix, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame^] | 646 | dimensions=dimensions, |
| 647 | env=dict(options.env), |
Marc-Antoine Ruel | a704987 | 2013-11-05 19:28:35 -0500 | [diff] [blame] | 648 | working_dir=options.working_dir, |
| 649 | verbose=options.verbose, |
| 650 | profile=options.profile, |
| 651 | priority=options.priority) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 652 | except Failure as e: |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 653 | tools.report_error(e) |
| 654 | return 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 655 | |
| 656 | |
| 657 | class OptionParserSwarming(tools.OptionParserWithLogging): |
| 658 | def __init__(self, **kwargs): |
| 659 | tools.OptionParserWithLogging.__init__( |
| 660 | self, prog='swarming.py', **kwargs) |
| 661 | self.add_option( |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 662 | '-S', '--swarming', |
| 663 | metavar='URL', default='', |
| 664 | help='Swarming server to use') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 665 | |
| 666 | def parse_args(self, *args, **kwargs): |
| 667 | options, args = tools.OptionParserWithLogging.parse_args( |
| 668 | self, *args, **kwargs) |
| 669 | options.swarming = options.swarming.rstrip('/') |
| 670 | if not options.swarming: |
| 671 | self.error('--swarming is required.') |
| 672 | return options, args |
| 673 | |
| 674 | |
| 675 | def main(args): |
| 676 | dispatcher = subcommand.CommandDispatcher(__name__) |
| 677 | try: |
| 678 | return dispatcher.execute(OptionParserSwarming(version=__version__), args) |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 679 | except Exception as e: |
| 680 | tools.report_error(e) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 681 | return 1 |
| 682 | |
| 683 | |
| 684 | if __name__ == '__main__': |
| 685 | fix_encoding.fix_encoding() |
| 686 | tools.disable_buffering() |
| 687 | colorama.init() |
| 688 | sys.exit(main(sys.argv[1:])) |