maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
maruel | ea586f3 | 2016-04-05 11:11:33 -0700 | [diff] [blame] | 2 | # Copyright 2013 The LUCI Authors. All rights reserved. |
maruel | f1f5e2a | 2016-05-25 17:10:39 -0700 | [diff] [blame] | 3 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 | # that can be found in the LICENSE file. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 5 | |
| 6 | """Client tool to trigger tasks or retrieve results from a Swarming server.""" |
| 7 | |
Takuto Ikuta | 0e3e1c4 | 2018-11-29 14:21:06 +0000 | [diff] [blame] | 8 | __version__ = '0.14' |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 9 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 10 | import collections |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 11 | import datetime |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 12 | import json |
| 13 | import logging |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 14 | import optparse |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 15 | import os |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 16 | import re |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 17 | import sys |
maruel | 11e31af | 2017-02-15 07:30:50 -0800 | [diff] [blame] | 18 | import textwrap |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 19 | import threading |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 20 | import time |
| 21 | import urllib |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 22 | |
| 23 | from third_party import colorama |
| 24 | from third_party.depot_tools import fix_encoding |
| 25 | from third_party.depot_tools import subcommand |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 26 | |
Marc-Antoine Ruel | 8806e62 | 2014-02-12 14:15:53 -0500 | [diff] [blame] | 27 | from utils import file_path |
maruel | 12e3001 | 2015-10-09 11:55:35 -0700 | [diff] [blame] | 28 | from utils import fs |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 29 | from utils import logging_utils |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 30 | from third_party.chromium import natsort |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 31 | from utils import net |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 32 | from utils import on_error |
maruel | 8e4e40c | 2016-05-30 06:21:07 -0700 | [diff] [blame] | 33 | from utils import subprocess42 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 34 | from utils import threading_utils |
vadimsh@chromium.org | 6b70621 | 2013-08-28 15:03:46 +0000 | [diff] [blame] | 35 | from utils import tools |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 36 | |
Vadim Shtayura | e34e13a | 2014-02-02 11:23:26 -0800 | [diff] [blame] | 37 | import auth |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 38 | import cipd |
maruel@chromium.org | 7b844a6 | 2013-09-17 13:04:59 +0000 | [diff] [blame] | 39 | import isolateserver |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 40 | import isolate_storage |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 41 | import isolated_format |
Marc-Antoine Ruel | 2666d9c | 2018-05-18 13:52:02 -0400 | [diff] [blame] | 42 | import local_caching |
maruel | c070e67 | 2016-02-22 17:32:57 -0800 | [diff] [blame] | 43 | import run_isolated |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 44 | |
| 45 | |
tansell | a494944 | 2016-06-23 22:34:32 -0700 | [diff] [blame] | 46 | ROOT_DIR = os.path.dirname(os.path.abspath( |
| 47 | __file__.decode(sys.getfilesystemencoding()))) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 48 | |
| 49 | |
| 50 | class Failure(Exception): |
| 51 | """Generic failure.""" |
| 52 | pass |
| 53 | |
| 54 | |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 55 | def default_task_name(options): |
| 56 | """Returns a default task name if not specified.""" |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 57 | if not options.task_name: |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 58 | task_name = u'%s/%s' % ( |
maruel | d9cc842 | 2017-05-09 12:07:02 -0700 | [diff] [blame] | 59 | options.user, |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 60 | '_'.join('%s=%s' % (k, v) for k, v in options.dimensions)) |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 61 | if options.isolated: |
| 62 | task_name += u'/' + options.isolated |
| 63 | return task_name |
| 64 | return options.task_name |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 65 | |
| 66 | |
| 67 | ### Triggering. |
| 68 | |
| 69 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 70 | # See ../appengine/swarming/swarming_rpcs.py. |
borenet | 02f772b | 2016-06-22 12:42:19 -0700 | [diff] [blame] | 71 | CipdPackage = collections.namedtuple( |
| 72 | 'CipdPackage', |
| 73 | [ |
| 74 | 'package_name', |
| 75 | 'path', |
| 76 | 'version', |
| 77 | ]) |
| 78 | |
| 79 | |
| 80 | # See ../appengine/swarming/swarming_rpcs.py. |
| 81 | CipdInput = collections.namedtuple( |
| 82 | 'CipdInput', |
| 83 | [ |
| 84 | 'client_package', |
| 85 | 'packages', |
| 86 | 'server', |
| 87 | ]) |
| 88 | |
| 89 | |
| 90 | # See ../appengine/swarming/swarming_rpcs.py. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 91 | FilesRef = collections.namedtuple( |
| 92 | 'FilesRef', |
| 93 | [ |
| 94 | 'isolated', |
| 95 | 'isolatedserver', |
| 96 | 'namespace', |
| 97 | ]) |
| 98 | |
| 99 | |
| 100 | # See ../appengine/swarming/swarming_rpcs.py. |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 101 | StringListPair = collections.namedtuple( |
| 102 | 'StringListPair', [ |
| 103 | 'key', |
| 104 | 'value', # repeated string |
| 105 | ] |
| 106 | ) |
| 107 | |
| 108 | |
| 109 | # See ../appengine/swarming/swarming_rpcs.py. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 110 | TaskProperties = collections.namedtuple( |
| 111 | 'TaskProperties', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 112 | [ |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 113 | 'caches', |
borenet | 02f772b | 2016-06-22 12:42:19 -0700 | [diff] [blame] | 114 | 'cipd_input', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 115 | 'command', |
Marc-Antoine Ruel | ba1bf22 | 2017-12-21 21:41:01 -0500 | [diff] [blame] | 116 | 'relative_cwd', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 117 | 'dimensions', |
| 118 | 'env', |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 119 | 'env_prefixes', |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 120 | 'execution_timeout_secs', |
| 121 | 'extra_args', |
| 122 | 'grace_period_secs', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 123 | 'idempotent', |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 124 | 'inputs_ref', |
| 125 | 'io_timeout_secs', |
aludwin | cc5524e | 2016-10-28 10:25:24 -0700 | [diff] [blame] | 126 | 'outputs', |
iannucci | dc80dfb | 2016-10-28 12:50:20 -0700 | [diff] [blame] | 127 | 'secret_bytes', |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 128 | ]) |
| 129 | |
| 130 | |
| 131 | # See ../appengine/swarming/swarming_rpcs.py. |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 132 | TaskSlice = collections.namedtuple( |
| 133 | 'TaskSlice', |
| 134 | [ |
| 135 | 'expiration_secs', |
| 136 | 'properties', |
| 137 | 'wait_for_capacity', |
| 138 | ]) |
| 139 | |
| 140 | |
| 141 | # See ../appengine/swarming/swarming_rpcs.py. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 142 | NewTaskRequest = collections.namedtuple( |
| 143 | 'NewTaskRequest', |
| 144 | [ |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 145 | 'name', |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 146 | 'parent_task_id', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 147 | 'priority', |
Marc-Antoine Ruel | b73066b | 2018-04-19 20:16:55 -0400 | [diff] [blame] | 148 | 'task_slices', |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 149 | 'service_account', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 150 | 'tags', |
| 151 | 'user', |
Robert Iannucci | fafa735 | 2018-06-13 17:08:17 +0000 | [diff] [blame] | 152 | 'pool_task_template', |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 153 | ]) |
| 154 | |
| 155 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 156 | def namedtuple_to_dict(value): |
| 157 | """Recursively converts a namedtuple to a dict.""" |
Marc-Antoine Ruel | b73066b | 2018-04-19 20:16:55 -0400 | [diff] [blame] | 158 | if hasattr(value, '_asdict'): |
| 159 | return namedtuple_to_dict(value._asdict()) |
| 160 | if isinstance(value, (list, tuple)): |
| 161 | return [namedtuple_to_dict(v) for v in value] |
| 162 | if isinstance(value, dict): |
| 163 | return {k: namedtuple_to_dict(v) for k, v in value.iteritems()} |
| 164 | return value |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 165 | |
| 166 | |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 167 | def task_request_to_raw_request(task_request): |
maruel | 71c61c8 | 2016-02-22 06:52:05 -0800 | [diff] [blame] | 168 | """Returns the json-compatible dict expected by the server for new request. |
maruel | af6269b | 2015-09-10 14:37:51 -0700 | [diff] [blame] | 169 | |
| 170 | This is for the v1 client Swarming API. |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 171 | """ |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 172 | out = namedtuple_to_dict(task_request) |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 173 | # Don't send 'service_account' if it is None to avoid confusing older |
| 174 | # version of the server that doesn't know about 'service_account' and don't |
| 175 | # use it at all. |
| 176 | if not out['service_account']: |
| 177 | out.pop('service_account') |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 178 | for task_slice in out['task_slices']: |
| 179 | task_slice['properties']['env'] = [ |
| 180 | {'key': k, 'value': v} |
| 181 | for k, v in task_slice['properties']['env'].iteritems() |
| 182 | ] |
| 183 | task_slice['properties']['env'].sort(key=lambda x: x['key']) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 184 | return out |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 185 | |
| 186 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 187 | def swarming_trigger(swarming, raw_request): |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 188 | """Triggers a request on the Swarming server and returns the json data. |
| 189 | |
| 190 | It's the low-level function. |
| 191 | |
| 192 | Returns: |
| 193 | { |
| 194 | 'request': { |
| 195 | 'created_ts': u'2010-01-02 03:04:05', |
| 196 | 'name': .. |
| 197 | }, |
| 198 | 'task_id': '12300', |
| 199 | } |
| 200 | """ |
| 201 | logging.info('Triggering: %s', raw_request['name']) |
| 202 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 203 | result = net.url_read_json( |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 204 | swarming + '/_ah/api/swarming/v1/tasks/new', data=raw_request) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 205 | if not result: |
| 206 | on_error.report('Failed to trigger task %s' % raw_request['name']) |
| 207 | return None |
maruel | e557bce | 2015-11-17 09:01:27 -0800 | [diff] [blame] | 208 | if result.get('error'): |
maruel | d4d1531 | 2015-11-16 17:22:59 -0800 | [diff] [blame] | 209 | # The reply is an error. |
maruel | e557bce | 2015-11-17 09:01:27 -0800 | [diff] [blame] | 210 | msg = 'Failed to trigger task %s' % raw_request['name'] |
| 211 | if result['error'].get('errors'): |
| 212 | for err in result['error']['errors']: |
| 213 | if err.get('message'): |
| 214 | msg += '\nMessage: %s' % err['message'] |
| 215 | if err.get('debugInfo'): |
| 216 | msg += '\nDebug info:\n%s' % err['debugInfo'] |
| 217 | elif result['error'].get('message'): |
maruel | bf53e04 | 2015-12-01 15:00:51 -0800 | [diff] [blame] | 218 | msg += '\nMessage: %s' % result['error']['message'] |
maruel | e557bce | 2015-11-17 09:01:27 -0800 | [diff] [blame] | 219 | |
| 220 | on_error.report(msg) |
maruel | d4d1531 | 2015-11-16 17:22:59 -0800 | [diff] [blame] | 221 | return None |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 222 | return result |
| 223 | |
| 224 | |
| 225 | def setup_googletest(env, shards, index): |
| 226 | """Sets googletest specific environment variables.""" |
| 227 | if shards > 1: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 228 | assert not any(i['key'] == 'GTEST_SHARD_INDEX' for i in env), env |
| 229 | assert not any(i['key'] == 'GTEST_TOTAL_SHARDS' for i in env), env |
| 230 | env = env[:] |
| 231 | env.append({'key': 'GTEST_SHARD_INDEX', 'value': str(index)}) |
| 232 | env.append({'key': 'GTEST_TOTAL_SHARDS', 'value': str(shards)}) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 233 | return env |
| 234 | |
| 235 | |
| 236 | def trigger_task_shards(swarming, task_request, shards): |
| 237 | """Triggers one or many subtasks of a sharded task. |
| 238 | |
| 239 | Returns: |
| 240 | Dict with task details, returned to caller as part of --dump-json output. |
| 241 | None in case of failure. |
| 242 | """ |
| 243 | def convert(index): |
Erik Chen | d50a88f | 2019-02-16 01:22:07 +0000 | [diff] [blame^] | 244 | """ |
| 245 | Args: |
| 246 | index: The index of the task request. |
| 247 | |
| 248 | Returns: |
| 249 | raw_request: A swarming compatible JSON dictionary of the request. |
| 250 | shard_index: The index of the shard, which may be different than the index |
| 251 | of the task request. |
| 252 | """ |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 253 | req = task_request_to_raw_request(task_request) |
Erik Chen | d50a88f | 2019-02-16 01:22:07 +0000 | [diff] [blame^] | 254 | shard_index = index |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 255 | if shards > 1: |
Brad Hall | 157bec8 | 2018-11-26 22:15:38 +0000 | [diff] [blame] | 256 | for task_slice in req['task_slices']: |
| 257 | task_slice['properties']['env'] = setup_googletest( |
| 258 | task_slice['properties']['env'], shards, index) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 259 | req['name'] += ':%s:%s' % (index, shards) |
Erik Chen | d50a88f | 2019-02-16 01:22:07 +0000 | [diff] [blame^] | 260 | else: |
| 261 | task_slices = req['task_slices'] |
| 262 | |
| 263 | total_shards = None |
| 264 | # Multiple tasks slices might exist if there are optional "slices", e.g. |
| 265 | # multiple ways of dispatching the task that should be equivalent. These |
| 266 | # should be functionally equivalent but we have cannot guarantee that. If |
| 267 | # we see the GTEST_SHARD_INDEX env var, we assume that it applies to all |
| 268 | # slices. |
| 269 | for task_slice in task_slices: |
| 270 | for env_var in task_slice['properties']['env']: |
| 271 | if env_var['key'] == 'GTEST_SHARD_INDEX': |
| 272 | shard_index = int(env_var['value']) |
| 273 | if env_var['key'] == 'GTEST_TOTAL_SHARDS': |
| 274 | total_shards = int(env_var['value']) |
| 275 | if total_shards > 1: |
| 276 | req['name'] += ':%s:%s' % (shard_index, total_shards) |
| 277 | |
| 278 | return req, shard_index |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 279 | |
| 280 | requests = [convert(index) for index in xrange(shards)] |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 281 | tasks = {} |
| 282 | priority_warning = False |
Erik Chen | d50a88f | 2019-02-16 01:22:07 +0000 | [diff] [blame^] | 283 | for request, shard_index in requests: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 284 | task = swarming_trigger(swarming, request) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 285 | if not task: |
| 286 | break |
| 287 | logging.info('Request result: %s', task) |
| 288 | if (not priority_warning and |
Marc-Antoine Ruel | b121676 | 2017-08-17 10:07:49 -0400 | [diff] [blame] | 289 | int(task['request']['priority']) != task_request.priority): |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 290 | priority_warning = True |
| 291 | print >> sys.stderr, ( |
| 292 | 'Priority was reset to %s' % task['request']['priority']) |
| 293 | tasks[request['name']] = { |
Erik Chen | d50a88f | 2019-02-16 01:22:07 +0000 | [diff] [blame^] | 294 | 'shard_index': shard_index, |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 295 | 'task_id': task['task_id'], |
| 296 | 'view_url': '%s/user/task/%s' % (swarming, task['task_id']), |
| 297 | } |
| 298 | |
| 299 | # Some shards weren't triggered. Abort everything. |
| 300 | if len(tasks) != len(requests): |
| 301 | if tasks: |
| 302 | print >> sys.stderr, 'Only %d shard(s) out of %d were triggered' % ( |
| 303 | len(tasks), len(requests)) |
| 304 | for task_dict in tasks.itervalues(): |
| 305 | abort_task(swarming, task_dict['task_id']) |
| 306 | return None |
| 307 | |
| 308 | return tasks |
| 309 | |
| 310 | |
| 311 | ### Collection. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 312 | |
| 313 | |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 314 | # How often to print status updates to stdout in 'collect'. |
Jao-ke Chin-Lee | ba184e6 | 2018-11-19 17:04:41 +0000 | [diff] [blame] | 315 | STATUS_UPDATE_INTERVAL = 5 * 60. |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 316 | |
Marc-Antoine Ruel | 2f6581a | 2014-10-03 11:09:53 -0400 | [diff] [blame] | 317 | |
Marc-Antoine Ruel | 20b764d | 2018-06-22 18:08:37 +0000 | [diff] [blame] | 318 | class TaskState(object): |
| 319 | """Represents the current task state. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 320 | |
Marc-Antoine Ruel | 20b764d | 2018-06-22 18:08:37 +0000 | [diff] [blame] | 321 | For documentation, see the comments in the swarming_rpcs.TaskState enum, which |
| 322 | is the source of truth for these values: |
| 323 | https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_rpcs.py?q=TaskState\( |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 324 | |
Marc-Antoine Ruel | 20b764d | 2018-06-22 18:08:37 +0000 | [diff] [blame] | 325 | It's in fact an enum. |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 326 | """ |
| 327 | RUNNING = 0x10 |
| 328 | PENDING = 0x20 |
| 329 | EXPIRED = 0x30 |
| 330 | TIMED_OUT = 0x40 |
| 331 | BOT_DIED = 0x50 |
| 332 | CANCELED = 0x60 |
| 333 | COMPLETED = 0x70 |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 334 | KILLED = 0x80 |
Marc-Antoine Ruel | fc70835 | 2018-05-04 20:25:43 -0400 | [diff] [blame] | 335 | NO_RESOURCE = 0x100 |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 336 | |
Marc-Antoine Ruel | 20b764d | 2018-06-22 18:08:37 +0000 | [diff] [blame] | 337 | STATES_RUNNING = ('PENDING', 'RUNNING') |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 338 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 339 | _ENUMS = { |
| 340 | 'RUNNING': RUNNING, |
| 341 | 'PENDING': PENDING, |
| 342 | 'EXPIRED': EXPIRED, |
| 343 | 'TIMED_OUT': TIMED_OUT, |
| 344 | 'BOT_DIED': BOT_DIED, |
| 345 | 'CANCELED': CANCELED, |
| 346 | 'COMPLETED': COMPLETED, |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 347 | 'KILLED': KILLED, |
Marc-Antoine Ruel | fc70835 | 2018-05-04 20:25:43 -0400 | [diff] [blame] | 348 | 'NO_RESOURCE': NO_RESOURCE, |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 349 | } |
| 350 | |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 351 | @classmethod |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 352 | def from_enum(cls, state): |
| 353 | """Returns int value based on the string.""" |
| 354 | if state not in cls._ENUMS: |
| 355 | raise ValueError('Invalid state %s' % state) |
| 356 | return cls._ENUMS[state] |
| 357 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 358 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 359 | class TaskOutputCollector(object): |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 360 | """Assembles task execution summary (for --task-summary-json output). |
| 361 | |
| 362 | Optionally fetches task outputs from isolate server to local disk (used when |
| 363 | --task-output-dir is passed). |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 364 | |
| 365 | This object is shared among multiple threads running 'retrieve_results' |
| 366 | function, in particular they call 'process_shard_result' method in parallel. |
| 367 | """ |
| 368 | |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 369 | def __init__(self, task_output_dir, task_output_stdout, shard_count, |
| 370 | filter_cb): |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 371 | """Initializes TaskOutputCollector, ensures |task_output_dir| exists. |
| 372 | |
| 373 | Args: |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 374 | task_output_dir: (optional) local directory to put fetched files to. |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 375 | shard_count: expected number of task shards. |
| 376 | """ |
maruel | 12e3001 | 2015-10-09 11:55:35 -0700 | [diff] [blame] | 377 | self.task_output_dir = ( |
| 378 | unicode(os.path.abspath(task_output_dir)) |
| 379 | if task_output_dir else task_output_dir) |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 380 | self.task_output_stdout = task_output_stdout |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 381 | self.shard_count = shard_count |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 382 | self.filter_cb = filter_cb |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 383 | |
| 384 | self._lock = threading.Lock() |
| 385 | self._per_shard_results = {} |
| 386 | self._storage = None |
| 387 | |
nodir | e5028a9 | 2016-04-29 14:38:21 -0700 | [diff] [blame] | 388 | if self.task_output_dir: |
| 389 | file_path.ensure_tree(self.task_output_dir) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 390 | |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 391 | def process_shard_result(self, shard_index, result): |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 392 | """Stores results of a single task shard, fetches output files if necessary. |
| 393 | |
Marc-Antoine Ruel | e4dcbb8 | 2014-10-01 09:30:56 -0400 | [diff] [blame] | 394 | Modifies |result| in place. |
| 395 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 396 | shard_index is 0-based. |
| 397 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 398 | Called concurrently from multiple threads. |
| 399 | """ |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 400 | # Sanity check index is in expected range. |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 401 | assert isinstance(shard_index, int) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 402 | if shard_index < 0 or shard_index >= self.shard_count: |
| 403 | logging.warning( |
| 404 | 'Shard index %d is outside of expected range: [0; %d]', |
| 405 | shard_index, self.shard_count - 1) |
| 406 | return |
| 407 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 408 | if result.get('outputs_ref'): |
| 409 | ref = result['outputs_ref'] |
| 410 | result['outputs_ref']['view_url'] = '%s/browse?%s' % ( |
| 411 | ref['isolatedserver'], |
| 412 | urllib.urlencode( |
| 413 | [('namespace', ref['namespace']), ('hash', ref['isolated'])])) |
Kevin Graney | c2c3b9e | 2014-08-26 09:04:17 -0400 | [diff] [blame] | 414 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 415 | # Store result dict of that shard, ignore results we've already seen. |
| 416 | with self._lock: |
| 417 | if shard_index in self._per_shard_results: |
| 418 | logging.warning('Ignoring duplicate shard index %d', shard_index) |
| 419 | return |
| 420 | self._per_shard_results[shard_index] = result |
| 421 | |
| 422 | # Fetch output files if necessary. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 423 | if self.task_output_dir and result.get('outputs_ref'): |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 424 | server_ref = isolate_storage.ServerRef( |
| 425 | result['outputs_ref']['isolatedserver'], |
| 426 | result['outputs_ref']['namespace']) |
| 427 | storage = self._get_storage(server_ref) |
Marc-Antoine Ruel | e4dcbb8 | 2014-10-01 09:30:56 -0400 | [diff] [blame] | 428 | if storage: |
| 429 | # Output files are supposed to be small and they are not reused across |
Marc-Antoine Ruel | 2666d9c | 2018-05-18 13:52:02 -0400 | [diff] [blame] | 430 | # tasks. So use MemoryContentAddressedCache for them instead of on-disk |
| 431 | # cache. Make files writable, so that calling script can delete them. |
Marc-Antoine Ruel | e4dcbb8 | 2014-10-01 09:30:56 -0400 | [diff] [blame] | 432 | isolateserver.fetch_isolated( |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 433 | result['outputs_ref']['isolated'], |
Marc-Antoine Ruel | e4dcbb8 | 2014-10-01 09:30:56 -0400 | [diff] [blame] | 434 | storage, |
Marc-Antoine Ruel | 2666d9c | 2018-05-18 13:52:02 -0400 | [diff] [blame] | 435 | local_caching.MemoryContentAddressedCache(file_mode_mask=0700), |
maruel | 4409e30 | 2016-07-19 14:25:51 -0700 | [diff] [blame] | 436 | os.path.join(self.task_output_dir, str(shard_index)), |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 437 | False, self.filter_cb) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 438 | |
| 439 | def finalize(self): |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 440 | """Assembles and returns task summary JSON, shutdowns underlying Storage.""" |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 441 | with self._lock: |
| 442 | # Write an array of shard results with None for missing shards. |
| 443 | summary = { |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 444 | 'shards': [ |
| 445 | self._per_shard_results.get(i) for i in xrange(self.shard_count) |
| 446 | ], |
| 447 | } |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 448 | |
| 449 | # Don't store stdout in the summary if not requested too. |
| 450 | if "json" not in self.task_output_stdout: |
| 451 | for shard_json in summary['shards']: |
| 452 | if not shard_json: |
| 453 | continue |
| 454 | if "output" in shard_json: |
| 455 | del shard_json["output"] |
| 456 | if "outputs" in shard_json: |
| 457 | del shard_json["outputs"] |
| 458 | |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 459 | # Write summary.json to task_output_dir as well. |
| 460 | if self.task_output_dir: |
| 461 | tools.write_json( |
maruel | 12e3001 | 2015-10-09 11:55:35 -0700 | [diff] [blame] | 462 | os.path.join(self.task_output_dir, u'summary.json'), |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 463 | summary, |
| 464 | False) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 465 | if self._storage: |
| 466 | self._storage.close() |
| 467 | self._storage = None |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 468 | return summary |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 469 | |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 470 | def _get_storage(self, server_ref): |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 471 | """Returns isolateserver.Storage to use to fetch files.""" |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 472 | assert self.task_output_dir |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 473 | with self._lock: |
| 474 | if not self._storage: |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 475 | self._storage = isolateserver.get_storage(server_ref) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 476 | else: |
| 477 | # Shards must all use exact same isolate server and namespace. |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 478 | if self._storage.server_ref.url != server_ref.url: |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 479 | logging.error( |
| 480 | 'Task shards are using multiple isolate servers: %s and %s', |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 481 | self._storage.server_ref.url, server_ref.url) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 482 | return None |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 483 | if self._storage.server_ref.namespace != server_ref.namespace: |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 484 | logging.error( |
| 485 | 'Task shards are using multiple namespaces: %s and %s', |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 486 | self._storage.server_ref.namespace, server_ref.namespace) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 487 | return None |
| 488 | return self._storage |
| 489 | |
| 490 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 491 | def now(): |
| 492 | """Exists so it can be mocked easily.""" |
| 493 | return time.time() |
| 494 | |
| 495 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 496 | def parse_time(value): |
| 497 | """Converts serialized time from the API to datetime.datetime.""" |
| 498 | # When microseconds are 0, the '.123456' suffix is elided. This means the |
| 499 | # serialized format is not consistent, which confuses the hell out of python. |
| 500 | for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S'): |
| 501 | try: |
| 502 | return datetime.datetime.strptime(value, fmt) |
| 503 | except ValueError: |
| 504 | pass |
| 505 | raise ValueError('Failed to parse %s' % value) |
| 506 | |
| 507 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 508 | def retrieve_results( |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 509 | base_url, shard_index, task_id, timeout, should_stop, output_collector, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 510 | include_perf, fetch_stdout): |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 511 | """Retrieves results for a single task ID. |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 512 | |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 513 | Returns: |
| 514 | <result dict> on success. |
| 515 | None on failure. |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 516 | """ |
maruel | 71c61c8 | 2016-02-22 06:52:05 -0800 | [diff] [blame] | 517 | assert timeout is None or isinstance(timeout, float), timeout |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 518 | result_url = '%s/_ah/api/swarming/v1/task/%s/result' % (base_url, task_id) |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 519 | if include_perf: |
| 520 | result_url += '?include_performance_stats=true' |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 521 | output_url = '%s/_ah/api/swarming/v1/task/%s/stdout' % (base_url, task_id) |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 522 | started = now() |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 523 | deadline = started + timeout if timeout > 0 else None |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 524 | attempt = 0 |
| 525 | |
| 526 | while not should_stop.is_set(): |
| 527 | attempt += 1 |
| 528 | |
| 529 | # Waiting for too long -> give up. |
| 530 | current_time = now() |
| 531 | if deadline and current_time >= deadline: |
| 532 | logging.error('retrieve_results(%s) timed out on attempt %d', |
| 533 | base_url, attempt) |
| 534 | return None |
| 535 | |
| 536 | # Do not spin too fast. Spin faster at the beginning though. |
| 537 | # Start with 1 sec delay and for each 30 sec of waiting add another second |
| 538 | # of delay, until hitting 15 sec ceiling. |
| 539 | if attempt > 1: |
| 540 | max_delay = min(15, 1 + (current_time - started) / 30.0) |
| 541 | delay = min(max_delay, deadline - current_time) if deadline else max_delay |
| 542 | if delay > 0: |
| 543 | logging.debug('Waiting %.1f sec before retrying', delay) |
| 544 | should_stop.wait(delay) |
| 545 | if should_stop.is_set(): |
| 546 | return None |
| 547 | |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 548 | # Disable internal retries in net.url_read_json, since we are doing retries |
| 549 | # ourselves. |
| 550 | # TODO(maruel): We'd need to know if it's a 404 and not retry at all. |
maruel | 0eb1d1b | 2015-10-02 14:48:21 -0700 | [diff] [blame] | 551 | # TODO(maruel): Sadly, we currently have to poll here. Use hanging HTTP |
| 552 | # request on GAE v2. |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 553 | # Retry on 500s only if no timeout is specified. |
| 554 | result = net.url_read_json(result_url, retry_50x=bool(timeout == -1)) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 555 | if not result: |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 556 | if timeout == -1: |
| 557 | return None |
Marc-Antoine Ruel | 200b395 | 2014-08-14 11:07:44 -0400 | [diff] [blame] | 558 | continue |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 559 | |
maruel | bf53e04 | 2015-12-01 15:00:51 -0800 | [diff] [blame] | 560 | if result.get('error'): |
| 561 | # An error occurred. |
| 562 | if result['error'].get('errors'): |
| 563 | for err in result['error']['errors']: |
| 564 | logging.warning( |
| 565 | 'Error while reading task: %s; %s', |
| 566 | err.get('message'), err.get('debugInfo')) |
| 567 | elif result['error'].get('message'): |
| 568 | logging.warning( |
| 569 | 'Error while reading task: %s', result['error']['message']) |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 570 | if timeout == -1: |
| 571 | return result |
maruel | bf53e04 | 2015-12-01 15:00:51 -0800 | [diff] [blame] | 572 | continue |
| 573 | |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 574 | # When timeout == -1, always return on first attempt. 500s are already |
| 575 | # retried in this case. |
Marc-Antoine Ruel | 20b764d | 2018-06-22 18:08:37 +0000 | [diff] [blame] | 576 | if result['state'] not in TaskState.STATES_RUNNING or timeout == -1: |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 577 | if fetch_stdout: |
| 578 | out = net.url_read_json(output_url) |
Vadim Shtayura | 6fd3c7b | 2017-11-03 15:32:51 -0700 | [diff] [blame] | 579 | result['output'] = out.get('output', '') if out else '' |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 580 | # Record the result, try to fetch attached output files (if any). |
| 581 | if output_collector: |
| 582 | # TODO(vadimsh): Respect |should_stop| and |deadline| when fetching. |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 583 | output_collector.process_shard_result(shard_index, result) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 584 | if result.get('internal_failure'): |
| 585 | logging.error('Internal error!') |
| 586 | elif result['state'] == 'BOT_DIED': |
| 587 | logging.error('Bot died!') |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 588 | return result |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 589 | |
| 590 | |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 591 | def yield_results( |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 592 | swarm_base_url, task_ids, timeout, max_threads, print_status_updates, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 593 | output_collector, include_perf, fetch_stdout): |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 594 | """Yields swarming task results from the swarming server as (index, result). |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 595 | |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 596 | Duplicate shards are ignored. Shards are yielded in order of completion. |
| 597 | Timed out shards are NOT yielded at all. Caller can compare number of yielded |
| 598 | shards with len(task_keys) to verify all shards completed. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 599 | |
| 600 | max_threads is optional and is used to limit the number of parallel fetches |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 601 | done. Since in general the number of task_keys is in the range <=10, it's not |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 602 | worth normally to limit the number threads. Mostly used for testing purposes. |
Marc-Antoine Ruel | 5c72034 | 2014-02-21 14:46:14 -0500 | [diff] [blame] | 603 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 604 | output_collector is an optional instance of TaskOutputCollector that will be |
| 605 | used to fetch files produced by a task from isolate server to the local disk. |
| 606 | |
Marc-Antoine Ruel | 5c72034 | 2014-02-21 14:46:14 -0500 | [diff] [blame] | 607 | Yields: |
| 608 | (index, result). In particular, 'result' is defined as the |
| 609 | GetRunnerResults() function in services/swarming/server/test_runner.py. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 610 | """ |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 611 | number_threads = ( |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 612 | min(max_threads, len(task_ids)) if max_threads else len(task_ids)) |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 613 | should_stop = threading.Event() |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 614 | results_channel = threading_utils.TaskChannel() |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 615 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 616 | with threading_utils.ThreadPool(number_threads, number_threads, 0) as pool: |
| 617 | try: |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 618 | # Adds a task to the thread pool to call 'retrieve_results' and return |
| 619 | # the results together with shard_index that produced them (as a tuple). |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 620 | def enqueue_retrieve_results(shard_index, task_id): |
Marc-Antoine Ruel | 486c9b5 | 2018-07-23 19:30:47 +0000 | [diff] [blame] | 621 | # pylint: disable=no-value-for-parameter |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 622 | task_fn = lambda *args: (shard_index, retrieve_results(*args)) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 623 | pool.add_task( |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 624 | 0, results_channel.wrap_task(task_fn), swarm_base_url, shard_index, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 625 | task_id, timeout, should_stop, output_collector, include_perf, |
| 626 | fetch_stdout) |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 627 | |
| 628 | # Enqueue 'retrieve_results' calls for each shard key to run in parallel. |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 629 | for shard_index, task_id in enumerate(task_ids): |
| 630 | enqueue_retrieve_results(shard_index, task_id) |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 631 | |
| 632 | # Wait for all of them to finish. |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 633 | shards_remaining = range(len(task_ids)) |
| 634 | active_task_count = len(task_ids) |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 635 | while active_task_count: |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 636 | shard_index, result = None, None |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 637 | try: |
Marc-Antoine Ruel | 4494b6c | 2018-11-28 21:00:41 +0000 | [diff] [blame] | 638 | shard_index, result = results_channel.next( |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 639 | timeout=STATUS_UPDATE_INTERVAL) |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 640 | except threading_utils.TaskChannel.Timeout: |
| 641 | if print_status_updates: |
Jao-ke Chin-Lee | ba184e6 | 2018-11-19 17:04:41 +0000 | [diff] [blame] | 642 | time_now = str(datetime.datetime.now()) |
| 643 | _, time_now = time_now.split(' ') |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 644 | print( |
Jao-ke Chin-Lee | ba184e6 | 2018-11-19 17:04:41 +0000 | [diff] [blame] | 645 | '%s ' |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 646 | 'Waiting for results from the following shards: %s' % |
Jao-ke Chin-Lee | ba184e6 | 2018-11-19 17:04:41 +0000 | [diff] [blame] | 647 | (time_now, ', '.join(map(str, shards_remaining))) |
| 648 | ) |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 649 | sys.stdout.flush() |
| 650 | continue |
| 651 | except Exception: |
| 652 | logging.exception('Unexpected exception in retrieve_results') |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 653 | |
| 654 | # A call to 'retrieve_results' finished (successfully or not). |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 655 | active_task_count -= 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 656 | if not result: |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 657 | logging.error('Failed to retrieve the results for a swarming key') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 658 | continue |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 659 | |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 660 | # Yield back results to the caller. |
| 661 | assert shard_index in shards_remaining |
| 662 | shards_remaining.remove(shard_index) |
| 663 | yield shard_index, result |
Vadim Shtayura | b19319e | 2014-04-27 08:50:06 -0700 | [diff] [blame] | 664 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 665 | finally: |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 666 | # Done or aborted with Ctrl+C, kill the remaining threads. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 667 | should_stop.set() |
| 668 | |
| 669 | |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 670 | def decorate_shard_output(swarming, shard_index, metadata, include_stdout): |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 671 | """Returns wrapped output for swarming task shard.""" |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 672 | if metadata.get('started_ts') and not metadata.get('deduped_from'): |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 673 | pending = '%.1fs' % ( |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 674 | parse_time(metadata['started_ts']) - parse_time(metadata['created_ts']) |
| 675 | ).total_seconds() |
Marc-Antoine Ruel | 3f9931a | 2017-11-03 14:34:49 -0400 | [diff] [blame] | 676 | elif (metadata.get('state') in ('BOT_DIED', 'CANCELED', 'EXPIRED') and |
| 677 | metadata.get('abandoned_ts')): |
| 678 | pending = '%.1fs' % ( |
| 679 | parse_time(metadata['abandoned_ts']) - |
| 680 | parse_time(metadata['created_ts']) |
| 681 | ).total_seconds() |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 682 | else: |
| 683 | pending = 'N/A' |
| 684 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 685 | if metadata.get('duration') is not None: |
| 686 | duration = '%.1fs' % metadata['duration'] |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 687 | else: |
| 688 | duration = 'N/A' |
| 689 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 690 | if metadata.get('exit_code') is not None: |
| 691 | # Integers are encoded as string to not loose precision. |
| 692 | exit_code = '%s' % metadata['exit_code'] |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 693 | else: |
| 694 | exit_code = 'N/A' |
| 695 | |
| 696 | bot_id = metadata.get('bot_id') or 'N/A' |
| 697 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 698 | url = '%s/user/task/%s' % (swarming, metadata['task_id']) |
Marc-Antoine Ruel | 4e6b73d | 2014-10-03 18:00:05 -0400 | [diff] [blame] | 699 | tag_header = 'Shard %d %s' % (shard_index, url) |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 700 | tag_footer1 = 'End of shard %d' % (shard_index) |
Marc-Antoine Ruel | 3f9931a | 2017-11-03 14:34:49 -0400 | [diff] [blame] | 701 | if metadata.get('state') == 'CANCELED': |
| 702 | tag_footer2 = ' Pending: %s CANCELED' % pending |
| 703 | elif metadata.get('state') == 'EXPIRED': |
| 704 | tag_footer2 = ' Pending: %s EXPIRED (lack of capacity)' % pending |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 705 | elif metadata.get('state') in ('BOT_DIED', 'TIMED_OUT', 'KILLED'): |
Marc-Antoine Ruel | 3f9931a | 2017-11-03 14:34:49 -0400 | [diff] [blame] | 706 | tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s %s' % ( |
| 707 | pending, duration, bot_id, exit_code, metadata['state']) |
| 708 | else: |
| 709 | tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s' % ( |
| 710 | pending, duration, bot_id, exit_code) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 711 | |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 712 | tag_len = max(len(x) for x in [tag_header, tag_footer1, tag_footer2]) |
| 713 | dash_pad = '+-%s-+' % ('-' * tag_len) |
| 714 | tag_header = '| %s |' % tag_header.ljust(tag_len) |
| 715 | tag_footer1 = '| %s |' % tag_footer1.ljust(tag_len) |
| 716 | tag_footer2 = '| %s |' % tag_footer2.ljust(tag_len) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 717 | |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 718 | if include_stdout: |
| 719 | return '\n'.join([ |
| 720 | dash_pad, |
| 721 | tag_header, |
| 722 | dash_pad, |
Marc-Antoine Ruel | 3f9931a | 2017-11-03 14:34:49 -0400 | [diff] [blame] | 723 | (metadata.get('output') or '').rstrip(), |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 724 | dash_pad, |
| 725 | tag_footer1, |
| 726 | tag_footer2, |
| 727 | dash_pad, |
| 728 | ]) |
| 729 | else: |
| 730 | return '\n'.join([ |
| 731 | dash_pad, |
| 732 | tag_header, |
| 733 | tag_footer2, |
| 734 | dash_pad, |
| 735 | ]) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 736 | |
| 737 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 738 | def collect( |
maruel | 0eb1d1b | 2015-10-02 14:48:21 -0700 | [diff] [blame] | 739 | swarming, task_ids, timeout, decorate, print_status_updates, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 740 | task_summary_json, task_output_dir, task_output_stdout, |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 741 | include_perf, filepath_filter): |
maruel | a549078 | 2015-09-30 10:56:59 -0700 | [diff] [blame] | 742 | """Retrieves results of a Swarming task. |
| 743 | |
| 744 | Returns: |
| 745 | process exit code that should be returned to the user. |
| 746 | """ |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 747 | |
| 748 | filter_cb = None |
| 749 | if filepath_filter: |
| 750 | filter_cb = re.compile(filepath_filter).match |
| 751 | |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 752 | # Collect summary JSON and output files (if task_output_dir is not None). |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 753 | output_collector = TaskOutputCollector( |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 754 | task_output_dir, task_output_stdout, len(task_ids), filter_cb) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 755 | |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 756 | seen_shards = set() |
maruel | a549078 | 2015-09-30 10:56:59 -0700 | [diff] [blame] | 757 | exit_code = None |
Marc-Antoine Ruel | d59e807 | 2014-10-21 18:54:45 -0400 | [diff] [blame] | 758 | total_duration = 0 |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 759 | try: |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 760 | for index, metadata in yield_results( |
Marc-Antoine Ruel | 4e6b73d | 2014-10-03 18:00:05 -0400 | [diff] [blame] | 761 | swarming, task_ids, timeout, None, print_status_updates, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 762 | output_collector, include_perf, |
| 763 | (len(task_output_stdout) > 0), |
| 764 | ): |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 765 | seen_shards.add(index) |
Vadim Shtayura | 473455a | 2014-05-14 15:22:35 -0700 | [diff] [blame] | 766 | |
Marc-Antoine Ruel | 5e6ccdb | 2015-04-02 15:55:13 -0400 | [diff] [blame] | 767 | # Default to failure if there was no process that even started. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 768 | shard_exit_code = metadata.get('exit_code') |
| 769 | if shard_exit_code: |
maruel | a549078 | 2015-09-30 10:56:59 -0700 | [diff] [blame] | 770 | # It's encoded as a string, so bool('0') is True. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 771 | shard_exit_code = int(shard_exit_code) |
maruel | a549078 | 2015-09-30 10:56:59 -0700 | [diff] [blame] | 772 | if shard_exit_code or exit_code is None: |
Marc-Antoine Ruel | 4e6b73d | 2014-10-03 18:00:05 -0400 | [diff] [blame] | 773 | exit_code = shard_exit_code |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 774 | total_duration += metadata.get('duration', 0) |
Vadim Shtayura | 473455a | 2014-05-14 15:22:35 -0700 | [diff] [blame] | 775 | |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 776 | if decorate: |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 777 | s = decorate_shard_output( |
| 778 | swarming, index, metadata, |
| 779 | "console" in task_output_stdout).encode( |
| 780 | 'utf-8', 'replace') |
leilei | ed18176 | 2016-10-13 14:24:59 -0700 | [diff] [blame] | 781 | print(s) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 782 | if len(seen_shards) < len(task_ids): |
| 783 | print('') |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 784 | else: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 785 | print('%s: %s %s' % ( |
| 786 | metadata.get('bot_id', 'N/A'), |
| 787 | metadata['task_id'], |
| 788 | shard_exit_code)) |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 789 | if "console" in task_output_stdout and metadata['output']: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 790 | output = metadata['output'].rstrip() |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 791 | if output: |
| 792 | print(''.join(' %s\n' % l for l in output.splitlines())) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 793 | finally: |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 794 | summary = output_collector.finalize() |
| 795 | if task_summary_json: |
| 796 | tools.write_json(task_summary_json, summary, False) |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 797 | |
Marc-Antoine Ruel | d59e807 | 2014-10-21 18:54:45 -0400 | [diff] [blame] | 798 | if decorate and total_duration: |
| 799 | print('Total duration: %.1fs' % total_duration) |
| 800 | |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 801 | if len(seen_shards) != len(task_ids): |
| 802 | missing_shards = [x for x in range(len(task_ids)) if x not in seen_shards] |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 803 | print >> sys.stderr, ('Results from some shards are missing: %s' % |
| 804 | ', '.join(map(str, missing_shards))) |
Vadim Shtayura | c524f51 | 2014-05-15 09:54:56 -0700 | [diff] [blame] | 805 | return 1 |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 806 | |
maruel | a549078 | 2015-09-30 10:56:59 -0700 | [diff] [blame] | 807 | return exit_code if exit_code is not None else 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 808 | |
| 809 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 810 | ### API management. |
| 811 | |
| 812 | |
| 813 | class APIError(Exception): |
| 814 | pass |
| 815 | |
| 816 | |
| 817 | def endpoints_api_discovery_apis(host): |
| 818 | """Uses Cloud Endpoints' API Discovery Service to returns metadata about all |
| 819 | the APIs exposed by a host. |
| 820 | |
| 821 | https://developers.google.com/discovery/v1/reference/apis/list |
| 822 | """ |
maruel | 380e326 | 2016-08-31 16:10:06 -0700 | [diff] [blame] | 823 | # Uses the real Cloud Endpoints. This needs to be fixed once the Cloud |
| 824 | # Endpoints version is turned down. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 825 | data = net.url_read_json(host + '/_ah/api/discovery/v1/apis') |
| 826 | if data is None: |
| 827 | raise APIError('Failed to discover APIs on %s' % host) |
| 828 | out = {} |
| 829 | for api in data['items']: |
| 830 | if api['id'] == 'discovery:v1': |
| 831 | continue |
| 832 | # URL is of the following form: |
| 833 | # url = host + ( |
| 834 | # '/_ah/api/discovery/v1/apis/%s/%s/rest' % (api['id'], api['version']) |
| 835 | api_data = net.url_read_json(api['discoveryRestUrl']) |
| 836 | if api_data is None: |
| 837 | raise APIError('Failed to discover %s on %s' % (api['id'], host)) |
| 838 | out[api['id']] = api_data |
| 839 | return out |
| 840 | |
| 841 | |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 842 | def get_yielder(base_url, limit): |
| 843 | """Returns the first query and a function that yields following items.""" |
| 844 | CHUNK_SIZE = 250 |
| 845 | |
| 846 | url = base_url |
| 847 | if limit: |
| 848 | url += '%slimit=%d' % ('&' if '?' in url else '?', min(CHUNK_SIZE, limit)) |
| 849 | data = net.url_read_json(url) |
| 850 | if data is None: |
| 851 | # TODO(maruel): Do basic diagnostic. |
| 852 | raise Failure('Failed to access %s' % url) |
| 853 | org_cursor = data.pop('cursor', None) |
| 854 | org_total = len(data.get('items') or []) |
| 855 | logging.info('get_yielder(%s) returning %d items', base_url, org_total) |
| 856 | if not org_cursor or not org_total: |
| 857 | # This is not an iterable resource. |
| 858 | return data, lambda: [] |
| 859 | |
| 860 | def yielder(): |
| 861 | cursor = org_cursor |
| 862 | total = org_total |
| 863 | # Some items support cursors. Try to get automatically if cursors are needed |
| 864 | # by looking at the 'cursor' items. |
| 865 | while cursor and (not limit or total < limit): |
| 866 | merge_char = '&' if '?' in base_url else '?' |
| 867 | url = base_url + '%scursor=%s' % (merge_char, urllib.quote(cursor)) |
| 868 | if limit: |
| 869 | url += '&limit=%d' % min(CHUNK_SIZE, limit - total) |
| 870 | new = net.url_read_json(url) |
| 871 | if new is None: |
| 872 | raise Failure('Failed to access %s' % url) |
| 873 | cursor = new.get('cursor') |
| 874 | new_items = new.get('items') |
| 875 | nb_items = len(new_items or []) |
| 876 | total += nb_items |
| 877 | logging.info('get_yielder(%s) yielding %d items', base_url, nb_items) |
| 878 | yield new_items |
| 879 | |
| 880 | return data, yielder |
| 881 | |
| 882 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 883 | ### Commands. |
| 884 | |
| 885 | |
| 886 | def abort_task(_swarming, _manifest): |
| 887 | """Given a task manifest that was triggered, aborts its execution.""" |
| 888 | # TODO(vadimsh): No supported by the server yet. |
| 889 | |
| 890 | |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 891 | def add_filter_options(parser): |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 892 | parser.filter_group = optparse.OptionGroup(parser, 'Bot selection') |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 893 | parser.filter_group.add_option( |
Marc-Antoine Ruel | b39e8cf | 2014-01-20 10:39:31 -0500 | [diff] [blame] | 894 | '-d', '--dimension', default=[], action='append', nargs=2, |
Marc-Antoine Ruel | 92f3242 | 2013-11-06 18:12:13 -0500 | [diff] [blame] | 895 | dest='dimensions', metavar='FOO bar', |
| 896 | help='dimension to filter on') |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 897 | parser.filter_group.add_option( |
| 898 | '--optional-dimension', default=[], action='append', nargs=3, |
| 899 | dest='optional_dimensions', metavar='key value expiration', |
| 900 | help='optional dimensions which will result in additional task slices ') |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 901 | parser.add_option_group(parser.filter_group) |
| 902 | |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 903 | |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 904 | def _validate_filter_option(parser, key, value, expiration, argname): |
| 905 | if ':' in key: |
| 906 | parser.error('%s key cannot contain ":"' % argname) |
| 907 | if key.strip() != key: |
| 908 | parser.error('%s key has whitespace' % argname) |
| 909 | if not key: |
| 910 | parser.error('%s key is empty' % argname) |
| 911 | |
| 912 | if value.strip() != value: |
| 913 | parser.error('%s value has whitespace' % argname) |
| 914 | if not value: |
| 915 | parser.error('%s value is empty' % argname) |
| 916 | |
| 917 | if expiration is not None: |
| 918 | try: |
| 919 | expiration = int(expiration) |
| 920 | except ValueError: |
| 921 | parser.error('%s expiration is not an integer' % argname) |
| 922 | if expiration <= 0: |
| 923 | parser.error('%s expiration should be positive' % argname) |
| 924 | if expiration % 60 != 0: |
| 925 | parser.error('%s expiration is not divisible by 60' % argname) |
| 926 | |
| 927 | |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 928 | def process_filter_options(parser, options): |
| 929 | for key, value in options.dimensions: |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 930 | _validate_filter_option(parser, key, value, None, 'dimension') |
| 931 | for key, value, exp in options.optional_dimensions: |
| 932 | _validate_filter_option(parser, key, value, exp, 'optional-dimension') |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 933 | options.dimensions.sort() |
| 934 | |
| 935 | |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 936 | def add_sharding_options(parser): |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 937 | parser.sharding_group = optparse.OptionGroup(parser, 'Sharding options') |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 938 | parser.sharding_group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 939 | '--shards', type='int', default=1, metavar='NUMBER', |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 940 | help='Number of shards to trigger and collect.') |
| 941 | parser.add_option_group(parser.sharding_group) |
| 942 | |
| 943 | |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 944 | def add_trigger_options(parser): |
| 945 | """Adds all options to trigger a task on Swarming.""" |
Marc-Antoine Ruel | f7d737d | 2014-12-10 15:36:29 -0500 | [diff] [blame] | 946 | isolateserver.add_isolate_server_options(parser) |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 947 | add_filter_options(parser) |
| 948 | |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 949 | group = optparse.OptionGroup(parser, 'TaskSlice properties') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 950 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 951 | '-s', '--isolated', metavar='HASH', |
Marc-Antoine Ruel | 185ded4 | 2015-01-28 20:49:18 -0500 | [diff] [blame] | 952 | help='Hash of the .isolated to grab from the isolate server') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 953 | group.add_option( |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 954 | '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar', |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 955 | help='Environment variables to set') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 956 | group.add_option( |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 957 | '--env-prefix', default=[], action='append', nargs=2, |
| 958 | metavar='VAR local/path', |
| 959 | help='Prepend task-relative `local/path` to the task\'s VAR environment ' |
| 960 | 'variable using os-appropriate pathsep character. Can be specified ' |
| 961 | 'multiple times for the same VAR to add multiple paths.') |
| 962 | group.add_option( |
Marc-Antoine Ruel | 0219639 | 2014-10-17 16:29:43 -0400 | [diff] [blame] | 963 | '--idempotent', action='store_true', default=False, |
| 964 | help='When set, the server will actively try to find a previous task ' |
| 965 | 'with the same parameter and return this result instead if possible') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 966 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 967 | '--secret-bytes-path', metavar='FILE', |
Stephen Martinis | f391c77 | 2019-02-01 01:22:12 +0000 | [diff] [blame] | 968 | help='The optional path to a file containing the secret_bytes to use ' |
| 969 | 'with this task.') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 970 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 971 | '--hard-timeout', type='int', default=60*60, metavar='SECS', |
Marc-Antoine Ruel | 2f6581a | 2014-10-03 11:09:53 -0400 | [diff] [blame] | 972 | help='Seconds to allow the task to complete.') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 973 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 974 | '--io-timeout', type='int', default=20*60, metavar='SECS', |
Marc-Antoine Ruel | 2f6581a | 2014-10-03 11:09:53 -0400 | [diff] [blame] | 975 | help='Seconds to allow the task to be silent.') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 976 | group.add_option( |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 977 | '--raw-cmd', action='store_true', default=False, |
| 978 | help='When set, the command after -- is used as-is without run_isolated. ' |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 979 | 'In this case, the .isolated file is expected to not have a command') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 980 | group.add_option( |
Marc-Antoine Ruel | ba1bf22 | 2017-12-21 21:41:01 -0500 | [diff] [blame] | 981 | '--relative-cwd', |
| 982 | help='Ignore the isolated \'relative_cwd\' and use this one instead; ' |
| 983 | 'requires --raw-cmd') |
| 984 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 985 | '--cipd-package', action='append', default=[], metavar='PKG', |
| 986 | help='CIPD packages to install on the Swarming bot. Uses the format: ' |
borenet | 02f772b | 2016-06-22 12:42:19 -0700 | [diff] [blame] | 987 | 'path:package_name:version') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 988 | group.add_option( |
| 989 | '--named-cache', action='append', nargs=2, default=[], |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 990 | metavar='NAME RELPATH', |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 991 | help='"<name> <relpath>" items to keep a persistent bot managed cache') |
| 992 | group.add_option( |
vadimsh | 93d167c | 2016-09-13 11:31:51 -0700 | [diff] [blame] | 993 | '--service-account', |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 994 | help='Email of a service account to run the task as, or literal "bot" ' |
| 995 | 'string to indicate that the task should use the same account the ' |
| 996 | 'bot itself is using to authenticate to Swarming. Don\'t use task ' |
| 997 | 'service accounts if not given (default).') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 998 | group.add_option( |
Robert Iannucci | fafa735 | 2018-06-13 17:08:17 +0000 | [diff] [blame] | 999 | '--pool-task-template', |
| 1000 | choices=('AUTO', 'CANARY_PREFER', 'CANARY_NEVER', 'SKIP'), |
| 1001 | default='AUTO', |
| 1002 | help='Set how you want swarming to apply the pool\'s TaskTemplate. ' |
| 1003 | 'By default, the pool\'s TaskTemplate is automatically selected, ' |
| 1004 | 'according the pool configuration on the server. Choices are: ' |
| 1005 | 'AUTO, CANARY_PREFER, CANARY_NEVER, and SKIP (default: AUTO).') |
| 1006 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 1007 | '-o', '--output', action='append', default=[], metavar='PATH', |
| 1008 | help='A list of files to return in addition to those written to ' |
| 1009 | '${ISOLATED_OUTDIR}. An error will occur if a file specified by' |
| 1010 | 'this option is also written directly to ${ISOLATED_OUTDIR}.') |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 1011 | group.add_option( |
| 1012 | '--wait-for-capacity', action='store_true', default=False, |
| 1013 | help='Instructs to leave the task PENDING even if there\'s no known bot ' |
| 1014 | 'that could run this task, otherwise the task will be denied with ' |
| 1015 | 'NO_RESOURCE') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1016 | parser.add_option_group(group) |
| 1017 | |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 1018 | group = optparse.OptionGroup(parser, 'TaskRequest details') |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1019 | group.add_option( |
Marc-Antoine Ruel | 486c9b5 | 2018-07-23 19:30:47 +0000 | [diff] [blame] | 1020 | '--priority', type='int', default=200, |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1021 | help='The lower value, the more important the task is') |
| 1022 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 1023 | '-T', '--task-name', metavar='NAME', |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1024 | help='Display name of the task. Defaults to ' |
| 1025 | '<base_name>/<dimensions>/<isolated hash>/<timestamp> if an ' |
| 1026 | 'isolated file is provided, if a hash is provided, it defaults to ' |
| 1027 | '<user>/<dimensions>/<isolated hash>/<timestamp>') |
| 1028 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 1029 | '--tags', action='append', default=[], metavar='FOO:BAR', |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1030 | help='Tags to assign to the task.') |
| 1031 | group.add_option( |
| 1032 | '--user', default='', |
| 1033 | help='User associated with the task. Defaults to authenticated user on ' |
| 1034 | 'the server.') |
| 1035 | group.add_option( |
maruel | 5475ba6 | 2017-05-31 15:35:47 -0700 | [diff] [blame] | 1036 | '--expiration', type='int', default=6*60*60, metavar='SECS', |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1037 | help='Seconds to allow the task to be pending for a bot to run before ' |
| 1038 | 'this task request expires.') |
| 1039 | group.add_option( |
| 1040 | '--deadline', type='int', dest='expiration', |
| 1041 | help=optparse.SUPPRESS_HELP) |
| 1042 | parser.add_option_group(group) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1043 | |
| 1044 | |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1045 | def process_trigger_options(parser, options, args): |
Marc-Antoine Ruel | 3a030bc | 2018-04-23 10:31:25 -0400 | [diff] [blame] | 1046 | """Processes trigger options and does preparatory steps. |
| 1047 | |
| 1048 | Returns: |
| 1049 | NewTaskRequest instance. |
| 1050 | """ |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1051 | process_filter_options(parser, options) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1052 | options.env = dict(options.env) |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1053 | if args and args[0] == '--': |
| 1054 | args = args[1:] |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1055 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1056 | if not options.dimensions: |
| 1057 | parser.error('Please at least specify one --dimension') |
Marc-Antoine Ruel | 33d198c | 2018-11-27 21:12:16 +0000 | [diff] [blame] | 1058 | if not any(k == 'pool' for k, _v in options.dimensions): |
| 1059 | parser.error('You must specify --dimension pool <value>') |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1060 | if not all(len(t.split(':', 1)) == 2 for t in options.tags): |
| 1061 | parser.error('--tags must be in the format key:value') |
| 1062 | if options.raw_cmd and not args: |
| 1063 | parser.error( |
| 1064 | 'Arguments with --raw-cmd should be passed after -- as command ' |
| 1065 | 'delimiter.') |
| 1066 | if options.isolate_server and not options.namespace: |
| 1067 | parser.error( |
| 1068 | '--namespace must be a valid value when --isolate-server is used') |
| 1069 | if not options.isolated and not options.raw_cmd: |
| 1070 | parser.error('Specify at least one of --raw-cmd or --isolated or both') |
| 1071 | |
| 1072 | # Isolated |
| 1073 | # --isolated is required only if --raw-cmd wasn't provided. |
| 1074 | # TODO(maruel): --isolate-server may be optional as Swarming may have its own |
| 1075 | # preferred server. |
| 1076 | isolateserver.process_isolate_server_options( |
| 1077 | parser, options, False, not options.raw_cmd) |
| 1078 | inputs_ref = None |
| 1079 | if options.isolate_server: |
| 1080 | inputs_ref = FilesRef( |
| 1081 | isolated=options.isolated, |
| 1082 | isolatedserver=options.isolate_server, |
| 1083 | namespace=options.namespace) |
| 1084 | |
| 1085 | # Command |
| 1086 | command = None |
| 1087 | extra_args = None |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1088 | if options.raw_cmd: |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1089 | command = args |
Marc-Antoine Ruel | ba1bf22 | 2017-12-21 21:41:01 -0500 | [diff] [blame] | 1090 | if options.relative_cwd: |
| 1091 | a = os.path.normpath(os.path.abspath(options.relative_cwd)) |
| 1092 | if not a.startswith(os.getcwd()): |
| 1093 | parser.error( |
| 1094 | '--relative-cwd must not try to escape the working directory') |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1095 | else: |
Marc-Antoine Ruel | ba1bf22 | 2017-12-21 21:41:01 -0500 | [diff] [blame] | 1096 | if options.relative_cwd: |
| 1097 | parser.error('--relative-cwd requires --raw-cmd') |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1098 | extra_args = args |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1099 | |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1100 | # CIPD |
borenet | 02f772b | 2016-06-22 12:42:19 -0700 | [diff] [blame] | 1101 | cipd_packages = [] |
| 1102 | for p in options.cipd_package: |
| 1103 | split = p.split(':', 2) |
| 1104 | if len(split) != 3: |
| 1105 | parser.error('CIPD packages must take the form: path:package:version') |
| 1106 | cipd_packages.append(CipdPackage( |
| 1107 | package_name=split[1], |
| 1108 | path=split[0], |
| 1109 | version=split[2])) |
| 1110 | cipd_input = None |
| 1111 | if cipd_packages: |
| 1112 | cipd_input = CipdInput( |
| 1113 | client_package=None, |
| 1114 | packages=cipd_packages, |
| 1115 | server=None) |
| 1116 | |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1117 | # Secrets |
iannucci | dc80dfb | 2016-10-28 12:50:20 -0700 | [diff] [blame] | 1118 | secret_bytes = None |
| 1119 | if options.secret_bytes_path: |
Marc-Antoine Ruel | 5c98fa7 | 2018-05-18 12:19:59 -0400 | [diff] [blame] | 1120 | with open(options.secret_bytes_path, 'rb') as f: |
iannucci | dc80dfb | 2016-10-28 12:50:20 -0700 | [diff] [blame] | 1121 | secret_bytes = f.read().encode('base64') |
| 1122 | |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1123 | # Named caches |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1124 | caches = [ |
| 1125 | {u'name': unicode(i[0]), u'path': unicode(i[1])} |
| 1126 | for i in options.named_cache |
| 1127 | ] |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1128 | |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 1129 | env_prefixes = {} |
| 1130 | for k, v in options.env_prefix: |
| 1131 | env_prefixes.setdefault(k, []).append(v) |
| 1132 | |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1133 | # Get dimensions into the key/value format we can manipulate later. |
| 1134 | orig_dims = [ |
| 1135 | {'key': key, 'value': value} for key, value in options.dimensions] |
| 1136 | orig_dims.sort(key=lambda x: (x['key'], x['value'])) |
| 1137 | |
| 1138 | # Construct base properties that we will use for all the slices, adding in |
| 1139 | # optional dimensions for the fallback slices. |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1140 | properties = TaskProperties( |
maruel | 681d680 | 2017-01-17 16:56:03 -0800 | [diff] [blame] | 1141 | caches=caches, |
borenet | 02f772b | 2016-06-22 12:42:19 -0700 | [diff] [blame] | 1142 | cipd_input=cipd_input, |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1143 | command=command, |
Marc-Antoine Ruel | ba1bf22 | 2017-12-21 21:41:01 -0500 | [diff] [blame] | 1144 | relative_cwd=options.relative_cwd, |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1145 | dimensions=orig_dims, |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1146 | env=options.env, |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 1147 | env_prefixes=[StringListPair(k, v) for k, v in env_prefixes.iteritems()], |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1148 | execution_timeout_secs=options.hard_timeout, |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1149 | extra_args=extra_args, |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1150 | grace_period_secs=30, |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1151 | idempotent=options.idempotent, |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1152 | inputs_ref=inputs_ref, |
aludwin | cc5524e | 2016-10-28 10:25:24 -0700 | [diff] [blame] | 1153 | io_timeout_secs=options.io_timeout, |
iannucci | dc80dfb | 2016-10-28 12:50:20 -0700 | [diff] [blame] | 1154 | outputs=options.output, |
| 1155 | secret_bytes=secret_bytes) |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1156 | |
| 1157 | slices = [] |
| 1158 | |
| 1159 | # Group the optional dimensions by expiration. |
| 1160 | dims_by_exp = {} |
| 1161 | for key, value, exp_secs in options.optional_dimensions: |
| 1162 | dims_by_exp.setdefault(int(exp_secs), []).append( |
| 1163 | {'key': key, 'value': value}) |
| 1164 | |
| 1165 | # Create the optional slices with expiration deltas, we fix up the properties |
| 1166 | # below. |
| 1167 | last_exp = 0 |
| 1168 | for expiration_secs in sorted(dims_by_exp): |
| 1169 | t = TaskSlice( |
| 1170 | expiration_secs=expiration_secs - last_exp, |
| 1171 | properties=properties, |
| 1172 | wait_for_capacity=False) |
| 1173 | slices.append(t) |
| 1174 | last_exp = expiration_secs |
| 1175 | |
| 1176 | # Add back in the default slice (the last one). |
| 1177 | exp = max(int(options.expiration) - last_exp, 60) |
| 1178 | base_task_slice = TaskSlice( |
| 1179 | expiration_secs=exp, |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 1180 | properties=properties, |
| 1181 | wait_for_capacity=options.wait_for_capacity) |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1182 | slices.append(base_task_slice) |
| 1183 | |
Brad Hall | 7f463e6 | 2018-11-16 16:13:30 +0000 | [diff] [blame] | 1184 | # Add optional dimensions to the task slices, replacing a dimension that |
| 1185 | # has the same key if it is a dimension where repeating isn't valid (otherwise |
| 1186 | # we append it). Currently the only dimension we can repeat is "caches"; the |
| 1187 | # rest (os, cpu, etc) shouldn't be repeated. |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1188 | extra_dims = [] |
Brad Hall | 7f463e6 | 2018-11-16 16:13:30 +0000 | [diff] [blame] | 1189 | for i, (_, kvs) in enumerate(sorted(dims_by_exp.iteritems(), reverse=True)): |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1190 | dims = list(orig_dims) |
Brad Hall | 7f463e6 | 2018-11-16 16:13:30 +0000 | [diff] [blame] | 1191 | # Replace or append the key/value pairs for this expiration in extra_dims; |
| 1192 | # we keep extra_dims around because we are iterating backwards and filling |
| 1193 | # in slices with shorter expirations. Dimensions expire as time goes on so |
| 1194 | # the slices that expire earlier will generally have more dimensions. |
| 1195 | for kv in kvs: |
| 1196 | if kv['key'] == 'caches': |
| 1197 | extra_dims.append(kv) |
| 1198 | else: |
| 1199 | extra_dims = [x for x in extra_dims if x['key'] != kv['key']] + [kv] |
| 1200 | # Then, add all the optional dimensions to the original dimension set, again |
| 1201 | # replacing if needed. |
| 1202 | for kv in extra_dims: |
| 1203 | if kv['key'] == 'caches': |
| 1204 | dims.append(kv) |
| 1205 | else: |
| 1206 | dims = [x for x in dims if x['key'] != kv['key']] + [kv] |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1207 | dims.sort(key=lambda x: (x['key'], x['value'])) |
| 1208 | slice_properties = properties._replace(dimensions=dims) |
| 1209 | slices[-2 - i] = slices[-2 - i]._replace(properties=slice_properties) |
| 1210 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1211 | return NewTaskRequest( |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1212 | name=default_task_name(options), |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1213 | parent_task_id=os.environ.get('SWARMING_TASK_ID', ''), |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1214 | priority=options.priority, |
Brad Hall | f78187a | 2018-10-19 17:08:55 +0000 | [diff] [blame] | 1215 | task_slices=slices, |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 1216 | service_account=options.service_account, |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1217 | tags=options.tags, |
Robert Iannucci | fafa735 | 2018-06-13 17:08:17 +0000 | [diff] [blame] | 1218 | user=options.user, |
| 1219 | pool_task_template=options.pool_task_template) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1220 | |
| 1221 | |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1222 | class TaskOutputStdoutOption(optparse.Option): |
| 1223 | """Where to output the each task's console output (stderr/stdout). |
| 1224 | |
| 1225 | The output will be; |
| 1226 | none - not be downloaded. |
| 1227 | json - stored in summary.json file *only*. |
| 1228 | console - shown on stdout *only*. |
| 1229 | all - stored in summary.json and shown on stdout. |
| 1230 | """ |
| 1231 | |
| 1232 | choices = ['all', 'json', 'console', 'none'] |
| 1233 | |
| 1234 | def __init__(self, *args, **kw): |
| 1235 | optparse.Option.__init__( |
| 1236 | self, |
| 1237 | *args, |
| 1238 | choices=self.choices, |
Marc-Antoine Ruel | 2848884 | 2017-09-12 18:09:17 -0400 | [diff] [blame] | 1239 | default=['console', 'json'], |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1240 | help=re.sub('\s\s*', ' ', self.__doc__), |
| 1241 | **kw) |
| 1242 | |
| 1243 | def convert_value(self, opt, value): |
| 1244 | if value not in self.choices: |
| 1245 | raise optparse.OptionValueError("%s must be one of %s not %r" % ( |
| 1246 | self.get_opt_string(), self.choices, value)) |
| 1247 | stdout_to = [] |
| 1248 | if value == 'all': |
Marc-Antoine Ruel | 2848884 | 2017-09-12 18:09:17 -0400 | [diff] [blame] | 1249 | stdout_to = ['console', 'json'] |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1250 | elif value != 'none': |
| 1251 | stdout_to = [value] |
| 1252 | return stdout_to |
| 1253 | |
| 1254 | |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1255 | def add_collect_options(parser): |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 1256 | parser.server_group.add_option( |
Marc-Antoine Ruel | e831f05 | 2018-04-20 15:01:03 -0400 | [diff] [blame] | 1257 | '-t', '--timeout', type='float', default=0., |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1258 | help='Timeout to wait for result, set to -1 for no timeout and get ' |
| 1259 | 'current state; defaults to waiting until the task completes') |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 1260 | parser.group_logging.add_option( |
| 1261 | '--decorate', action='store_true', help='Decorate output') |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 1262 | parser.group_logging.add_option( |
| 1263 | '--print-status-updates', action='store_true', |
| 1264 | help='Print periodic status updates') |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 1265 | parser.task_output_group = optparse.OptionGroup(parser, 'Task output') |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 1266 | parser.task_output_group.add_option( |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 1267 | '--task-summary-json', |
| 1268 | metavar='FILE', |
| 1269 | help='Dump a summary of task results to this file as json. It contains ' |
| 1270 | 'only shards statuses as know to server directly. Any output files ' |
| 1271 | 'emitted by the task can be collected by using --task-output-dir') |
| 1272 | parser.task_output_group.add_option( |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 1273 | '--task-output-dir', |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 1274 | metavar='DIR', |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 1275 | help='Directory to put task results into. When the task finishes, this ' |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 1276 | 'directory contains per-shard directory with output files produced ' |
| 1277 | 'by shards: <task-output-dir>/<zero-based-shard-index>/.') |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1278 | parser.task_output_group.add_option(TaskOutputStdoutOption( |
Marc-Antoine Ruel | 2848884 | 2017-09-12 18:09:17 -0400 | [diff] [blame] | 1279 | '--task-output-stdout')) |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 1280 | parser.task_output_group.add_option( |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 1281 | '--filepath-filter', |
| 1282 | help='This is regexp filter used to specify downloaded filepath when ' |
| 1283 | 'collecting isolated output.') |
| 1284 | parser.task_output_group.add_option( |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 1285 | '--perf', action='store_true', default=False, |
| 1286 | help='Includes performance statistics') |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 1287 | parser.add_option_group(parser.task_output_group) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1288 | |
| 1289 | |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1290 | def process_collect_options(parser, options): |
| 1291 | # Only negative -1 is allowed, disallow other negative values. |
| 1292 | if options.timeout != -1 and options.timeout < 0: |
| 1293 | parser.error('Invalid --timeout value') |
| 1294 | |
| 1295 | |
Marc-Antoine Ruel | 13e7c88 | 2015-03-26 18:19:10 -0400 | [diff] [blame] | 1296 | @subcommand.usage('bots...') |
| 1297 | def CMDbot_delete(parser, args): |
| 1298 | """Forcibly deletes bots from the Swarming server.""" |
| 1299 | parser.add_option( |
| 1300 | '-f', '--force', action='store_true', |
| 1301 | help='Do not prompt for confirmation') |
| 1302 | options, args = parser.parse_args(args) |
| 1303 | if not args: |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1304 | parser.error('Please specify bots to delete') |
Marc-Antoine Ruel | 13e7c88 | 2015-03-26 18:19:10 -0400 | [diff] [blame] | 1305 | |
| 1306 | bots = sorted(args) |
| 1307 | if not options.force: |
| 1308 | print('Delete the following bots?') |
| 1309 | for bot in bots: |
| 1310 | print(' %s' % bot) |
| 1311 | if raw_input('Continue? [y/N] ') not in ('y', 'Y'): |
| 1312 | print('Goodbye.') |
| 1313 | return 1 |
| 1314 | |
| 1315 | result = 0 |
| 1316 | for bot in bots: |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1317 | url = '%s/_ah/api/swarming/v1/bot/%s/delete' % (options.swarming, bot) |
vadimsh | e4c0e24 | 2015-09-30 11:53:54 -0700 | [diff] [blame] | 1318 | if net.url_read_json(url, data={}, method='POST') is None: |
| 1319 | print('Deleting %s failed. Probably already gone' % bot) |
Marc-Antoine Ruel | 13e7c88 | 2015-03-26 18:19:10 -0400 | [diff] [blame] | 1320 | result = 1 |
| 1321 | return result |
| 1322 | |
| 1323 | |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1324 | def CMDbots(parser, args): |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 1325 | """Returns information about the bots connected to the Swarming server.""" |
| 1326 | add_filter_options(parser) |
| 1327 | parser.filter_group.add_option( |
Marc-Antoine Ruel | 2808311 | 2014-03-13 16:34:04 -0400 | [diff] [blame] | 1328 | '--dead-only', action='store_true', |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1329 | help='Filter out bots alive, useful to reap them and reimage broken bots') |
Marc-Antoine Ruel | 2808311 | 2014-03-13 16:34:04 -0400 | [diff] [blame] | 1330 | parser.filter_group.add_option( |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 1331 | '-k', '--keep-dead', action='store_true', |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1332 | help='Keep both dead and alive bots') |
| 1333 | parser.filter_group.add_option( |
| 1334 | '--busy', action='store_true', help='Keep only busy bots') |
| 1335 | parser.filter_group.add_option( |
| 1336 | '--idle', action='store_true', help='Keep only idle bots') |
| 1337 | parser.filter_group.add_option( |
| 1338 | '--mp', action='store_true', |
| 1339 | help='Keep only Machine Provider managed bots') |
| 1340 | parser.filter_group.add_option( |
| 1341 | '--non-mp', action='store_true', |
| 1342 | help='Keep only non Machine Provider managed bots') |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 1343 | parser.filter_group.add_option( |
| 1344 | '-b', '--bare', action='store_true', |
Marc-Antoine Ruel | e7b0016 | 2014-03-12 16:59:01 -0400 | [diff] [blame] | 1345 | help='Do not print out dimensions') |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 1346 | options, args = parser.parse_args(args) |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1347 | process_filter_options(parser, options) |
Marc-Antoine Ruel | 2808311 | 2014-03-13 16:34:04 -0400 | [diff] [blame] | 1348 | |
| 1349 | if options.keep_dead and options.dead_only: |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1350 | parser.error('Use only one of --keep-dead or --dead-only') |
| 1351 | if options.busy and options.idle: |
| 1352 | parser.error('Use only one of --busy or --idle') |
| 1353 | if options.mp and options.non_mp: |
| 1354 | parser.error('Use only one of --mp or --non-mp') |
Vadim Shtayura | 6b555c1 | 2014-07-23 16:22:18 -0700 | [diff] [blame] | 1355 | |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1356 | url = options.swarming + '/_ah/api/swarming/v1/bots/list?' |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1357 | values = [] |
| 1358 | if options.dead_only: |
| 1359 | values.append(('is_dead', 'TRUE')) |
| 1360 | elif options.keep_dead: |
| 1361 | values.append(('is_dead', 'NONE')) |
| 1362 | else: |
| 1363 | values.append(('is_dead', 'FALSE')) |
Marc-Antoine Ruel | c6c579e | 2014-09-08 18:43:45 -0400 | [diff] [blame] | 1364 | |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1365 | if options.busy: |
| 1366 | values.append(('is_busy', 'TRUE')) |
| 1367 | elif options.idle: |
| 1368 | values.append(('is_busy', 'FALSE')) |
| 1369 | else: |
| 1370 | values.append(('is_busy', 'NONE')) |
| 1371 | |
| 1372 | if options.mp: |
| 1373 | values.append(('is_mp', 'TRUE')) |
| 1374 | elif options.non_mp: |
| 1375 | values.append(('is_mp', 'FALSE')) |
| 1376 | else: |
| 1377 | values.append(('is_mp', 'NONE')) |
| 1378 | |
| 1379 | for key, value in options.dimensions: |
| 1380 | values.append(('dimensions', '%s:%s' % (key, value))) |
| 1381 | url += urllib.urlencode(values) |
| 1382 | try: |
| 1383 | data, yielder = get_yielder(url, 0) |
| 1384 | bots = data.get('items') or [] |
| 1385 | for items in yielder(): |
| 1386 | if items: |
| 1387 | bots.extend(items) |
| 1388 | except Failure as e: |
| 1389 | sys.stderr.write('\n%s\n' % e) |
| 1390 | return 1 |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1391 | for bot in natsort.natsorted(bots, key=lambda x: x['bot_id']): |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1392 | print bot['bot_id'] |
| 1393 | if not options.bare: |
| 1394 | dimensions = {i['key']: i.get('value') for i in bot.get('dimensions', {})} |
| 1395 | print ' %s' % json.dumps(dimensions, sort_keys=True) |
| 1396 | if bot.get('task_id'): |
| 1397 | print ' task: %s' % bot['task_id'] |
Marc-Antoine Ruel | 819fb16 | 2014-03-12 16:38:26 -0400 | [diff] [blame] | 1398 | return 0 |
| 1399 | |
| 1400 | |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1401 | @subcommand.usage('task_id') |
| 1402 | def CMDcancel(parser, args): |
| 1403 | """Cancels a task.""" |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 1404 | parser.add_option( |
| 1405 | '-k', '--kill-running', action='store_true', default=False, |
| 1406 | help='Kill the task even if it was running') |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1407 | options, args = parser.parse_args(args) |
| 1408 | if not args: |
| 1409 | parser.error('Please specify the task to cancel') |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 1410 | data = {'kill_running': options.kill_running} |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1411 | for task_id in args: |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1412 | url = '%s/_ah/api/swarming/v1/task/%s/cancel' % (options.swarming, task_id) |
Marc-Antoine Ruel | 2e52c55 | 2018-03-26 19:27:36 -0400 | [diff] [blame] | 1413 | resp = net.url_read_json(url, data=data, method='POST') |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1414 | if resp is None: |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1415 | print('Deleting %s failed. Probably already gone' % task_id) |
| 1416 | return 1 |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1417 | logging.info('%s', resp) |
maruel | fd0a90c | 2016-06-10 11:51:10 -0700 | [diff] [blame] | 1418 | return 0 |
| 1419 | |
| 1420 | |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1421 | @subcommand.usage('--json file | task_id...') |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1422 | def CMDcollect(parser, args): |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1423 | """Retrieves results of one or multiple Swarming task by its ID. |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1424 | |
| 1425 | The result can be in multiple part if the execution was sharded. It can |
| 1426 | potentially have retries. |
| 1427 | """ |
| 1428 | add_collect_options(parser) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1429 | parser.add_option( |
| 1430 | '-j', '--json', |
| 1431 | help='Load the task ids from .json as saved by trigger --dump-json') |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1432 | options, args = parser.parse_args(args) |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1433 | process_collect_options(parser, options) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1434 | if not args and not options.json: |
| 1435 | parser.error('Must specify at least one task id or --json.') |
| 1436 | if args and options.json: |
| 1437 | parser.error('Only use one of task id or --json.') |
| 1438 | |
| 1439 | if options.json: |
maruel | 1ceb387 | 2015-10-14 06:10:44 -0700 | [diff] [blame] | 1440 | options.json = unicode(os.path.abspath(options.json)) |
Marc-Antoine Ruel | 9025a78 | 2015-03-17 16:42:59 -0400 | [diff] [blame] | 1441 | try: |
maruel | 1ceb387 | 2015-10-14 06:10:44 -0700 | [diff] [blame] | 1442 | with fs.open(options.json, 'rb') as f: |
maruel | 71c61c8 | 2016-02-22 06:52:05 -0800 | [diff] [blame] | 1443 | data = json.load(f) |
| 1444 | except (IOError, ValueError): |
| 1445 | parser.error('Failed to open %s' % options.json) |
| 1446 | try: |
| 1447 | tasks = sorted( |
| 1448 | data['tasks'].itervalues(), key=lambda x: x['shard_index']) |
| 1449 | args = [t['task_id'] for t in tasks] |
| 1450 | except (KeyError, TypeError): |
| 1451 | parser.error('Failed to process %s' % options.json) |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1452 | if not options.timeout: |
Marc-Antoine Ruel | b73066b | 2018-04-19 20:16:55 -0400 | [diff] [blame] | 1453 | # Take in account all the task slices. |
| 1454 | offset = 0 |
| 1455 | for s in data['request']['task_slices']: |
| 1456 | m = (offset + s['properties']['execution_timeout_secs'] + |
| 1457 | s['expiration_secs']) |
| 1458 | if m > options.timeout: |
| 1459 | options.timeout = m |
| 1460 | offset += s['expiration_secs'] |
Marc-Antoine Ruel | 9fc4261 | 2018-04-20 08:34:22 -0400 | [diff] [blame] | 1461 | options.timeout += 10. |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1462 | else: |
| 1463 | valid = frozenset('0123456789abcdef') |
| 1464 | if any(not valid.issuperset(task_id) for task_id in args): |
| 1465 | parser.error('Task ids are 0-9a-f.') |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1466 | |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1467 | try: |
| 1468 | return collect( |
| 1469 | options.swarming, |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1470 | args, |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1471 | options.timeout, |
| 1472 | options.decorate, |
| 1473 | options.print_status_updates, |
| 1474 | options.task_summary_json, |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 1475 | options.task_output_dir, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1476 | options.task_output_stdout, |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 1477 | options.perf, |
| 1478 | options.filepath_filter) |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1479 | except Failure: |
| 1480 | on_error.report(None) |
| 1481 | return 1 |
| 1482 | |
| 1483 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1484 | @subcommand.usage('[method name]') |
Marc-Antoine Ruel | 833f5eb | 2018-04-25 16:49:40 -0400 | [diff] [blame] | 1485 | def CMDpost(parser, args): |
| 1486 | """Sends a JSON RPC POST to one API endpoint and prints out the raw result. |
| 1487 | |
| 1488 | Input data must be sent to stdin, result is printed to stdout. |
| 1489 | |
| 1490 | If HTTP response code >= 400, returns non-zero. |
| 1491 | """ |
| 1492 | options, args = parser.parse_args(args) |
| 1493 | if len(args) != 1: |
| 1494 | parser.error('Must specify only API name') |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1495 | url = options.swarming + '/_ah/api/swarming/v1/' + args[0] |
Marc-Antoine Ruel | 833f5eb | 2018-04-25 16:49:40 -0400 | [diff] [blame] | 1496 | data = sys.stdin.read() |
| 1497 | try: |
| 1498 | resp = net.url_read(url, data=data, method='POST') |
| 1499 | except net.TimeoutError: |
| 1500 | sys.stderr.write('Timeout!\n') |
| 1501 | return 1 |
| 1502 | if not resp: |
| 1503 | sys.stderr.write('No response!\n') |
| 1504 | return 1 |
| 1505 | sys.stdout.write(resp) |
| 1506 | return 0 |
| 1507 | |
| 1508 | |
| 1509 | @subcommand.usage('[method name]') |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1510 | def CMDquery(parser, args): |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1511 | """Returns raw JSON information via an URL endpoint. Use 'query-list' to |
| 1512 | gather the list of API methods from the server. |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1513 | |
| 1514 | Examples: |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1515 | Raw task request and results: |
| 1516 | swarming.py query -S server-url.com task/123456/request |
| 1517 | swarming.py query -S server-url.com task/123456/result |
| 1518 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1519 | Listing all bots: |
maruel | 84e77aa | 2015-10-21 06:37:24 -0700 | [diff] [blame] | 1520 | swarming.py query -S server-url.com bots/list |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1521 | |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1522 | Listing last 10 tasks on a specific bot named 'bot1': |
| 1523 | swarming.py query -S server-url.com --limit 10 bot/bot1/tasks |
maruel | 84e77aa | 2015-10-21 06:37:24 -0700 | [diff] [blame] | 1524 | |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1525 | Listing last 10 tasks with tags os:Ubuntu-14.04 and pool:Chrome. Note that |
maruel | 84e77aa | 2015-10-21 06:37:24 -0700 | [diff] [blame] | 1526 | quoting is important!: |
| 1527 | swarming.py query -S server-url.com --limit 10 \\ |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1528 | 'tasks/list?tags=os:Ubuntu-14.04&tags=pool:Chrome' |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1529 | """ |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1530 | parser.add_option( |
| 1531 | '-L', '--limit', type='int', default=200, |
| 1532 | help='Limit to enforce on limitless items (like number of tasks); ' |
| 1533 | 'default=%default') |
Paweł Hajdan, Jr | 53ef013 | 2015-03-20 17:49:18 +0100 | [diff] [blame] | 1534 | parser.add_option( |
| 1535 | '--json', help='Path to JSON output file (otherwise prints to stdout)') |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1536 | parser.add_option( |
| 1537 | '--progress', action='store_true', |
| 1538 | help='Prints a dot at each request to show progress') |
| 1539 | options, args = parser.parse_args(args) |
maruel | d8aba22 | 2015-09-03 12:21:19 -0700 | [diff] [blame] | 1540 | if len(args) != 1: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1541 | parser.error( |
| 1542 | 'Must specify only method name and optionally query args properly ' |
| 1543 | 'escaped.') |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1544 | base_url = options.swarming + '/_ah/api/swarming/v1/' + args[0] |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1545 | try: |
| 1546 | data, yielder = get_yielder(base_url, options.limit) |
| 1547 | for items in yielder(): |
| 1548 | if items: |
| 1549 | data['items'].extend(items) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1550 | if options.progress: |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1551 | sys.stderr.write('.') |
| 1552 | sys.stderr.flush() |
| 1553 | except Failure as e: |
| 1554 | sys.stderr.write('\n%s\n' % e) |
| 1555 | return 1 |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1556 | if options.progress: |
maruel | af6b06c | 2017-06-08 06:26:53 -0700 | [diff] [blame] | 1557 | sys.stderr.write('\n') |
| 1558 | sys.stderr.flush() |
Paweł Hajdan, Jr | 53ef013 | 2015-03-20 17:49:18 +0100 | [diff] [blame] | 1559 | if options.json: |
maruel | 1ceb387 | 2015-10-14 06:10:44 -0700 | [diff] [blame] | 1560 | options.json = unicode(os.path.abspath(options.json)) |
| 1561 | tools.write_json(options.json, data, True) |
Paweł Hajdan, Jr | 53ef013 | 2015-03-20 17:49:18 +0100 | [diff] [blame] | 1562 | else: |
Marc-Antoine Ruel | cda90ee | 2015-03-23 15:13:20 -0400 | [diff] [blame] | 1563 | try: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1564 | tools.write_json(sys.stdout, data, False) |
Marc-Antoine Ruel | cda90ee | 2015-03-23 15:13:20 -0400 | [diff] [blame] | 1565 | sys.stdout.write('\n') |
| 1566 | except IOError: |
| 1567 | pass |
Marc-Antoine Ruel | 79940ae | 2014-09-23 17:55:41 -0400 | [diff] [blame] | 1568 | return 0 |
| 1569 | |
| 1570 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1571 | def CMDquery_list(parser, args): |
| 1572 | """Returns list of all the Swarming APIs that can be used with command |
| 1573 | 'query'. |
| 1574 | """ |
| 1575 | parser.add_option( |
| 1576 | '--json', help='Path to JSON output file (otherwise prints to stdout)') |
| 1577 | options, args = parser.parse_args(args) |
| 1578 | if args: |
| 1579 | parser.error('No argument allowed.') |
| 1580 | |
| 1581 | try: |
| 1582 | apis = endpoints_api_discovery_apis(options.swarming) |
| 1583 | except APIError as e: |
| 1584 | parser.error(str(e)) |
| 1585 | if options.json: |
maruel | 1ceb387 | 2015-10-14 06:10:44 -0700 | [diff] [blame] | 1586 | options.json = unicode(os.path.abspath(options.json)) |
| 1587 | with fs.open(options.json, 'wb') as f: |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1588 | json.dump(apis, f) |
| 1589 | else: |
| 1590 | help_url = ( |
| 1591 | 'https://apis-explorer.appspot.com/apis-explorer/?base=%s/_ah/api#p/' % |
| 1592 | options.swarming) |
maruel | 11e31af | 2017-02-15 07:30:50 -0800 | [diff] [blame] | 1593 | for i, (api_id, api) in enumerate(sorted(apis.iteritems())): |
| 1594 | if i: |
| 1595 | print('') |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1596 | print api_id |
maruel | 11e31af | 2017-02-15 07:30:50 -0800 | [diff] [blame] | 1597 | print ' ' + api['description'].strip() |
| 1598 | if 'resources' in api: |
| 1599 | # Old. |
| 1600 | for j, (resource_name, resource) in enumerate( |
| 1601 | sorted(api['resources'].iteritems())): |
| 1602 | if j: |
| 1603 | print('') |
| 1604 | for method_name, method in sorted(resource['methods'].iteritems()): |
| 1605 | # Only list the GET ones. |
| 1606 | if method['httpMethod'] != 'GET': |
| 1607 | continue |
| 1608 | print '- %s.%s: %s' % ( |
| 1609 | resource_name, method_name, method['path']) |
| 1610 | print('\n'.join( |
Sergey Berezin | a269e1a | 2018-05-16 16:55:12 -0700 | [diff] [blame] | 1611 | ' ' + l for l in textwrap.wrap( |
| 1612 | method.get('description', 'No description'), 78))) |
maruel | 11e31af | 2017-02-15 07:30:50 -0800 | [diff] [blame] | 1613 | print ' %s%s%s' % (help_url, api['servicePath'], method['id']) |
| 1614 | else: |
| 1615 | # New. |
| 1616 | for method_name, method in sorted(api['methods'].iteritems()): |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1617 | # Only list the GET ones. |
| 1618 | if method['httpMethod'] != 'GET': |
| 1619 | continue |
maruel | 11e31af | 2017-02-15 07:30:50 -0800 | [diff] [blame] | 1620 | print '- %s: %s' % (method['id'], method['path']) |
| 1621 | print('\n'.join( |
| 1622 | ' ' + l for l in textwrap.wrap(method['description'], 78))) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1623 | print ' %s%s%s' % (help_url, api['servicePath'], method['id']) |
| 1624 | return 0 |
| 1625 | |
| 1626 | |
Vadim Shtayura | ae8085b | 2014-05-02 17:13:10 -0700 | [diff] [blame] | 1627 | @subcommand.usage('(hash|isolated) [-- extra_args]') |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1628 | def CMDrun(parser, args): |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1629 | """Triggers a task and wait for the results. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1630 | |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1631 | Basically, does everything to run a command remotely. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1632 | """ |
| 1633 | add_trigger_options(parser) |
| 1634 | add_collect_options(parser) |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 1635 | add_sharding_options(parser) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1636 | options, args = parser.parse_args(args) |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1637 | process_collect_options(parser, options) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1638 | task_request = process_trigger_options(parser, options, args) |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1639 | try: |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1640 | tasks = trigger_task_shards( |
| 1641 | options.swarming, task_request, options.shards) |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1642 | except Failure as e: |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 1643 | on_error.report( |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1644 | 'Failed to trigger %s(%s): %s' % |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1645 | (task_request.name, args[0], e.args[0])) |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1646 | return 1 |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1647 | if not tasks: |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 1648 | on_error.report('Failed to trigger the task.') |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1649 | return 1 |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1650 | print('Triggered task: %s' % task_request.name) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1651 | task_ids = [ |
| 1652 | t['task_id'] |
| 1653 | for t in sorted(tasks.itervalues(), key=lambda x: x['shard_index']) |
| 1654 | ] |
Marc-Antoine Ruel | f24f09c | 2018-03-23 16:06:18 -0400 | [diff] [blame] | 1655 | if not options.timeout: |
Marc-Antoine Ruel | 3a030bc | 2018-04-23 10:31:25 -0400 | [diff] [blame] | 1656 | offset = 0 |
| 1657 | for s in task_request.task_slices: |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 1658 | m = (offset + s.properties.execution_timeout_secs + |
| 1659 | s.expiration_secs) |
Marc-Antoine Ruel | 3a030bc | 2018-04-23 10:31:25 -0400 | [diff] [blame] | 1660 | if m > options.timeout: |
| 1661 | options.timeout = m |
Marc-Antoine Ruel | 1f835c7 | 2018-05-25 12:29:42 -0400 | [diff] [blame] | 1662 | offset += s.expiration_secs |
Marc-Antoine Ruel | 3a030bc | 2018-04-23 10:31:25 -0400 | [diff] [blame] | 1663 | options.timeout += 10. |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1664 | try: |
| 1665 | return collect( |
| 1666 | options.swarming, |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1667 | task_ids, |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1668 | options.timeout, |
Vadim Shtayura | 86a2cef | 2014-04-18 11:13:39 -0700 | [diff] [blame] | 1669 | options.decorate, |
Vadim Shtayura | e3fbd10 | 2014-04-29 17:05:21 -0700 | [diff] [blame] | 1670 | options.print_status_updates, |
Vadim Shtayura | c8437bf | 2014-07-09 19:45:36 -0700 | [diff] [blame] | 1671 | options.task_summary_json, |
maruel | 9531ce0 | 2016-04-13 06:11:23 -0700 | [diff] [blame] | 1672 | options.task_output_dir, |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1673 | options.task_output_stdout, |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 1674 | options.perf, |
| 1675 | options.filepath_filter) |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 1676 | except Failure: |
| 1677 | on_error.report(None) |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1678 | return 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1679 | |
| 1680 | |
maruel | 18122c6 | 2015-10-23 06:31:23 -0700 | [diff] [blame] | 1681 | @subcommand.usage('task_id -- <extra_args>') |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1682 | def CMDreproduce(parser, args): |
| 1683 | """Runs a task locally that was triggered on the server. |
| 1684 | |
| 1685 | This running locally the same commands that have been run on the bot. The data |
| 1686 | downloaded will be in a subdirectory named 'work' of the current working |
| 1687 | directory. |
maruel | 18122c6 | 2015-10-23 06:31:23 -0700 | [diff] [blame] | 1688 | |
| 1689 | You can pass further additional arguments to the target command by passing |
| 1690 | them after --. |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1691 | """ |
maruel | c070e67 | 2016-02-22 17:32:57 -0800 | [diff] [blame] | 1692 | parser.add_option( |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1693 | '--output', metavar='DIR', default='out', |
maruel | c070e67 | 2016-02-22 17:32:57 -0800 | [diff] [blame] | 1694 | help='Directory that will have results stored into') |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1695 | parser.add_option( |
| 1696 | '--work', metavar='DIR', default='work', |
| 1697 | help='Directory to map the task input files into') |
| 1698 | parser.add_option( |
| 1699 | '--cache', metavar='DIR', default='cache', |
| 1700 | help='Directory that contains the input cache') |
| 1701 | parser.add_option( |
| 1702 | '--leak', action='store_true', |
| 1703 | help='Do not delete the working directory after execution') |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1704 | options, args = parser.parse_args(args) |
maruel | 18122c6 | 2015-10-23 06:31:23 -0700 | [diff] [blame] | 1705 | extra_args = [] |
| 1706 | if not args: |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1707 | parser.error('Must specify exactly one task id.') |
maruel | 18122c6 | 2015-10-23 06:31:23 -0700 | [diff] [blame] | 1708 | if len(args) > 1: |
| 1709 | if args[1] == '--': |
| 1710 | if len(args) > 2: |
| 1711 | extra_args = args[2:] |
| 1712 | else: |
| 1713 | extra_args = args[1:] |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1714 | |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1715 | url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0] |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1716 | request = net.url_read_json(url) |
| 1717 | if not request: |
| 1718 | print >> sys.stderr, 'Failed to retrieve request data for the task' |
| 1719 | return 1 |
| 1720 | |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1721 | workdir = unicode(os.path.abspath(options.work)) |
maruel | e7cd38e | 2016-03-01 19:12:48 -0800 | [diff] [blame] | 1722 | if fs.isdir(workdir): |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1723 | parser.error('Please delete the directory %r first' % options.work) |
maruel | e7cd38e | 2016-03-01 19:12:48 -0800 | [diff] [blame] | 1724 | fs.mkdir(workdir) |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 1725 | cachedir = unicode(os.path.abspath('cipd_cache')) |
| 1726 | if not fs.exists(cachedir): |
| 1727 | fs.mkdir(cachedir) |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1728 | |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1729 | properties = request['properties'] |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 1730 | env = os.environ.copy() |
| 1731 | env['SWARMING_BOT_ID'] = 'reproduce' |
| 1732 | env['SWARMING_TASK_ID'] = 'reproduce' |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1733 | if properties.get('env'): |
Marc-Antoine Ruel | 119b084 | 2014-12-19 15:27:58 -0500 | [diff] [blame] | 1734 | logging.info('env: %r', properties['env']) |
maruel | b76604c | 2015-11-11 11:53:44 -0800 | [diff] [blame] | 1735 | for i in properties['env']: |
Marc-Antoine Ruel | 36e0979 | 2018-01-09 14:03:25 -0500 | [diff] [blame] | 1736 | key = i['key'] |
maruel | b76604c | 2015-11-11 11:53:44 -0800 | [diff] [blame] | 1737 | if not i['value']: |
| 1738 | env.pop(key, None) |
| 1739 | else: |
Marc-Antoine Ruel | 36e0979 | 2018-01-09 14:03:25 -0500 | [diff] [blame] | 1740 | env[key] = i['value'] |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1741 | |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 1742 | if properties.get('env_prefixes'): |
Marc-Antoine Ruel | 36e0979 | 2018-01-09 14:03:25 -0500 | [diff] [blame] | 1743 | env_prefixes = properties['env_prefixes'] |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 1744 | logging.info('env_prefixes: %r', env_prefixes) |
Marc-Antoine Ruel | 36e0979 | 2018-01-09 14:03:25 -0500 | [diff] [blame] | 1745 | for i in env_prefixes: |
| 1746 | key = i['key'] |
| 1747 | paths = [os.path.normpath(os.path.join(workdir, p)) for p in i['value']] |
Robert Iannucci | bf5f84c | 2017-11-22 12:56:50 -0800 | [diff] [blame] | 1748 | cur = env.get(key) |
| 1749 | if cur: |
| 1750 | paths.append(cur) |
| 1751 | env[key] = os.path.pathsep.join(paths) |
| 1752 | |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 1753 | command = [] |
nodir | 152cba6 | 2016-05-12 16:08:56 -0700 | [diff] [blame] | 1754 | if (properties.get('inputs_ref') or {}).get('isolated'): |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1755 | # Create the tree. |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 1756 | server_ref = isolate_storage.ServerRef( |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1757 | properties['inputs_ref']['isolatedserver'], |
Marc-Antoine Ruel | b851313 | 2018-11-20 19:48:53 +0000 | [diff] [blame] | 1758 | properties['inputs_ref']['namespace']) |
| 1759 | with isolateserver.get_storage(server_ref) as storage: |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1760 | # Do not use MemoryContentAddressedCache here, as on 32-bits python, |
| 1761 | # inputs larger than ~1GiB will not fit in memory. This is effectively a |
| 1762 | # leak. |
| 1763 | policies = local_caching.CachePolicies(0, 0, 0, 0) |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1764 | cache = local_caching.DiskContentAddressedCache( |
Marc-Antoine Ruel | 79d4219 | 2019-02-06 19:24:16 +0000 | [diff] [blame] | 1765 | unicode(os.path.abspath(options.cache)), policies, False) |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1766 | bundle = isolateserver.fetch_isolated( |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1767 | properties['inputs_ref']['isolated'], storage, cache, workdir, False) |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1768 | command = bundle.command |
| 1769 | if bundle.relative_cwd: |
| 1770 | workdir = os.path.join(workdir, bundle.relative_cwd) |
maruel | a1b9e55 | 2016-01-06 12:42:03 -0800 | [diff] [blame] | 1771 | command.extend(properties.get('extra_args') or []) |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 1772 | |
| 1773 | if properties.get('command'): |
| 1774 | command.extend(properties['command']) |
| 1775 | |
Marc-Antoine Ruel | c724359 | 2018-05-24 17:04:04 -0400 | [diff] [blame] | 1776 | # https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Magic-Values.md |
Robert Iannucci | 24ae76a | 2018-02-26 12:51:18 -0800 | [diff] [blame] | 1777 | command = tools.fix_python_cmd(command, env) |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1778 | if not options.output: |
Marc-Antoine Ruel | 8822987 | 2018-01-10 16:35:29 -0500 | [diff] [blame] | 1779 | new_command = run_isolated.process_command(command, 'invalid', None) |
| 1780 | if new_command != command: |
Marc-Antoine Ruel | 29ba75c | 2018-01-10 15:04:14 -0500 | [diff] [blame] | 1781 | parser.error('The task has outputs, you must use --output-dir') |
Marc-Antoine Ruel | 8822987 | 2018-01-10 16:35:29 -0500 | [diff] [blame] | 1782 | else: |
| 1783 | # Make the path absolute, as the process will run from a subdirectory. |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1784 | options.output = os.path.abspath(options.output) |
Marc-Antoine Ruel | 8822987 | 2018-01-10 16:35:29 -0500 | [diff] [blame] | 1785 | new_command = run_isolated.process_command( |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1786 | command, options.output, None) |
| 1787 | if not os.path.isdir(options.output): |
| 1788 | os.makedirs(options.output) |
iannucci | 31ab919 | 2017-05-02 19:11:56 -0700 | [diff] [blame] | 1789 | command = new_command |
| 1790 | file_path.ensure_command_has_abs_path(command, workdir) |
| 1791 | |
| 1792 | if properties.get('cipd_input'): |
| 1793 | ci = properties['cipd_input'] |
| 1794 | cp = ci['client_package'] |
| 1795 | client_manager = cipd.get_client( |
| 1796 | ci['server'], cp['package_name'], cp['version'], cachedir) |
| 1797 | |
| 1798 | with client_manager as client: |
| 1799 | by_path = collections.defaultdict(list) |
| 1800 | for pkg in ci['packages']: |
| 1801 | path = pkg['path'] |
| 1802 | # cipd deals with 'root' as '' |
| 1803 | if path == '.': |
| 1804 | path = '' |
| 1805 | by_path[path].append((pkg['package_name'], pkg['version'])) |
| 1806 | client.ensure(workdir, by_path, cache_dir=cachedir) |
| 1807 | |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1808 | try: |
Marc-Antoine Ruel | 95c2187 | 2018-01-10 14:24:28 -0500 | [diff] [blame] | 1809 | return subprocess42.call(command + extra_args, env=env, cwd=workdir) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1810 | except OSError as e: |
maruel | 29ab2fd | 2015-10-16 11:44:01 -0700 | [diff] [blame] | 1811 | print >> sys.stderr, 'Failed to run: %s' % ' '.join(command) |
maruel | 77f720b | 2015-09-15 12:35:22 -0700 | [diff] [blame] | 1812 | print >> sys.stderr, str(e) |
| 1813 | return 1 |
Marc-Antoine Ruel | 5aeb3bb | 2018-06-16 13:11:02 +0000 | [diff] [blame] | 1814 | finally: |
| 1815 | # Do not delete options.cache. |
| 1816 | if not options.leak: |
| 1817 | file_path.rmtree(workdir) |
Marc-Antoine Ruel | 13a8127 | 2014-10-07 20:16:43 -0400 | [diff] [blame] | 1818 | |
| 1819 | |
maruel | 0eb1d1b | 2015-10-02 14:48:21 -0700 | [diff] [blame] | 1820 | @subcommand.usage('bot_id') |
| 1821 | def CMDterminate(parser, args): |
| 1822 | """Tells a bot to gracefully shut itself down as soon as it can. |
| 1823 | |
| 1824 | This is done by completing whatever current task there is then exiting the bot |
| 1825 | process. |
| 1826 | """ |
| 1827 | parser.add_option( |
| 1828 | '--wait', action='store_true', help='Wait for the bot to terminate') |
| 1829 | options, args = parser.parse_args(args) |
| 1830 | if len(args) != 1: |
| 1831 | parser.error('Please provide the bot id') |
smut | 281c390 | 2018-05-30 17:50:05 -0700 | [diff] [blame] | 1832 | url = options.swarming + '/_ah/api/swarming/v1/bot/%s/terminate' % args[0] |
maruel | 0eb1d1b | 2015-10-02 14:48:21 -0700 | [diff] [blame] | 1833 | request = net.url_read_json(url, data={}) |
| 1834 | if not request: |
| 1835 | print >> sys.stderr, 'Failed to ask for termination' |
| 1836 | return 1 |
| 1837 | if options.wait: |
| 1838 | return collect( |
Tim 'mithro' Ansell | 5e8001d | 2017-09-08 09:32:52 +1000 | [diff] [blame] | 1839 | options.swarming, |
| 1840 | [request['task_id']], |
| 1841 | 0., |
| 1842 | False, |
| 1843 | False, |
| 1844 | None, |
| 1845 | None, |
| 1846 | [], |
Takuto Ikuta | 1e6072c | 2018-11-06 20:42:43 +0000 | [diff] [blame] | 1847 | False, |
| 1848 | None) |
maruel | bfc5f87 | 2017-06-10 16:43:17 -0700 | [diff] [blame] | 1849 | else: |
| 1850 | print request['task_id'] |
maruel | 0eb1d1b | 2015-10-02 14:48:21 -0700 | [diff] [blame] | 1851 | return 0 |
| 1852 | |
| 1853 | |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1854 | @subcommand.usage("(hash|isolated) [-- extra_args|raw command]") |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1855 | def CMDtrigger(parser, args): |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1856 | """Triggers a Swarming task. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1857 | |
Vadim Shtayura | ae8085b | 2014-05-02 17:13:10 -0700 | [diff] [blame] | 1858 | Passes all extra arguments provided after '--' as additional command line |
| 1859 | arguments for an isolated command specified in *.isolate file. |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1860 | """ |
| 1861 | add_trigger_options(parser) |
Vadim Shtayura | b450c60 | 2014-05-12 19:23:25 -0700 | [diff] [blame] | 1862 | add_sharding_options(parser) |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1863 | parser.add_option( |
| 1864 | '--dump-json', |
| 1865 | metavar='FILE', |
| 1866 | help='Dump details about the triggered task(s) to this file as json') |
Marc-Antoine Ruel | 7c54327 | 2013-11-26 13:26:15 -0500 | [diff] [blame] | 1867 | options, args = parser.parse_args(args) |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1868 | task_request = process_trigger_options(parser, options, args) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1869 | try: |
Marc-Antoine Ruel | efdc528 | 2014-12-12 19:31:00 -0500 | [diff] [blame] | 1870 | tasks = trigger_task_shards( |
| 1871 | options.swarming, task_request, options.shards) |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1872 | if tasks: |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1873 | print('Triggered task: %s' % task_request.name) |
Marc-Antoine Ruel | 2f6581a | 2014-10-03 11:09:53 -0400 | [diff] [blame] | 1874 | tasks_sorted = sorted( |
| 1875 | tasks.itervalues(), key=lambda x: x['shard_index']) |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1876 | if options.dump_json: |
| 1877 | data = { |
maruel | 0a25f6c | 2017-05-10 10:43:23 -0700 | [diff] [blame] | 1878 | 'base_task_name': task_request.name, |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1879 | 'tasks': tasks, |
Vadim Shtayura | 2d83a94 | 2017-08-14 17:41:24 -0700 | [diff] [blame] | 1880 | 'request': task_request_to_raw_request(task_request), |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1881 | } |
maruel | 46b015f | 2015-10-13 18:40:35 -0700 | [diff] [blame] | 1882 | tools.write_json(unicode(options.dump_json), data, True) |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1883 | print('To collect results, use:') |
Bruce Dawson | f0a5ae4 | 2018-09-04 20:06:46 +0000 | [diff] [blame] | 1884 | print(' tools/swarming_client/swarming.py collect -S %s --json %s' % |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1885 | (options.swarming, options.dump_json)) |
| 1886 | else: |
Marc-Antoine Ruel | 12a7da4 | 2014-10-01 08:29:47 -0400 | [diff] [blame] | 1887 | print('To collect results, use:') |
Bruce Dawson | f0a5ae4 | 2018-09-04 20:06:46 +0000 | [diff] [blame] | 1888 | print(' tools/swarming_client/swarming.py collect -S %s %s' % |
Marc-Antoine Ruel | 2f6581a | 2014-10-03 11:09:53 -0400 | [diff] [blame] | 1889 | (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted))) |
| 1890 | print('Or visit:') |
| 1891 | for t in tasks_sorted: |
| 1892 | print(' ' + t['view_url']) |
Marc-Antoine Ruel | d6dbe76 | 2014-06-18 13:49:42 -0400 | [diff] [blame] | 1893 | return int(not tasks) |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 1894 | except Failure: |
| 1895 | on_error.report(None) |
vadimsh@chromium.org | d908a54 | 2013-10-30 01:36:17 +0000 | [diff] [blame] | 1896 | return 1 |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1897 | |
| 1898 | |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 1899 | class OptionParserSwarming(logging_utils.OptionParserWithLogging): |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1900 | def __init__(self, **kwargs): |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 1901 | logging_utils.OptionParserWithLogging.__init__( |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1902 | self, prog='swarming.py', **kwargs) |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 1903 | self.server_group = optparse.OptionGroup(self, 'Server') |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 1904 | self.server_group.add_option( |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 1905 | '-S', '--swarming', |
Kevin Graney | 5346c16 | 2014-01-24 12:20:01 -0500 | [diff] [blame] | 1906 | metavar='URL', default=os.environ.get('SWARMING_SERVER', ''), |
maruel@chromium.org | e9403ab | 2013-09-20 18:03:49 +0000 | [diff] [blame] | 1907 | help='Swarming server to use') |
Marc-Antoine Ruel | 5471e3d | 2013-11-11 19:10:32 -0500 | [diff] [blame] | 1908 | self.add_option_group(self.server_group) |
Vadim Shtayura | e34e13a | 2014-02-02 11:23:26 -0800 | [diff] [blame] | 1909 | auth.add_auth_options(self) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1910 | |
| 1911 | def parse_args(self, *args, **kwargs): |
Marc-Antoine Ruel | f74cffe | 2015-07-15 15:21:34 -0400 | [diff] [blame] | 1912 | options, args = logging_utils.OptionParserWithLogging.parse_args( |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1913 | self, *args, **kwargs) |
Marc-Antoine Ruel | 012067b | 2014-12-10 15:45:42 -0500 | [diff] [blame] | 1914 | auth.process_auth_options(self, options) |
| 1915 | user = self._process_swarming(options) |
| 1916 | if hasattr(options, 'user') and not options.user: |
| 1917 | options.user = user |
| 1918 | return options, args |
| 1919 | |
| 1920 | def _process_swarming(self, options): |
| 1921 | """Processes the --swarming option and aborts if not specified. |
| 1922 | |
| 1923 | Returns the identity as determined by the server. |
| 1924 | """ |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1925 | if not options.swarming: |
| 1926 | self.error('--swarming is required.') |
Marc-Antoine Ruel | 012067b | 2014-12-10 15:45:42 -0500 | [diff] [blame] | 1927 | try: |
| 1928 | options.swarming = net.fix_url(options.swarming) |
| 1929 | except ValueError as e: |
| 1930 | self.error('--swarming %s' % e) |
| 1931 | on_error.report_on_exception_exit(options.swarming) |
Marc-Antoine Ruel | f7d737d | 2014-12-10 15:36:29 -0500 | [diff] [blame] | 1932 | try: |
| 1933 | user = auth.ensure_logged_in(options.swarming) |
| 1934 | except ValueError as e: |
| 1935 | self.error(str(e)) |
Marc-Antoine Ruel | 012067b | 2014-12-10 15:45:42 -0500 | [diff] [blame] | 1936 | return user |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1937 | |
| 1938 | |
| 1939 | def main(args): |
| 1940 | dispatcher = subcommand.CommandDispatcher(__name__) |
Marc-Antoine Ruel | cfb6085 | 2014-07-02 15:22:00 -0400 | [diff] [blame] | 1941 | return dispatcher.execute(OptionParserSwarming(version=__version__), args) |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1942 | |
| 1943 | |
| 1944 | if __name__ == '__main__': |
maruel | 8e4e40c | 2016-05-30 06:21:07 -0700 | [diff] [blame] | 1945 | subprocess42.inhibit_os_error_reporting() |
maruel@chromium.org | 0437a73 | 2013-08-27 16:05:52 +0000 | [diff] [blame] | 1946 | fix_encoding.fix_encoding() |
| 1947 | tools.disable_buffering() |
| 1948 | colorama.init() |
| 1949 | sys.exit(main(sys.argv[1:])) |