blob: 191224d8ca82785d584bc6bf68b6dda7908b2228 [file] [log] [blame]
maruel@chromium.org0437a732013-08-27 16:05:52 +00001#!/usr/bin/env python
maruelea586f32016-04-05 11:11:33 -07002# Copyright 2013 The LUCI Authors. All rights reserved.
maruelf1f5e2a2016-05-25 17:10:39 -07003# Use of this source code is governed under the Apache License, Version 2.0
4# that can be found in the LICENSE file.
maruel@chromium.org0437a732013-08-27 16:05:52 +00005
6"""Client tool to trigger tasks or retrieve results from a Swarming server."""
7
Takuto Ikuta0e3e1c42018-11-29 14:21:06 +00008__version__ = '0.14'
maruel@chromium.org0437a732013-08-27 16:05:52 +00009
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050010import collections
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -040011import datetime
maruel@chromium.org0437a732013-08-27 16:05:52 +000012import json
13import logging
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -040014import optparse
maruel@chromium.org0437a732013-08-27 16:05:52 +000015import os
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +100016import re
maruel@chromium.org0437a732013-08-27 16:05:52 +000017import sys
maruel11e31af2017-02-15 07:30:50 -080018import textwrap
Vadim Shtayurab19319e2014-04-27 08:50:06 -070019import threading
maruel@chromium.org0437a732013-08-27 16:05:52 +000020import time
21import urllib
maruel@chromium.org0437a732013-08-27 16:05:52 +000022
23from third_party import colorama
24from third_party.depot_tools import fix_encoding
25from third_party.depot_tools import subcommand
vadimsh@chromium.org6b706212013-08-28 15:03:46 +000026
Marc-Antoine Ruel8806e622014-02-12 14:15:53 -050027from utils import file_path
maruel12e30012015-10-09 11:55:35 -070028from utils import fs
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -040029from utils import logging_utils
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -040030from third_party.chromium import natsort
vadimsh@chromium.org6b706212013-08-28 15:03:46 +000031from utils import net
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -040032from utils import on_error
maruel8e4e40c2016-05-30 06:21:07 -070033from utils import subprocess42
maruel@chromium.org0437a732013-08-27 16:05:52 +000034from utils import threading_utils
vadimsh@chromium.org6b706212013-08-28 15:03:46 +000035from utils import tools
maruel@chromium.org0437a732013-08-27 16:05:52 +000036
Vadim Shtayurae34e13a2014-02-02 11:23:26 -080037import auth
iannucci31ab9192017-05-02 19:11:56 -070038import cipd
maruel@chromium.org7b844a62013-09-17 13:04:59 +000039import isolateserver
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +000040import isolate_storage
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +000041import isolated_format
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -040042import local_caching
maruelc070e672016-02-22 17:32:57 -080043import run_isolated
maruel@chromium.org0437a732013-08-27 16:05:52 +000044
45
tansella4949442016-06-23 22:34:32 -070046ROOT_DIR = os.path.dirname(os.path.abspath(
47 __file__.decode(sys.getfilesystemencoding())))
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050048
49
50class Failure(Exception):
51 """Generic failure."""
52 pass
53
54
maruel0a25f6c2017-05-10 10:43:23 -070055def default_task_name(options):
56 """Returns a default task name if not specified."""
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050057 if not options.task_name:
maruel0a25f6c2017-05-10 10:43:23 -070058 task_name = u'%s/%s' % (
marueld9cc8422017-05-09 12:07:02 -070059 options.user,
maruelaf6b06c2017-06-08 06:26:53 -070060 '_'.join('%s=%s' % (k, v) for k, v in options.dimensions))
maruel0a25f6c2017-05-10 10:43:23 -070061 if options.isolated:
62 task_name += u'/' + options.isolated
63 return task_name
64 return options.task_name
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050065
66
67### Triggering.
68
69
maruel77f720b2015-09-15 12:35:22 -070070# See ../appengine/swarming/swarming_rpcs.py.
borenet02f772b2016-06-22 12:42:19 -070071CipdPackage = collections.namedtuple(
72 'CipdPackage',
73 [
74 'package_name',
75 'path',
76 'version',
77 ])
78
79
80# See ../appengine/swarming/swarming_rpcs.py.
81CipdInput = collections.namedtuple(
82 'CipdInput',
83 [
84 'client_package',
85 'packages',
86 'server',
87 ])
88
89
90# See ../appengine/swarming/swarming_rpcs.py.
maruel77f720b2015-09-15 12:35:22 -070091FilesRef = collections.namedtuple(
92 'FilesRef',
93 [
94 'isolated',
95 'isolatedserver',
96 'namespace',
97 ])
98
99
100# See ../appengine/swarming/swarming_rpcs.py.
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800101StringListPair = collections.namedtuple(
102 'StringListPair', [
103 'key',
104 'value', # repeated string
105 ]
106)
107
108
109# See ../appengine/swarming/swarming_rpcs.py.
maruel77f720b2015-09-15 12:35:22 -0700110TaskProperties = collections.namedtuple(
111 'TaskProperties',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500112 [
maruel681d6802017-01-17 16:56:03 -0800113 'caches',
borenet02f772b2016-06-22 12:42:19 -0700114 'cipd_input',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500115 'command',
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -0500116 'relative_cwd',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500117 'dimensions',
118 'env',
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800119 'env_prefixes',
maruel77f720b2015-09-15 12:35:22 -0700120 'execution_timeout_secs',
121 'extra_args',
122 'grace_period_secs',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500123 'idempotent',
maruel77f720b2015-09-15 12:35:22 -0700124 'inputs_ref',
125 'io_timeout_secs',
aludwincc5524e2016-10-28 10:25:24 -0700126 'outputs',
iannuccidc80dfb2016-10-28 12:50:20 -0700127 'secret_bytes',
maruel77f720b2015-09-15 12:35:22 -0700128 ])
129
130
131# See ../appengine/swarming/swarming_rpcs.py.
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -0400132TaskSlice = collections.namedtuple(
133 'TaskSlice',
134 [
135 'expiration_secs',
136 'properties',
137 'wait_for_capacity',
138 ])
139
140
141# See ../appengine/swarming/swarming_rpcs.py.
maruel77f720b2015-09-15 12:35:22 -0700142NewTaskRequest = collections.namedtuple(
143 'NewTaskRequest',
144 [
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500145 'name',
maruel77f720b2015-09-15 12:35:22 -0700146 'parent_task_id',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500147 'priority',
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -0400148 'task_slices',
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700149 'service_account',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500150 'tags',
151 'user',
Robert Iannuccifafa7352018-06-13 17:08:17 +0000152 'pool_task_template',
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500153 ])
154
155
maruel77f720b2015-09-15 12:35:22 -0700156def namedtuple_to_dict(value):
157 """Recursively converts a namedtuple to a dict."""
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -0400158 if hasattr(value, '_asdict'):
159 return namedtuple_to_dict(value._asdict())
160 if isinstance(value, (list, tuple)):
161 return [namedtuple_to_dict(v) for v in value]
162 if isinstance(value, dict):
163 return {k: namedtuple_to_dict(v) for k, v in value.iteritems()}
164 return value
maruel77f720b2015-09-15 12:35:22 -0700165
166
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700167def task_request_to_raw_request(task_request):
maruel71c61c82016-02-22 06:52:05 -0800168 """Returns the json-compatible dict expected by the server for new request.
maruelaf6269b2015-09-10 14:37:51 -0700169
170 This is for the v1 client Swarming API.
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500171 """
maruel77f720b2015-09-15 12:35:22 -0700172 out = namedtuple_to_dict(task_request)
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700173 # Don't send 'service_account' if it is None to avoid confusing older
174 # version of the server that doesn't know about 'service_account' and don't
175 # use it at all.
176 if not out['service_account']:
177 out.pop('service_account')
Brad Hallf78187a2018-10-19 17:08:55 +0000178 for task_slice in out['task_slices']:
179 task_slice['properties']['env'] = [
180 {'key': k, 'value': v}
181 for k, v in task_slice['properties']['env'].iteritems()
182 ]
183 task_slice['properties']['env'].sort(key=lambda x: x['key'])
maruel77f720b2015-09-15 12:35:22 -0700184 return out
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500185
186
maruel77f720b2015-09-15 12:35:22 -0700187def swarming_trigger(swarming, raw_request):
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500188 """Triggers a request on the Swarming server and returns the json data.
189
190 It's the low-level function.
191
192 Returns:
193 {
194 'request': {
195 'created_ts': u'2010-01-02 03:04:05',
196 'name': ..
197 },
198 'task_id': '12300',
199 }
200 """
201 logging.info('Triggering: %s', raw_request['name'])
202
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500203 result = net.url_read_json(
smut281c3902018-05-30 17:50:05 -0700204 swarming + '/_ah/api/swarming/v1/tasks/new', data=raw_request)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500205 if not result:
206 on_error.report('Failed to trigger task %s' % raw_request['name'])
207 return None
maruele557bce2015-11-17 09:01:27 -0800208 if result.get('error'):
marueld4d15312015-11-16 17:22:59 -0800209 # The reply is an error.
maruele557bce2015-11-17 09:01:27 -0800210 msg = 'Failed to trigger task %s' % raw_request['name']
211 if result['error'].get('errors'):
212 for err in result['error']['errors']:
213 if err.get('message'):
214 msg += '\nMessage: %s' % err['message']
215 if err.get('debugInfo'):
216 msg += '\nDebug info:\n%s' % err['debugInfo']
217 elif result['error'].get('message'):
maruelbf53e042015-12-01 15:00:51 -0800218 msg += '\nMessage: %s' % result['error']['message']
maruele557bce2015-11-17 09:01:27 -0800219
220 on_error.report(msg)
marueld4d15312015-11-16 17:22:59 -0800221 return None
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500222 return result
223
224
225def setup_googletest(env, shards, index):
226 """Sets googletest specific environment variables."""
227 if shards > 1:
maruel77f720b2015-09-15 12:35:22 -0700228 assert not any(i['key'] == 'GTEST_SHARD_INDEX' for i in env), env
229 assert not any(i['key'] == 'GTEST_TOTAL_SHARDS' for i in env), env
230 env = env[:]
231 env.append({'key': 'GTEST_SHARD_INDEX', 'value': str(index)})
232 env.append({'key': 'GTEST_TOTAL_SHARDS', 'value': str(shards)})
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500233 return env
234
235
236def trigger_task_shards(swarming, task_request, shards):
237 """Triggers one or many subtasks of a sharded task.
238
239 Returns:
240 Dict with task details, returned to caller as part of --dump-json output.
241 None in case of failure.
242 """
243 def convert(index):
Erik Chend50a88f2019-02-16 01:22:07 +0000244 """
245 Args:
246 index: The index of the task request.
247
248 Returns:
249 raw_request: A swarming compatible JSON dictionary of the request.
250 shard_index: The index of the shard, which may be different than the index
251 of the task request.
252 """
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700253 req = task_request_to_raw_request(task_request)
Erik Chend50a88f2019-02-16 01:22:07 +0000254 shard_index = index
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500255 if shards > 1:
Brad Hall157bec82018-11-26 22:15:38 +0000256 for task_slice in req['task_slices']:
257 task_slice['properties']['env'] = setup_googletest(
258 task_slice['properties']['env'], shards, index)
maruel77f720b2015-09-15 12:35:22 -0700259 req['name'] += ':%s:%s' % (index, shards)
Erik Chend50a88f2019-02-16 01:22:07 +0000260 else:
261 task_slices = req['task_slices']
262
263 total_shards = None
264 # Multiple tasks slices might exist if there are optional "slices", e.g.
265 # multiple ways of dispatching the task that should be equivalent. These
266 # should be functionally equivalent but we have cannot guarantee that. If
267 # we see the GTEST_SHARD_INDEX env var, we assume that it applies to all
268 # slices.
269 for task_slice in task_slices:
270 for env_var in task_slice['properties']['env']:
271 if env_var['key'] == 'GTEST_SHARD_INDEX':
272 shard_index = int(env_var['value'])
273 if env_var['key'] == 'GTEST_TOTAL_SHARDS':
274 total_shards = int(env_var['value'])
275 if total_shards > 1:
276 req['name'] += ':%s:%s' % (shard_index, total_shards)
277
278 return req, shard_index
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500279
280 requests = [convert(index) for index in xrange(shards)]
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500281 tasks = {}
282 priority_warning = False
Erik Chend50a88f2019-02-16 01:22:07 +0000283 for request, shard_index in requests:
maruel77f720b2015-09-15 12:35:22 -0700284 task = swarming_trigger(swarming, request)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500285 if not task:
286 break
287 logging.info('Request result: %s', task)
288 if (not priority_warning and
Marc-Antoine Ruelb1216762017-08-17 10:07:49 -0400289 int(task['request']['priority']) != task_request.priority):
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500290 priority_warning = True
291 print >> sys.stderr, (
292 'Priority was reset to %s' % task['request']['priority'])
293 tasks[request['name']] = {
Erik Chend50a88f2019-02-16 01:22:07 +0000294 'shard_index': shard_index,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500295 'task_id': task['task_id'],
296 'view_url': '%s/user/task/%s' % (swarming, task['task_id']),
297 }
298
299 # Some shards weren't triggered. Abort everything.
300 if len(tasks) != len(requests):
301 if tasks:
302 print >> sys.stderr, 'Only %d shard(s) out of %d were triggered' % (
303 len(tasks), len(requests))
304 for task_dict in tasks.itervalues():
305 abort_task(swarming, task_dict['task_id'])
306 return None
307
308 return tasks
309
310
311### Collection.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000312
313
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700314# How often to print status updates to stdout in 'collect'.
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000315STATUS_UPDATE_INTERVAL = 5 * 60.
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700316
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400317
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000318class TaskState(object):
319 """Represents the current task state.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000320
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000321 For documentation, see the comments in the swarming_rpcs.TaskState enum, which
322 is the source of truth for these values:
323 https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_rpcs.py?q=TaskState\(
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400324
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000325 It's in fact an enum.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400326 """
327 RUNNING = 0x10
328 PENDING = 0x20
329 EXPIRED = 0x30
330 TIMED_OUT = 0x40
331 BOT_DIED = 0x50
332 CANCELED = 0x60
333 COMPLETED = 0x70
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -0400334 KILLED = 0x80
Marc-Antoine Ruelfc708352018-05-04 20:25:43 -0400335 NO_RESOURCE = 0x100
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400336
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000337 STATES_RUNNING = ('PENDING', 'RUNNING')
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400338
maruel77f720b2015-09-15 12:35:22 -0700339 _ENUMS = {
340 'RUNNING': RUNNING,
341 'PENDING': PENDING,
342 'EXPIRED': EXPIRED,
343 'TIMED_OUT': TIMED_OUT,
344 'BOT_DIED': BOT_DIED,
345 'CANCELED': CANCELED,
346 'COMPLETED': COMPLETED,
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -0400347 'KILLED': KILLED,
Marc-Antoine Ruelfc708352018-05-04 20:25:43 -0400348 'NO_RESOURCE': NO_RESOURCE,
maruel77f720b2015-09-15 12:35:22 -0700349 }
350
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400351 @classmethod
maruel77f720b2015-09-15 12:35:22 -0700352 def from_enum(cls, state):
353 """Returns int value based on the string."""
354 if state not in cls._ENUMS:
355 raise ValueError('Invalid state %s' % state)
356 return cls._ENUMS[state]
357
maruel@chromium.org0437a732013-08-27 16:05:52 +0000358
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700359class TaskOutputCollector(object):
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700360 """Assembles task execution summary (for --task-summary-json output).
361
362 Optionally fetches task outputs from isolate server to local disk (used when
363 --task-output-dir is passed).
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700364
365 This object is shared among multiple threads running 'retrieve_results'
366 function, in particular they call 'process_shard_result' method in parallel.
367 """
368
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000369 def __init__(self, task_output_dir, task_output_stdout, shard_count,
370 filter_cb):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700371 """Initializes TaskOutputCollector, ensures |task_output_dir| exists.
372
373 Args:
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700374 task_output_dir: (optional) local directory to put fetched files to.
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700375 shard_count: expected number of task shards.
376 """
maruel12e30012015-10-09 11:55:35 -0700377 self.task_output_dir = (
378 unicode(os.path.abspath(task_output_dir))
379 if task_output_dir else task_output_dir)
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000380 self.task_output_stdout = task_output_stdout
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700381 self.shard_count = shard_count
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000382 self.filter_cb = filter_cb
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700383
384 self._lock = threading.Lock()
385 self._per_shard_results = {}
386 self._storage = None
387
nodire5028a92016-04-29 14:38:21 -0700388 if self.task_output_dir:
389 file_path.ensure_tree(self.task_output_dir)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700390
Vadim Shtayurab450c602014-05-12 19:23:25 -0700391 def process_shard_result(self, shard_index, result):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700392 """Stores results of a single task shard, fetches output files if necessary.
393
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400394 Modifies |result| in place.
395
maruel77f720b2015-09-15 12:35:22 -0700396 shard_index is 0-based.
397
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700398 Called concurrently from multiple threads.
399 """
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700400 # Sanity check index is in expected range.
Vadim Shtayurab450c602014-05-12 19:23:25 -0700401 assert isinstance(shard_index, int)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700402 if shard_index < 0 or shard_index >= self.shard_count:
403 logging.warning(
404 'Shard index %d is outside of expected range: [0; %d]',
405 shard_index, self.shard_count - 1)
406 return
407
maruel77f720b2015-09-15 12:35:22 -0700408 if result.get('outputs_ref'):
409 ref = result['outputs_ref']
410 result['outputs_ref']['view_url'] = '%s/browse?%s' % (
411 ref['isolatedserver'],
412 urllib.urlencode(
413 [('namespace', ref['namespace']), ('hash', ref['isolated'])]))
Kevin Graneyc2c3b9e2014-08-26 09:04:17 -0400414
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700415 # Store result dict of that shard, ignore results we've already seen.
416 with self._lock:
417 if shard_index in self._per_shard_results:
418 logging.warning('Ignoring duplicate shard index %d', shard_index)
419 return
420 self._per_shard_results[shard_index] = result
421
422 # Fetch output files if necessary.
maruel77f720b2015-09-15 12:35:22 -0700423 if self.task_output_dir and result.get('outputs_ref'):
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000424 server_ref = isolate_storage.ServerRef(
425 result['outputs_ref']['isolatedserver'],
426 result['outputs_ref']['namespace'])
427 storage = self._get_storage(server_ref)
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400428 if storage:
429 # Output files are supposed to be small and they are not reused across
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -0400430 # tasks. So use MemoryContentAddressedCache for them instead of on-disk
431 # cache. Make files writable, so that calling script can delete them.
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400432 isolateserver.fetch_isolated(
maruel77f720b2015-09-15 12:35:22 -0700433 result['outputs_ref']['isolated'],
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400434 storage,
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -0400435 local_caching.MemoryContentAddressedCache(file_mode_mask=0700),
maruel4409e302016-07-19 14:25:51 -0700436 os.path.join(self.task_output_dir, str(shard_index)),
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000437 False, self.filter_cb)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700438
439 def finalize(self):
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700440 """Assembles and returns task summary JSON, shutdowns underlying Storage."""
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700441 with self._lock:
442 # Write an array of shard results with None for missing shards.
443 summary = {
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700444 'shards': [
445 self._per_shard_results.get(i) for i in xrange(self.shard_count)
446 ],
447 }
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000448
449 # Don't store stdout in the summary if not requested too.
450 if "json" not in self.task_output_stdout:
451 for shard_json in summary['shards']:
452 if not shard_json:
453 continue
454 if "output" in shard_json:
455 del shard_json["output"]
456 if "outputs" in shard_json:
457 del shard_json["outputs"]
458
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700459 # Write summary.json to task_output_dir as well.
460 if self.task_output_dir:
461 tools.write_json(
maruel12e30012015-10-09 11:55:35 -0700462 os.path.join(self.task_output_dir, u'summary.json'),
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700463 summary,
464 False)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700465 if self._storage:
466 self._storage.close()
467 self._storage = None
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700468 return summary
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700469
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000470 def _get_storage(self, server_ref):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700471 """Returns isolateserver.Storage to use to fetch files."""
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700472 assert self.task_output_dir
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700473 with self._lock:
474 if not self._storage:
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000475 self._storage = isolateserver.get_storage(server_ref)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700476 else:
477 # Shards must all use exact same isolate server and namespace.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000478 if self._storage.server_ref.url != server_ref.url:
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700479 logging.error(
480 'Task shards are using multiple isolate servers: %s and %s',
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000481 self._storage.server_ref.url, server_ref.url)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700482 return None
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000483 if self._storage.server_ref.namespace != server_ref.namespace:
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700484 logging.error(
485 'Task shards are using multiple namespaces: %s and %s',
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000486 self._storage.server_ref.namespace, server_ref.namespace)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700487 return None
488 return self._storage
489
490
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500491def now():
492 """Exists so it can be mocked easily."""
493 return time.time()
494
495
maruel77f720b2015-09-15 12:35:22 -0700496def parse_time(value):
497 """Converts serialized time from the API to datetime.datetime."""
498 # When microseconds are 0, the '.123456' suffix is elided. This means the
499 # serialized format is not consistent, which confuses the hell out of python.
500 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S'):
501 try:
502 return datetime.datetime.strptime(value, fmt)
503 except ValueError:
504 pass
505 raise ValueError('Failed to parse %s' % value)
506
507
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700508def retrieve_results(
maruel9531ce02016-04-13 06:11:23 -0700509 base_url, shard_index, task_id, timeout, should_stop, output_collector,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000510 include_perf, fetch_stdout):
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400511 """Retrieves results for a single task ID.
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700512
Vadim Shtayurab450c602014-05-12 19:23:25 -0700513 Returns:
514 <result dict> on success.
515 None on failure.
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700516 """
maruel71c61c82016-02-22 06:52:05 -0800517 assert timeout is None or isinstance(timeout, float), timeout
smut281c3902018-05-30 17:50:05 -0700518 result_url = '%s/_ah/api/swarming/v1/task/%s/result' % (base_url, task_id)
maruel9531ce02016-04-13 06:11:23 -0700519 if include_perf:
520 result_url += '?include_performance_stats=true'
smut281c3902018-05-30 17:50:05 -0700521 output_url = '%s/_ah/api/swarming/v1/task/%s/stdout' % (base_url, task_id)
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700522 started = now()
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400523 deadline = started + timeout if timeout > 0 else None
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700524 attempt = 0
525
526 while not should_stop.is_set():
527 attempt += 1
528
529 # Waiting for too long -> give up.
530 current_time = now()
531 if deadline and current_time >= deadline:
532 logging.error('retrieve_results(%s) timed out on attempt %d',
533 base_url, attempt)
534 return None
535
536 # Do not spin too fast. Spin faster at the beginning though.
537 # Start with 1 sec delay and for each 30 sec of waiting add another second
538 # of delay, until hitting 15 sec ceiling.
539 if attempt > 1:
540 max_delay = min(15, 1 + (current_time - started) / 30.0)
541 delay = min(max_delay, deadline - current_time) if deadline else max_delay
542 if delay > 0:
543 logging.debug('Waiting %.1f sec before retrying', delay)
544 should_stop.wait(delay)
545 if should_stop.is_set():
546 return None
547
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400548 # Disable internal retries in net.url_read_json, since we are doing retries
549 # ourselves.
550 # TODO(maruel): We'd need to know if it's a 404 and not retry at all.
maruel0eb1d1b2015-10-02 14:48:21 -0700551 # TODO(maruel): Sadly, we currently have to poll here. Use hanging HTTP
552 # request on GAE v2.
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400553 # Retry on 500s only if no timeout is specified.
554 result = net.url_read_json(result_url, retry_50x=bool(timeout == -1))
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400555 if not result:
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400556 if timeout == -1:
557 return None
Marc-Antoine Ruel200b3952014-08-14 11:07:44 -0400558 continue
maruel77f720b2015-09-15 12:35:22 -0700559
maruelbf53e042015-12-01 15:00:51 -0800560 if result.get('error'):
561 # An error occurred.
562 if result['error'].get('errors'):
563 for err in result['error']['errors']:
564 logging.warning(
565 'Error while reading task: %s; %s',
566 err.get('message'), err.get('debugInfo'))
567 elif result['error'].get('message'):
568 logging.warning(
569 'Error while reading task: %s', result['error']['message'])
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400570 if timeout == -1:
571 return result
maruelbf53e042015-12-01 15:00:51 -0800572 continue
573
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400574 # When timeout == -1, always return on first attempt. 500s are already
575 # retried in this case.
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000576 if result['state'] not in TaskState.STATES_RUNNING or timeout == -1:
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000577 if fetch_stdout:
578 out = net.url_read_json(output_url)
Vadim Shtayura6fd3c7b2017-11-03 15:32:51 -0700579 result['output'] = out.get('output', '') if out else ''
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700580 # Record the result, try to fetch attached output files (if any).
581 if output_collector:
582 # TODO(vadimsh): Respect |should_stop| and |deadline| when fetching.
Vadim Shtayurab450c602014-05-12 19:23:25 -0700583 output_collector.process_shard_result(shard_index, result)
maruel77f720b2015-09-15 12:35:22 -0700584 if result.get('internal_failure'):
585 logging.error('Internal error!')
586 elif result['state'] == 'BOT_DIED':
587 logging.error('Bot died!')
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700588 return result
maruel@chromium.org0437a732013-08-27 16:05:52 +0000589
590
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700591def yield_results(
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400592 swarm_base_url, task_ids, timeout, max_threads, print_status_updates,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000593 output_collector, include_perf, fetch_stdout):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500594 """Yields swarming task results from the swarming server as (index, result).
maruel@chromium.org0437a732013-08-27 16:05:52 +0000595
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700596 Duplicate shards are ignored. Shards are yielded in order of completion.
597 Timed out shards are NOT yielded at all. Caller can compare number of yielded
598 shards with len(task_keys) to verify all shards completed.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000599
600 max_threads is optional and is used to limit the number of parallel fetches
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500601 done. Since in general the number of task_keys is in the range <=10, it's not
maruel@chromium.org0437a732013-08-27 16:05:52 +0000602 worth normally to limit the number threads. Mostly used for testing purposes.
Marc-Antoine Ruel5c720342014-02-21 14:46:14 -0500603
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700604 output_collector is an optional instance of TaskOutputCollector that will be
605 used to fetch files produced by a task from isolate server to the local disk.
606
Marc-Antoine Ruel5c720342014-02-21 14:46:14 -0500607 Yields:
608 (index, result). In particular, 'result' is defined as the
609 GetRunnerResults() function in services/swarming/server/test_runner.py.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000610 """
maruel@chromium.org0437a732013-08-27 16:05:52 +0000611 number_threads = (
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400612 min(max_threads, len(task_ids)) if max_threads else len(task_ids))
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700613 should_stop = threading.Event()
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700614 results_channel = threading_utils.TaskChannel()
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700615
maruel@chromium.org0437a732013-08-27 16:05:52 +0000616 with threading_utils.ThreadPool(number_threads, number_threads, 0) as pool:
617 try:
Vadim Shtayurab450c602014-05-12 19:23:25 -0700618 # Adds a task to the thread pool to call 'retrieve_results' and return
619 # the results together with shard_index that produced them (as a tuple).
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400620 def enqueue_retrieve_results(shard_index, task_id):
Marc-Antoine Ruel486c9b52018-07-23 19:30:47 +0000621 # pylint: disable=no-value-for-parameter
Vadim Shtayurab450c602014-05-12 19:23:25 -0700622 task_fn = lambda *args: (shard_index, retrieve_results(*args))
maruel@chromium.org0437a732013-08-27 16:05:52 +0000623 pool.add_task(
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400624 0, results_channel.wrap_task(task_fn), swarm_base_url, shard_index,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000625 task_id, timeout, should_stop, output_collector, include_perf,
626 fetch_stdout)
Vadim Shtayurab450c602014-05-12 19:23:25 -0700627
628 # Enqueue 'retrieve_results' calls for each shard key to run in parallel.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400629 for shard_index, task_id in enumerate(task_ids):
630 enqueue_retrieve_results(shard_index, task_id)
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700631
632 # Wait for all of them to finish.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400633 shards_remaining = range(len(task_ids))
634 active_task_count = len(task_ids)
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700635 while active_task_count:
Vadim Shtayurab450c602014-05-12 19:23:25 -0700636 shard_index, result = None, None
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700637 try:
Marc-Antoine Ruel4494b6c2018-11-28 21:00:41 +0000638 shard_index, result = results_channel.next(
Vadim Shtayurab450c602014-05-12 19:23:25 -0700639 timeout=STATUS_UPDATE_INTERVAL)
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700640 except threading_utils.TaskChannel.Timeout:
641 if print_status_updates:
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000642 time_now = str(datetime.datetime.now())
643 _, time_now = time_now.split(' ')
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700644 print(
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000645 '%s '
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700646 'Waiting for results from the following shards: %s' %
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000647 (time_now, ', '.join(map(str, shards_remaining)))
648 )
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700649 sys.stdout.flush()
650 continue
651 except Exception:
652 logging.exception('Unexpected exception in retrieve_results')
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700653
654 # A call to 'retrieve_results' finished (successfully or not).
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700655 active_task_count -= 1
maruel@chromium.org0437a732013-08-27 16:05:52 +0000656 if not result:
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500657 logging.error('Failed to retrieve the results for a swarming key')
maruel@chromium.org0437a732013-08-27 16:05:52 +0000658 continue
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700659
Vadim Shtayurab450c602014-05-12 19:23:25 -0700660 # Yield back results to the caller.
661 assert shard_index in shards_remaining
662 shards_remaining.remove(shard_index)
663 yield shard_index, result
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700664
maruel@chromium.org0437a732013-08-27 16:05:52 +0000665 finally:
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700666 # Done or aborted with Ctrl+C, kill the remaining threads.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000667 should_stop.set()
668
669
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000670def decorate_shard_output(swarming, shard_index, metadata, include_stdout):
maruel@chromium.org0437a732013-08-27 16:05:52 +0000671 """Returns wrapped output for swarming task shard."""
maruel77f720b2015-09-15 12:35:22 -0700672 if metadata.get('started_ts') and not metadata.get('deduped_from'):
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400673 pending = '%.1fs' % (
maruel77f720b2015-09-15 12:35:22 -0700674 parse_time(metadata['started_ts']) - parse_time(metadata['created_ts'])
675 ).total_seconds()
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400676 elif (metadata.get('state') in ('BOT_DIED', 'CANCELED', 'EXPIRED') and
677 metadata.get('abandoned_ts')):
678 pending = '%.1fs' % (
679 parse_time(metadata['abandoned_ts']) -
680 parse_time(metadata['created_ts'])
681 ).total_seconds()
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400682 else:
683 pending = 'N/A'
684
maruel77f720b2015-09-15 12:35:22 -0700685 if metadata.get('duration') is not None:
686 duration = '%.1fs' % metadata['duration']
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400687 else:
688 duration = 'N/A'
689
maruel77f720b2015-09-15 12:35:22 -0700690 if metadata.get('exit_code') is not None:
691 # Integers are encoded as string to not loose precision.
692 exit_code = '%s' % metadata['exit_code']
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400693 else:
694 exit_code = 'N/A'
695
696 bot_id = metadata.get('bot_id') or 'N/A'
697
maruel77f720b2015-09-15 12:35:22 -0700698 url = '%s/user/task/%s' % (swarming, metadata['task_id'])
Marc-Antoine Ruel4e6b73d2014-10-03 18:00:05 -0400699 tag_header = 'Shard %d %s' % (shard_index, url)
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000700 tag_footer1 = 'End of shard %d' % (shard_index)
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400701 if metadata.get('state') == 'CANCELED':
702 tag_footer2 = ' Pending: %s CANCELED' % pending
703 elif metadata.get('state') == 'EXPIRED':
704 tag_footer2 = ' Pending: %s EXPIRED (lack of capacity)' % pending
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -0400705 elif metadata.get('state') in ('BOT_DIED', 'TIMED_OUT', 'KILLED'):
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400706 tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s %s' % (
707 pending, duration, bot_id, exit_code, metadata['state'])
708 else:
709 tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s' % (
710 pending, duration, bot_id, exit_code)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400711
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000712 tag_len = max(len(x) for x in [tag_header, tag_footer1, tag_footer2])
713 dash_pad = '+-%s-+' % ('-' * tag_len)
714 tag_header = '| %s |' % tag_header.ljust(tag_len)
715 tag_footer1 = '| %s |' % tag_footer1.ljust(tag_len)
716 tag_footer2 = '| %s |' % tag_footer2.ljust(tag_len)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400717
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000718 if include_stdout:
719 return '\n'.join([
720 dash_pad,
721 tag_header,
722 dash_pad,
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400723 (metadata.get('output') or '').rstrip(),
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000724 dash_pad,
725 tag_footer1,
726 tag_footer2,
727 dash_pad,
728 ])
729 else:
730 return '\n'.join([
731 dash_pad,
732 tag_header,
733 tag_footer2,
734 dash_pad,
735 ])
maruel@chromium.org0437a732013-08-27 16:05:52 +0000736
737
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700738def collect(
maruel0eb1d1b2015-10-02 14:48:21 -0700739 swarming, task_ids, timeout, decorate, print_status_updates,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000740 task_summary_json, task_output_dir, task_output_stdout,
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000741 include_perf, filepath_filter):
maruela5490782015-09-30 10:56:59 -0700742 """Retrieves results of a Swarming task.
743
744 Returns:
745 process exit code that should be returned to the user.
746 """
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000747
748 filter_cb = None
749 if filepath_filter:
750 filter_cb = re.compile(filepath_filter).match
751
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700752 # Collect summary JSON and output files (if task_output_dir is not None).
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000753 output_collector = TaskOutputCollector(
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000754 task_output_dir, task_output_stdout, len(task_ids), filter_cb)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700755
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700756 seen_shards = set()
maruela5490782015-09-30 10:56:59 -0700757 exit_code = None
Marc-Antoine Rueld59e8072014-10-21 18:54:45 -0400758 total_duration = 0
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700759 try:
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400760 for index, metadata in yield_results(
Marc-Antoine Ruel4e6b73d2014-10-03 18:00:05 -0400761 swarming, task_ids, timeout, None, print_status_updates,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000762 output_collector, include_perf,
763 (len(task_output_stdout) > 0),
764 ):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700765 seen_shards.add(index)
Vadim Shtayura473455a2014-05-14 15:22:35 -0700766
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400767 # Default to failure if there was no process that even started.
maruel77f720b2015-09-15 12:35:22 -0700768 shard_exit_code = metadata.get('exit_code')
769 if shard_exit_code:
maruela5490782015-09-30 10:56:59 -0700770 # It's encoded as a string, so bool('0') is True.
maruel77f720b2015-09-15 12:35:22 -0700771 shard_exit_code = int(shard_exit_code)
maruela5490782015-09-30 10:56:59 -0700772 if shard_exit_code or exit_code is None:
Marc-Antoine Ruel4e6b73d2014-10-03 18:00:05 -0400773 exit_code = shard_exit_code
maruel77f720b2015-09-15 12:35:22 -0700774 total_duration += metadata.get('duration', 0)
Vadim Shtayura473455a2014-05-14 15:22:35 -0700775
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700776 if decorate:
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000777 s = decorate_shard_output(
778 swarming, index, metadata,
779 "console" in task_output_stdout).encode(
780 'utf-8', 'replace')
leileied181762016-10-13 14:24:59 -0700781 print(s)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400782 if len(seen_shards) < len(task_ids):
783 print('')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700784 else:
maruel77f720b2015-09-15 12:35:22 -0700785 print('%s: %s %s' % (
786 metadata.get('bot_id', 'N/A'),
787 metadata['task_id'],
788 shard_exit_code))
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000789 if "console" in task_output_stdout and metadata['output']:
maruel77f720b2015-09-15 12:35:22 -0700790 output = metadata['output'].rstrip()
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400791 if output:
792 print(''.join(' %s\n' % l for l in output.splitlines()))
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700793 finally:
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700794 summary = output_collector.finalize()
795 if task_summary_json:
796 tools.write_json(task_summary_json, summary, False)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700797
Marc-Antoine Rueld59e8072014-10-21 18:54:45 -0400798 if decorate and total_duration:
799 print('Total duration: %.1fs' % total_duration)
800
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400801 if len(seen_shards) != len(task_ids):
802 missing_shards = [x for x in range(len(task_ids)) if x not in seen_shards]
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700803 print >> sys.stderr, ('Results from some shards are missing: %s' %
804 ', '.join(map(str, missing_shards)))
Vadim Shtayurac524f512014-05-15 09:54:56 -0700805 return 1
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700806
maruela5490782015-09-30 10:56:59 -0700807 return exit_code if exit_code is not None else 1
maruel@chromium.org0437a732013-08-27 16:05:52 +0000808
809
maruel77f720b2015-09-15 12:35:22 -0700810### API management.
811
812
813class APIError(Exception):
814 pass
815
816
817def endpoints_api_discovery_apis(host):
818 """Uses Cloud Endpoints' API Discovery Service to returns metadata about all
819 the APIs exposed by a host.
820
821 https://developers.google.com/discovery/v1/reference/apis/list
822 """
maruel380e3262016-08-31 16:10:06 -0700823 # Uses the real Cloud Endpoints. This needs to be fixed once the Cloud
824 # Endpoints version is turned down.
maruel77f720b2015-09-15 12:35:22 -0700825 data = net.url_read_json(host + '/_ah/api/discovery/v1/apis')
826 if data is None:
827 raise APIError('Failed to discover APIs on %s' % host)
828 out = {}
829 for api in data['items']:
830 if api['id'] == 'discovery:v1':
831 continue
832 # URL is of the following form:
833 # url = host + (
834 # '/_ah/api/discovery/v1/apis/%s/%s/rest' % (api['id'], api['version'])
835 api_data = net.url_read_json(api['discoveryRestUrl'])
836 if api_data is None:
837 raise APIError('Failed to discover %s on %s' % (api['id'], host))
838 out[api['id']] = api_data
839 return out
840
841
maruelaf6b06c2017-06-08 06:26:53 -0700842def get_yielder(base_url, limit):
843 """Returns the first query and a function that yields following items."""
844 CHUNK_SIZE = 250
845
846 url = base_url
847 if limit:
848 url += '%slimit=%d' % ('&' if '?' in url else '?', min(CHUNK_SIZE, limit))
849 data = net.url_read_json(url)
850 if data is None:
851 # TODO(maruel): Do basic diagnostic.
852 raise Failure('Failed to access %s' % url)
853 org_cursor = data.pop('cursor', None)
854 org_total = len(data.get('items') or [])
855 logging.info('get_yielder(%s) returning %d items', base_url, org_total)
856 if not org_cursor or not org_total:
857 # This is not an iterable resource.
858 return data, lambda: []
859
860 def yielder():
861 cursor = org_cursor
862 total = org_total
863 # Some items support cursors. Try to get automatically if cursors are needed
864 # by looking at the 'cursor' items.
865 while cursor and (not limit or total < limit):
866 merge_char = '&' if '?' in base_url else '?'
867 url = base_url + '%scursor=%s' % (merge_char, urllib.quote(cursor))
868 if limit:
869 url += '&limit=%d' % min(CHUNK_SIZE, limit - total)
870 new = net.url_read_json(url)
871 if new is None:
872 raise Failure('Failed to access %s' % url)
873 cursor = new.get('cursor')
874 new_items = new.get('items')
875 nb_items = len(new_items or [])
876 total += nb_items
877 logging.info('get_yielder(%s) yielding %d items', base_url, nb_items)
878 yield new_items
879
880 return data, yielder
881
882
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500883### Commands.
884
885
886def abort_task(_swarming, _manifest):
887 """Given a task manifest that was triggered, aborts its execution."""
888 # TODO(vadimsh): No supported by the server yet.
889
890
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400891def add_filter_options(parser):
maruel681d6802017-01-17 16:56:03 -0800892 parser.filter_group = optparse.OptionGroup(parser, 'Bot selection')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -0500893 parser.filter_group.add_option(
Marc-Antoine Ruelb39e8cf2014-01-20 10:39:31 -0500894 '-d', '--dimension', default=[], action='append', nargs=2,
Marc-Antoine Ruel92f32422013-11-06 18:12:13 -0500895 dest='dimensions', metavar='FOO bar',
896 help='dimension to filter on')
Brad Hallf78187a2018-10-19 17:08:55 +0000897 parser.filter_group.add_option(
898 '--optional-dimension', default=[], action='append', nargs=3,
899 dest='optional_dimensions', metavar='key value expiration',
900 help='optional dimensions which will result in additional task slices ')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -0500901 parser.add_option_group(parser.filter_group)
902
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400903
Brad Hallf78187a2018-10-19 17:08:55 +0000904def _validate_filter_option(parser, key, value, expiration, argname):
905 if ':' in key:
906 parser.error('%s key cannot contain ":"' % argname)
907 if key.strip() != key:
908 parser.error('%s key has whitespace' % argname)
909 if not key:
910 parser.error('%s key is empty' % argname)
911
912 if value.strip() != value:
913 parser.error('%s value has whitespace' % argname)
914 if not value:
915 parser.error('%s value is empty' % argname)
916
917 if expiration is not None:
918 try:
919 expiration = int(expiration)
920 except ValueError:
921 parser.error('%s expiration is not an integer' % argname)
922 if expiration <= 0:
923 parser.error('%s expiration should be positive' % argname)
924 if expiration % 60 != 0:
925 parser.error('%s expiration is not divisible by 60' % argname)
926
927
maruelaf6b06c2017-06-08 06:26:53 -0700928def process_filter_options(parser, options):
929 for key, value in options.dimensions:
Brad Hallf78187a2018-10-19 17:08:55 +0000930 _validate_filter_option(parser, key, value, None, 'dimension')
931 for key, value, exp in options.optional_dimensions:
932 _validate_filter_option(parser, key, value, exp, 'optional-dimension')
maruelaf6b06c2017-06-08 06:26:53 -0700933 options.dimensions.sort()
934
935
Vadim Shtayurab450c602014-05-12 19:23:25 -0700936def add_sharding_options(parser):
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -0400937 parser.sharding_group = optparse.OptionGroup(parser, 'Sharding options')
Vadim Shtayurab450c602014-05-12 19:23:25 -0700938 parser.sharding_group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700939 '--shards', type='int', default=1, metavar='NUMBER',
Vadim Shtayurab450c602014-05-12 19:23:25 -0700940 help='Number of shards to trigger and collect.')
941 parser.add_option_group(parser.sharding_group)
942
943
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400944def add_trigger_options(parser):
945 """Adds all options to trigger a task on Swarming."""
Marc-Antoine Ruelf7d737d2014-12-10 15:36:29 -0500946 isolateserver.add_isolate_server_options(parser)
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400947 add_filter_options(parser)
948
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -0400949 group = optparse.OptionGroup(parser, 'TaskSlice properties')
maruel681d6802017-01-17 16:56:03 -0800950 group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700951 '-s', '--isolated', metavar='HASH',
Marc-Antoine Ruel185ded42015-01-28 20:49:18 -0500952 help='Hash of the .isolated to grab from the isolate server')
maruel681d6802017-01-17 16:56:03 -0800953 group.add_option(
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -0500954 '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
Vadim Shtayurab450c602014-05-12 19:23:25 -0700955 help='Environment variables to set')
maruel681d6802017-01-17 16:56:03 -0800956 group.add_option(
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800957 '--env-prefix', default=[], action='append', nargs=2,
958 metavar='VAR local/path',
959 help='Prepend task-relative `local/path` to the task\'s VAR environment '
960 'variable using os-appropriate pathsep character. Can be specified '
961 'multiple times for the same VAR to add multiple paths.')
962 group.add_option(
Marc-Antoine Ruel02196392014-10-17 16:29:43 -0400963 '--idempotent', action='store_true', default=False,
964 help='When set, the server will actively try to find a previous task '
965 'with the same parameter and return this result instead if possible')
maruel681d6802017-01-17 16:56:03 -0800966 group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700967 '--secret-bytes-path', metavar='FILE',
Stephen Martinisf391c772019-02-01 01:22:12 +0000968 help='The optional path to a file containing the secret_bytes to use '
969 'with this task.')
maruel681d6802017-01-17 16:56:03 -0800970 group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700971 '--hard-timeout', type='int', default=60*60, metavar='SECS',
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400972 help='Seconds to allow the task to complete.')
maruel681d6802017-01-17 16:56:03 -0800973 group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700974 '--io-timeout', type='int', default=20*60, metavar='SECS',
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400975 help='Seconds to allow the task to be silent.')
maruel681d6802017-01-17 16:56:03 -0800976 group.add_option(
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500977 '--raw-cmd', action='store_true', default=False,
978 help='When set, the command after -- is used as-is without run_isolated. '
maruel0a25f6c2017-05-10 10:43:23 -0700979 'In this case, the .isolated file is expected to not have a command')
maruel681d6802017-01-17 16:56:03 -0800980 group.add_option(
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -0500981 '--relative-cwd',
982 help='Ignore the isolated \'relative_cwd\' and use this one instead; '
983 'requires --raw-cmd')
984 group.add_option(
maruel5475ba62017-05-31 15:35:47 -0700985 '--cipd-package', action='append', default=[], metavar='PKG',
986 help='CIPD packages to install on the Swarming bot. Uses the format: '
borenet02f772b2016-06-22 12:42:19 -0700987 'path:package_name:version')
maruel681d6802017-01-17 16:56:03 -0800988 group.add_option(
989 '--named-cache', action='append', nargs=2, default=[],
maruel5475ba62017-05-31 15:35:47 -0700990 metavar='NAME RELPATH',
maruel681d6802017-01-17 16:56:03 -0800991 help='"<name> <relpath>" items to keep a persistent bot managed cache')
992 group.add_option(
vadimsh93d167c2016-09-13 11:31:51 -0700993 '--service-account',
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700994 help='Email of a service account to run the task as, or literal "bot" '
995 'string to indicate that the task should use the same account the '
996 'bot itself is using to authenticate to Swarming. Don\'t use task '
997 'service accounts if not given (default).')
maruel681d6802017-01-17 16:56:03 -0800998 group.add_option(
Robert Iannuccifafa7352018-06-13 17:08:17 +0000999 '--pool-task-template',
1000 choices=('AUTO', 'CANARY_PREFER', 'CANARY_NEVER', 'SKIP'),
1001 default='AUTO',
1002 help='Set how you want swarming to apply the pool\'s TaskTemplate. '
1003 'By default, the pool\'s TaskTemplate is automatically selected, '
1004 'according the pool configuration on the server. Choices are: '
1005 'AUTO, CANARY_PREFER, CANARY_NEVER, and SKIP (default: AUTO).')
1006 group.add_option(
maruel5475ba62017-05-31 15:35:47 -07001007 '-o', '--output', action='append', default=[], metavar='PATH',
1008 help='A list of files to return in addition to those written to '
1009 '${ISOLATED_OUTDIR}. An error will occur if a file specified by'
1010 'this option is also written directly to ${ISOLATED_OUTDIR}.')
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001011 group.add_option(
1012 '--wait-for-capacity', action='store_true', default=False,
1013 help='Instructs to leave the task PENDING even if there\'s no known bot '
1014 'that could run this task, otherwise the task will be denied with '
1015 'NO_RESOURCE')
maruel681d6802017-01-17 16:56:03 -08001016 parser.add_option_group(group)
1017
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001018 group = optparse.OptionGroup(parser, 'TaskRequest details')
maruel681d6802017-01-17 16:56:03 -08001019 group.add_option(
Marc-Antoine Ruel486c9b52018-07-23 19:30:47 +00001020 '--priority', type='int', default=200,
maruel681d6802017-01-17 16:56:03 -08001021 help='The lower value, the more important the task is')
1022 group.add_option(
maruel5475ba62017-05-31 15:35:47 -07001023 '-T', '--task-name', metavar='NAME',
maruel681d6802017-01-17 16:56:03 -08001024 help='Display name of the task. Defaults to '
1025 '<base_name>/<dimensions>/<isolated hash>/<timestamp> if an '
1026 'isolated file is provided, if a hash is provided, it defaults to '
1027 '<user>/<dimensions>/<isolated hash>/<timestamp>')
1028 group.add_option(
maruel5475ba62017-05-31 15:35:47 -07001029 '--tags', action='append', default=[], metavar='FOO:BAR',
maruel681d6802017-01-17 16:56:03 -08001030 help='Tags to assign to the task.')
1031 group.add_option(
1032 '--user', default='',
1033 help='User associated with the task. Defaults to authenticated user on '
1034 'the server.')
1035 group.add_option(
maruel5475ba62017-05-31 15:35:47 -07001036 '--expiration', type='int', default=6*60*60, metavar='SECS',
maruel681d6802017-01-17 16:56:03 -08001037 help='Seconds to allow the task to be pending for a bot to run before '
1038 'this task request expires.')
1039 group.add_option(
1040 '--deadline', type='int', dest='expiration',
1041 help=optparse.SUPPRESS_HELP)
1042 parser.add_option_group(group)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001043
1044
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001045def process_trigger_options(parser, options, args):
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001046 """Processes trigger options and does preparatory steps.
1047
1048 Returns:
1049 NewTaskRequest instance.
1050 """
maruelaf6b06c2017-06-08 06:26:53 -07001051 process_filter_options(parser, options)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001052 options.env = dict(options.env)
maruel0a25f6c2017-05-10 10:43:23 -07001053 if args and args[0] == '--':
1054 args = args[1:]
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001055
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001056 if not options.dimensions:
1057 parser.error('Please at least specify one --dimension')
Marc-Antoine Ruel33d198c2018-11-27 21:12:16 +00001058 if not any(k == 'pool' for k, _v in options.dimensions):
1059 parser.error('You must specify --dimension pool <value>')
maruel0a25f6c2017-05-10 10:43:23 -07001060 if not all(len(t.split(':', 1)) == 2 for t in options.tags):
1061 parser.error('--tags must be in the format key:value')
1062 if options.raw_cmd and not args:
1063 parser.error(
1064 'Arguments with --raw-cmd should be passed after -- as command '
1065 'delimiter.')
1066 if options.isolate_server and not options.namespace:
1067 parser.error(
1068 '--namespace must be a valid value when --isolate-server is used')
1069 if not options.isolated and not options.raw_cmd:
1070 parser.error('Specify at least one of --raw-cmd or --isolated or both')
1071
1072 # Isolated
1073 # --isolated is required only if --raw-cmd wasn't provided.
1074 # TODO(maruel): --isolate-server may be optional as Swarming may have its own
1075 # preferred server.
1076 isolateserver.process_isolate_server_options(
1077 parser, options, False, not options.raw_cmd)
1078 inputs_ref = None
1079 if options.isolate_server:
1080 inputs_ref = FilesRef(
1081 isolated=options.isolated,
1082 isolatedserver=options.isolate_server,
1083 namespace=options.namespace)
1084
1085 # Command
1086 command = None
1087 extra_args = None
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001088 if options.raw_cmd:
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001089 command = args
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001090 if options.relative_cwd:
1091 a = os.path.normpath(os.path.abspath(options.relative_cwd))
1092 if not a.startswith(os.getcwd()):
1093 parser.error(
1094 '--relative-cwd must not try to escape the working directory')
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001095 else:
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001096 if options.relative_cwd:
1097 parser.error('--relative-cwd requires --raw-cmd')
maruel0a25f6c2017-05-10 10:43:23 -07001098 extra_args = args
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001099
maruel0a25f6c2017-05-10 10:43:23 -07001100 # CIPD
borenet02f772b2016-06-22 12:42:19 -07001101 cipd_packages = []
1102 for p in options.cipd_package:
1103 split = p.split(':', 2)
1104 if len(split) != 3:
1105 parser.error('CIPD packages must take the form: path:package:version')
1106 cipd_packages.append(CipdPackage(
1107 package_name=split[1],
1108 path=split[0],
1109 version=split[2]))
1110 cipd_input = None
1111 if cipd_packages:
1112 cipd_input = CipdInput(
1113 client_package=None,
1114 packages=cipd_packages,
1115 server=None)
1116
maruel0a25f6c2017-05-10 10:43:23 -07001117 # Secrets
iannuccidc80dfb2016-10-28 12:50:20 -07001118 secret_bytes = None
1119 if options.secret_bytes_path:
Marc-Antoine Ruel5c98fa72018-05-18 12:19:59 -04001120 with open(options.secret_bytes_path, 'rb') as f:
iannuccidc80dfb2016-10-28 12:50:20 -07001121 secret_bytes = f.read().encode('base64')
1122
maruel0a25f6c2017-05-10 10:43:23 -07001123 # Named caches
maruel681d6802017-01-17 16:56:03 -08001124 caches = [
1125 {u'name': unicode(i[0]), u'path': unicode(i[1])}
1126 for i in options.named_cache
1127 ]
maruel0a25f6c2017-05-10 10:43:23 -07001128
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001129 env_prefixes = {}
1130 for k, v in options.env_prefix:
1131 env_prefixes.setdefault(k, []).append(v)
1132
Brad Hallf78187a2018-10-19 17:08:55 +00001133 # Get dimensions into the key/value format we can manipulate later.
1134 orig_dims = [
1135 {'key': key, 'value': value} for key, value in options.dimensions]
1136 orig_dims.sort(key=lambda x: (x['key'], x['value']))
1137
1138 # Construct base properties that we will use for all the slices, adding in
1139 # optional dimensions for the fallback slices.
maruel77f720b2015-09-15 12:35:22 -07001140 properties = TaskProperties(
maruel681d6802017-01-17 16:56:03 -08001141 caches=caches,
borenet02f772b2016-06-22 12:42:19 -07001142 cipd_input=cipd_input,
maruel0a25f6c2017-05-10 10:43:23 -07001143 command=command,
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001144 relative_cwd=options.relative_cwd,
Brad Hallf78187a2018-10-19 17:08:55 +00001145 dimensions=orig_dims,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001146 env=options.env,
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001147 env_prefixes=[StringListPair(k, v) for k, v in env_prefixes.iteritems()],
maruel77f720b2015-09-15 12:35:22 -07001148 execution_timeout_secs=options.hard_timeout,
maruel0a25f6c2017-05-10 10:43:23 -07001149 extra_args=extra_args,
maruel77f720b2015-09-15 12:35:22 -07001150 grace_period_secs=30,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001151 idempotent=options.idempotent,
maruel77f720b2015-09-15 12:35:22 -07001152 inputs_ref=inputs_ref,
aludwincc5524e2016-10-28 10:25:24 -07001153 io_timeout_secs=options.io_timeout,
iannuccidc80dfb2016-10-28 12:50:20 -07001154 outputs=options.output,
1155 secret_bytes=secret_bytes)
Brad Hallf78187a2018-10-19 17:08:55 +00001156
1157 slices = []
1158
1159 # Group the optional dimensions by expiration.
1160 dims_by_exp = {}
1161 for key, value, exp_secs in options.optional_dimensions:
1162 dims_by_exp.setdefault(int(exp_secs), []).append(
1163 {'key': key, 'value': value})
1164
1165 # Create the optional slices with expiration deltas, we fix up the properties
1166 # below.
1167 last_exp = 0
1168 for expiration_secs in sorted(dims_by_exp):
1169 t = TaskSlice(
1170 expiration_secs=expiration_secs - last_exp,
1171 properties=properties,
1172 wait_for_capacity=False)
1173 slices.append(t)
1174 last_exp = expiration_secs
1175
1176 # Add back in the default slice (the last one).
1177 exp = max(int(options.expiration) - last_exp, 60)
1178 base_task_slice = TaskSlice(
1179 expiration_secs=exp,
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001180 properties=properties,
1181 wait_for_capacity=options.wait_for_capacity)
Brad Hallf78187a2018-10-19 17:08:55 +00001182 slices.append(base_task_slice)
1183
Brad Hall7f463e62018-11-16 16:13:30 +00001184 # Add optional dimensions to the task slices, replacing a dimension that
1185 # has the same key if it is a dimension where repeating isn't valid (otherwise
1186 # we append it). Currently the only dimension we can repeat is "caches"; the
1187 # rest (os, cpu, etc) shouldn't be repeated.
Brad Hallf78187a2018-10-19 17:08:55 +00001188 extra_dims = []
Brad Hall7f463e62018-11-16 16:13:30 +00001189 for i, (_, kvs) in enumerate(sorted(dims_by_exp.iteritems(), reverse=True)):
Brad Hallf78187a2018-10-19 17:08:55 +00001190 dims = list(orig_dims)
Brad Hall7f463e62018-11-16 16:13:30 +00001191 # Replace or append the key/value pairs for this expiration in extra_dims;
1192 # we keep extra_dims around because we are iterating backwards and filling
1193 # in slices with shorter expirations. Dimensions expire as time goes on so
1194 # the slices that expire earlier will generally have more dimensions.
1195 for kv in kvs:
1196 if kv['key'] == 'caches':
1197 extra_dims.append(kv)
1198 else:
1199 extra_dims = [x for x in extra_dims if x['key'] != kv['key']] + [kv]
1200 # Then, add all the optional dimensions to the original dimension set, again
1201 # replacing if needed.
1202 for kv in extra_dims:
1203 if kv['key'] == 'caches':
1204 dims.append(kv)
1205 else:
1206 dims = [x for x in dims if x['key'] != kv['key']] + [kv]
Brad Hallf78187a2018-10-19 17:08:55 +00001207 dims.sort(key=lambda x: (x['key'], x['value']))
1208 slice_properties = properties._replace(dimensions=dims)
1209 slices[-2 - i] = slices[-2 - i]._replace(properties=slice_properties)
1210
maruel77f720b2015-09-15 12:35:22 -07001211 return NewTaskRequest(
maruel0a25f6c2017-05-10 10:43:23 -07001212 name=default_task_name(options),
maruel77f720b2015-09-15 12:35:22 -07001213 parent_task_id=os.environ.get('SWARMING_TASK_ID', ''),
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001214 priority=options.priority,
Brad Hallf78187a2018-10-19 17:08:55 +00001215 task_slices=slices,
Vadim Shtayura2d83a942017-08-14 17:41:24 -07001216 service_account=options.service_account,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001217 tags=options.tags,
Robert Iannuccifafa7352018-06-13 17:08:17 +00001218 user=options.user,
1219 pool_task_template=options.pool_task_template)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001220
1221
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001222class TaskOutputStdoutOption(optparse.Option):
1223 """Where to output the each task's console output (stderr/stdout).
1224
1225 The output will be;
1226 none - not be downloaded.
1227 json - stored in summary.json file *only*.
1228 console - shown on stdout *only*.
1229 all - stored in summary.json and shown on stdout.
1230 """
1231
1232 choices = ['all', 'json', 'console', 'none']
1233
1234 def __init__(self, *args, **kw):
1235 optparse.Option.__init__(
1236 self,
1237 *args,
1238 choices=self.choices,
Marc-Antoine Ruel28488842017-09-12 18:09:17 -04001239 default=['console', 'json'],
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001240 help=re.sub('\s\s*', ' ', self.__doc__),
1241 **kw)
1242
1243 def convert_value(self, opt, value):
1244 if value not in self.choices:
1245 raise optparse.OptionValueError("%s must be one of %s not %r" % (
1246 self.get_opt_string(), self.choices, value))
1247 stdout_to = []
1248 if value == 'all':
Marc-Antoine Ruel28488842017-09-12 18:09:17 -04001249 stdout_to = ['console', 'json']
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001250 elif value != 'none':
1251 stdout_to = [value]
1252 return stdout_to
1253
1254
maruel@chromium.org0437a732013-08-27 16:05:52 +00001255def add_collect_options(parser):
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001256 parser.server_group.add_option(
Marc-Antoine Ruele831f052018-04-20 15:01:03 -04001257 '-t', '--timeout', type='float', default=0.,
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001258 help='Timeout to wait for result, set to -1 for no timeout and get '
1259 'current state; defaults to waiting until the task completes')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001260 parser.group_logging.add_option(
1261 '--decorate', action='store_true', help='Decorate output')
Vadim Shtayura86a2cef2014-04-18 11:13:39 -07001262 parser.group_logging.add_option(
1263 '--print-status-updates', action='store_true',
1264 help='Print periodic status updates')
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001265 parser.task_output_group = optparse.OptionGroup(parser, 'Task output')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001266 parser.task_output_group.add_option(
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001267 '--task-summary-json',
1268 metavar='FILE',
1269 help='Dump a summary of task results to this file as json. It contains '
1270 'only shards statuses as know to server directly. Any output files '
1271 'emitted by the task can be collected by using --task-output-dir')
1272 parser.task_output_group.add_option(
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001273 '--task-output-dir',
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001274 metavar='DIR',
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001275 help='Directory to put task results into. When the task finishes, this '
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001276 'directory contains per-shard directory with output files produced '
1277 'by shards: <task-output-dir>/<zero-based-shard-index>/.')
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001278 parser.task_output_group.add_option(TaskOutputStdoutOption(
Marc-Antoine Ruel28488842017-09-12 18:09:17 -04001279 '--task-output-stdout'))
maruel9531ce02016-04-13 06:11:23 -07001280 parser.task_output_group.add_option(
Takuto Ikuta1e6072c2018-11-06 20:42:43 +00001281 '--filepath-filter',
1282 help='This is regexp filter used to specify downloaded filepath when '
1283 'collecting isolated output.')
1284 parser.task_output_group.add_option(
maruel9531ce02016-04-13 06:11:23 -07001285 '--perf', action='store_true', default=False,
1286 help='Includes performance statistics')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001287 parser.add_option_group(parser.task_output_group)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001288
1289
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001290def process_collect_options(parser, options):
1291 # Only negative -1 is allowed, disallow other negative values.
1292 if options.timeout != -1 and options.timeout < 0:
1293 parser.error('Invalid --timeout value')
1294
1295
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001296@subcommand.usage('bots...')
1297def CMDbot_delete(parser, args):
1298 """Forcibly deletes bots from the Swarming server."""
1299 parser.add_option(
1300 '-f', '--force', action='store_true',
1301 help='Do not prompt for confirmation')
1302 options, args = parser.parse_args(args)
1303 if not args:
maruelfd0a90c2016-06-10 11:51:10 -07001304 parser.error('Please specify bots to delete')
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001305
1306 bots = sorted(args)
1307 if not options.force:
1308 print('Delete the following bots?')
1309 for bot in bots:
1310 print(' %s' % bot)
1311 if raw_input('Continue? [y/N] ') not in ('y', 'Y'):
1312 print('Goodbye.')
1313 return 1
1314
1315 result = 0
1316 for bot in bots:
smut281c3902018-05-30 17:50:05 -07001317 url = '%s/_ah/api/swarming/v1/bot/%s/delete' % (options.swarming, bot)
vadimshe4c0e242015-09-30 11:53:54 -07001318 if net.url_read_json(url, data={}, method='POST') is None:
1319 print('Deleting %s failed. Probably already gone' % bot)
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001320 result = 1
1321 return result
1322
1323
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001324def CMDbots(parser, args):
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001325 """Returns information about the bots connected to the Swarming server."""
1326 add_filter_options(parser)
1327 parser.filter_group.add_option(
Marc-Antoine Ruel28083112014-03-13 16:34:04 -04001328 '--dead-only', action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001329 help='Filter out bots alive, useful to reap them and reimage broken bots')
Marc-Antoine Ruel28083112014-03-13 16:34:04 -04001330 parser.filter_group.add_option(
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001331 '-k', '--keep-dead', action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001332 help='Keep both dead and alive bots')
1333 parser.filter_group.add_option(
1334 '--busy', action='store_true', help='Keep only busy bots')
1335 parser.filter_group.add_option(
1336 '--idle', action='store_true', help='Keep only idle bots')
1337 parser.filter_group.add_option(
1338 '--mp', action='store_true',
1339 help='Keep only Machine Provider managed bots')
1340 parser.filter_group.add_option(
1341 '--non-mp', action='store_true',
1342 help='Keep only non Machine Provider managed bots')
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001343 parser.filter_group.add_option(
1344 '-b', '--bare', action='store_true',
Marc-Antoine Ruele7b00162014-03-12 16:59:01 -04001345 help='Do not print out dimensions')
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001346 options, args = parser.parse_args(args)
maruelaf6b06c2017-06-08 06:26:53 -07001347 process_filter_options(parser, options)
Marc-Antoine Ruel28083112014-03-13 16:34:04 -04001348
1349 if options.keep_dead and options.dead_only:
maruelaf6b06c2017-06-08 06:26:53 -07001350 parser.error('Use only one of --keep-dead or --dead-only')
1351 if options.busy and options.idle:
1352 parser.error('Use only one of --busy or --idle')
1353 if options.mp and options.non_mp:
1354 parser.error('Use only one of --mp or --non-mp')
Vadim Shtayura6b555c12014-07-23 16:22:18 -07001355
smut281c3902018-05-30 17:50:05 -07001356 url = options.swarming + '/_ah/api/swarming/v1/bots/list?'
maruelaf6b06c2017-06-08 06:26:53 -07001357 values = []
1358 if options.dead_only:
1359 values.append(('is_dead', 'TRUE'))
1360 elif options.keep_dead:
1361 values.append(('is_dead', 'NONE'))
1362 else:
1363 values.append(('is_dead', 'FALSE'))
Marc-Antoine Ruelc6c579e2014-09-08 18:43:45 -04001364
maruelaf6b06c2017-06-08 06:26:53 -07001365 if options.busy:
1366 values.append(('is_busy', 'TRUE'))
1367 elif options.idle:
1368 values.append(('is_busy', 'FALSE'))
1369 else:
1370 values.append(('is_busy', 'NONE'))
1371
1372 if options.mp:
1373 values.append(('is_mp', 'TRUE'))
1374 elif options.non_mp:
1375 values.append(('is_mp', 'FALSE'))
1376 else:
1377 values.append(('is_mp', 'NONE'))
1378
1379 for key, value in options.dimensions:
1380 values.append(('dimensions', '%s:%s' % (key, value)))
1381 url += urllib.urlencode(values)
1382 try:
1383 data, yielder = get_yielder(url, 0)
1384 bots = data.get('items') or []
1385 for items in yielder():
1386 if items:
1387 bots.extend(items)
1388 except Failure as e:
1389 sys.stderr.write('\n%s\n' % e)
1390 return 1
maruel77f720b2015-09-15 12:35:22 -07001391 for bot in natsort.natsorted(bots, key=lambda x: x['bot_id']):
maruelaf6b06c2017-06-08 06:26:53 -07001392 print bot['bot_id']
1393 if not options.bare:
1394 dimensions = {i['key']: i.get('value') for i in bot.get('dimensions', {})}
1395 print ' %s' % json.dumps(dimensions, sort_keys=True)
1396 if bot.get('task_id'):
1397 print ' task: %s' % bot['task_id']
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001398 return 0
1399
1400
maruelfd0a90c2016-06-10 11:51:10 -07001401@subcommand.usage('task_id')
1402def CMDcancel(parser, args):
1403 """Cancels a task."""
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001404 parser.add_option(
1405 '-k', '--kill-running', action='store_true', default=False,
1406 help='Kill the task even if it was running')
maruelfd0a90c2016-06-10 11:51:10 -07001407 options, args = parser.parse_args(args)
1408 if not args:
1409 parser.error('Please specify the task to cancel')
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001410 data = {'kill_running': options.kill_running}
maruelfd0a90c2016-06-10 11:51:10 -07001411 for task_id in args:
smut281c3902018-05-30 17:50:05 -07001412 url = '%s/_ah/api/swarming/v1/task/%s/cancel' % (options.swarming, task_id)
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001413 resp = net.url_read_json(url, data=data, method='POST')
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001414 if resp is None:
maruelfd0a90c2016-06-10 11:51:10 -07001415 print('Deleting %s failed. Probably already gone' % task_id)
1416 return 1
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001417 logging.info('%s', resp)
maruelfd0a90c2016-06-10 11:51:10 -07001418 return 0
1419
1420
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001421@subcommand.usage('--json file | task_id...')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001422def CMDcollect(parser, args):
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001423 """Retrieves results of one or multiple Swarming task by its ID.
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001424
1425 The result can be in multiple part if the execution was sharded. It can
1426 potentially have retries.
1427 """
1428 add_collect_options(parser)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001429 parser.add_option(
1430 '-j', '--json',
1431 help='Load the task ids from .json as saved by trigger --dump-json')
maruel77f720b2015-09-15 12:35:22 -07001432 options, args = parser.parse_args(args)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001433 process_collect_options(parser, options)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001434 if not args and not options.json:
1435 parser.error('Must specify at least one task id or --json.')
1436 if args and options.json:
1437 parser.error('Only use one of task id or --json.')
1438
1439 if options.json:
maruel1ceb3872015-10-14 06:10:44 -07001440 options.json = unicode(os.path.abspath(options.json))
Marc-Antoine Ruel9025a782015-03-17 16:42:59 -04001441 try:
maruel1ceb3872015-10-14 06:10:44 -07001442 with fs.open(options.json, 'rb') as f:
maruel71c61c82016-02-22 06:52:05 -08001443 data = json.load(f)
1444 except (IOError, ValueError):
1445 parser.error('Failed to open %s' % options.json)
1446 try:
1447 tasks = sorted(
1448 data['tasks'].itervalues(), key=lambda x: x['shard_index'])
1449 args = [t['task_id'] for t in tasks]
1450 except (KeyError, TypeError):
1451 parser.error('Failed to process %s' % options.json)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001452 if not options.timeout:
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -04001453 # Take in account all the task slices.
1454 offset = 0
1455 for s in data['request']['task_slices']:
1456 m = (offset + s['properties']['execution_timeout_secs'] +
1457 s['expiration_secs'])
1458 if m > options.timeout:
1459 options.timeout = m
1460 offset += s['expiration_secs']
Marc-Antoine Ruel9fc42612018-04-20 08:34:22 -04001461 options.timeout += 10.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001462 else:
1463 valid = frozenset('0123456789abcdef')
1464 if any(not valid.issuperset(task_id) for task_id in args):
1465 parser.error('Task ids are 0-9a-f.')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001466
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001467 try:
1468 return collect(
1469 options.swarming,
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001470 args,
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001471 options.timeout,
1472 options.decorate,
1473 options.print_status_updates,
1474 options.task_summary_json,
maruel9531ce02016-04-13 06:11:23 -07001475 options.task_output_dir,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001476 options.task_output_stdout,
Takuto Ikuta1e6072c2018-11-06 20:42:43 +00001477 options.perf,
1478 options.filepath_filter)
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001479 except Failure:
1480 on_error.report(None)
1481 return 1
1482
1483
maruel77f720b2015-09-15 12:35:22 -07001484@subcommand.usage('[method name]')
Marc-Antoine Ruel833f5eb2018-04-25 16:49:40 -04001485def CMDpost(parser, args):
1486 """Sends a JSON RPC POST to one API endpoint and prints out the raw result.
1487
1488 Input data must be sent to stdin, result is printed to stdout.
1489
1490 If HTTP response code >= 400, returns non-zero.
1491 """
1492 options, args = parser.parse_args(args)
1493 if len(args) != 1:
1494 parser.error('Must specify only API name')
smut281c3902018-05-30 17:50:05 -07001495 url = options.swarming + '/_ah/api/swarming/v1/' + args[0]
Marc-Antoine Ruel833f5eb2018-04-25 16:49:40 -04001496 data = sys.stdin.read()
1497 try:
1498 resp = net.url_read(url, data=data, method='POST')
1499 except net.TimeoutError:
1500 sys.stderr.write('Timeout!\n')
1501 return 1
1502 if not resp:
1503 sys.stderr.write('No response!\n')
1504 return 1
1505 sys.stdout.write(resp)
1506 return 0
1507
1508
1509@subcommand.usage('[method name]')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001510def CMDquery(parser, args):
maruel77f720b2015-09-15 12:35:22 -07001511 """Returns raw JSON information via an URL endpoint. Use 'query-list' to
1512 gather the list of API methods from the server.
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001513
1514 Examples:
maruelaf6b06c2017-06-08 06:26:53 -07001515 Raw task request and results:
1516 swarming.py query -S server-url.com task/123456/request
1517 swarming.py query -S server-url.com task/123456/result
1518
maruel77f720b2015-09-15 12:35:22 -07001519 Listing all bots:
maruel84e77aa2015-10-21 06:37:24 -07001520 swarming.py query -S server-url.com bots/list
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001521
maruelaf6b06c2017-06-08 06:26:53 -07001522 Listing last 10 tasks on a specific bot named 'bot1':
1523 swarming.py query -S server-url.com --limit 10 bot/bot1/tasks
maruel84e77aa2015-10-21 06:37:24 -07001524
maruelaf6b06c2017-06-08 06:26:53 -07001525 Listing last 10 tasks with tags os:Ubuntu-14.04 and pool:Chrome. Note that
maruel84e77aa2015-10-21 06:37:24 -07001526 quoting is important!:
1527 swarming.py query -S server-url.com --limit 10 \\
maruelaf6b06c2017-06-08 06:26:53 -07001528 'tasks/list?tags=os:Ubuntu-14.04&tags=pool:Chrome'
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001529 """
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001530 parser.add_option(
1531 '-L', '--limit', type='int', default=200,
1532 help='Limit to enforce on limitless items (like number of tasks); '
1533 'default=%default')
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001534 parser.add_option(
1535 '--json', help='Path to JSON output file (otherwise prints to stdout)')
maruel77f720b2015-09-15 12:35:22 -07001536 parser.add_option(
1537 '--progress', action='store_true',
1538 help='Prints a dot at each request to show progress')
1539 options, args = parser.parse_args(args)
marueld8aba222015-09-03 12:21:19 -07001540 if len(args) != 1:
maruel77f720b2015-09-15 12:35:22 -07001541 parser.error(
1542 'Must specify only method name and optionally query args properly '
1543 'escaped.')
smut281c3902018-05-30 17:50:05 -07001544 base_url = options.swarming + '/_ah/api/swarming/v1/' + args[0]
maruelaf6b06c2017-06-08 06:26:53 -07001545 try:
1546 data, yielder = get_yielder(base_url, options.limit)
1547 for items in yielder():
1548 if items:
1549 data['items'].extend(items)
maruel77f720b2015-09-15 12:35:22 -07001550 if options.progress:
maruelaf6b06c2017-06-08 06:26:53 -07001551 sys.stderr.write('.')
1552 sys.stderr.flush()
1553 except Failure as e:
1554 sys.stderr.write('\n%s\n' % e)
1555 return 1
maruel77f720b2015-09-15 12:35:22 -07001556 if options.progress:
maruelaf6b06c2017-06-08 06:26:53 -07001557 sys.stderr.write('\n')
1558 sys.stderr.flush()
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001559 if options.json:
maruel1ceb3872015-10-14 06:10:44 -07001560 options.json = unicode(os.path.abspath(options.json))
1561 tools.write_json(options.json, data, True)
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001562 else:
Marc-Antoine Ruelcda90ee2015-03-23 15:13:20 -04001563 try:
maruel77f720b2015-09-15 12:35:22 -07001564 tools.write_json(sys.stdout, data, False)
Marc-Antoine Ruelcda90ee2015-03-23 15:13:20 -04001565 sys.stdout.write('\n')
1566 except IOError:
1567 pass
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001568 return 0
1569
1570
maruel77f720b2015-09-15 12:35:22 -07001571def CMDquery_list(parser, args):
1572 """Returns list of all the Swarming APIs that can be used with command
1573 'query'.
1574 """
1575 parser.add_option(
1576 '--json', help='Path to JSON output file (otherwise prints to stdout)')
1577 options, args = parser.parse_args(args)
1578 if args:
1579 parser.error('No argument allowed.')
1580
1581 try:
1582 apis = endpoints_api_discovery_apis(options.swarming)
1583 except APIError as e:
1584 parser.error(str(e))
1585 if options.json:
maruel1ceb3872015-10-14 06:10:44 -07001586 options.json = unicode(os.path.abspath(options.json))
1587 with fs.open(options.json, 'wb') as f:
maruel77f720b2015-09-15 12:35:22 -07001588 json.dump(apis, f)
1589 else:
1590 help_url = (
1591 'https://apis-explorer.appspot.com/apis-explorer/?base=%s/_ah/api#p/' %
1592 options.swarming)
maruel11e31af2017-02-15 07:30:50 -08001593 for i, (api_id, api) in enumerate(sorted(apis.iteritems())):
1594 if i:
1595 print('')
maruel77f720b2015-09-15 12:35:22 -07001596 print api_id
maruel11e31af2017-02-15 07:30:50 -08001597 print ' ' + api['description'].strip()
1598 if 'resources' in api:
1599 # Old.
1600 for j, (resource_name, resource) in enumerate(
1601 sorted(api['resources'].iteritems())):
1602 if j:
1603 print('')
1604 for method_name, method in sorted(resource['methods'].iteritems()):
1605 # Only list the GET ones.
1606 if method['httpMethod'] != 'GET':
1607 continue
1608 print '- %s.%s: %s' % (
1609 resource_name, method_name, method['path'])
1610 print('\n'.join(
Sergey Berezina269e1a2018-05-16 16:55:12 -07001611 ' ' + l for l in textwrap.wrap(
1612 method.get('description', 'No description'), 78)))
maruel11e31af2017-02-15 07:30:50 -08001613 print ' %s%s%s' % (help_url, api['servicePath'], method['id'])
1614 else:
1615 # New.
1616 for method_name, method in sorted(api['methods'].iteritems()):
maruel77f720b2015-09-15 12:35:22 -07001617 # Only list the GET ones.
1618 if method['httpMethod'] != 'GET':
1619 continue
maruel11e31af2017-02-15 07:30:50 -08001620 print '- %s: %s' % (method['id'], method['path'])
1621 print('\n'.join(
1622 ' ' + l for l in textwrap.wrap(method['description'], 78)))
maruel77f720b2015-09-15 12:35:22 -07001623 print ' %s%s%s' % (help_url, api['servicePath'], method['id'])
1624 return 0
1625
1626
Vadim Shtayuraae8085b2014-05-02 17:13:10 -07001627@subcommand.usage('(hash|isolated) [-- extra_args]')
maruel@chromium.org0437a732013-08-27 16:05:52 +00001628def CMDrun(parser, args):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001629 """Triggers a task and wait for the results.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001630
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001631 Basically, does everything to run a command remotely.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001632 """
1633 add_trigger_options(parser)
1634 add_collect_options(parser)
Vadim Shtayurab450c602014-05-12 19:23:25 -07001635 add_sharding_options(parser)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001636 options, args = parser.parse_args(args)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001637 process_collect_options(parser, options)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001638 task_request = process_trigger_options(parser, options, args)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001639 try:
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001640 tasks = trigger_task_shards(
1641 options.swarming, task_request, options.shards)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001642 except Failure as e:
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001643 on_error.report(
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001644 'Failed to trigger %s(%s): %s' %
maruel0a25f6c2017-05-10 10:43:23 -07001645 (task_request.name, args[0], e.args[0]))
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001646 return 1
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001647 if not tasks:
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001648 on_error.report('Failed to trigger the task.')
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001649 return 1
maruel0a25f6c2017-05-10 10:43:23 -07001650 print('Triggered task: %s' % task_request.name)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001651 task_ids = [
1652 t['task_id']
1653 for t in sorted(tasks.itervalues(), key=lambda x: x['shard_index'])
1654 ]
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001655 if not options.timeout:
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001656 offset = 0
1657 for s in task_request.task_slices:
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001658 m = (offset + s.properties.execution_timeout_secs +
1659 s.expiration_secs)
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001660 if m > options.timeout:
1661 options.timeout = m
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001662 offset += s.expiration_secs
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001663 options.timeout += 10.
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001664 try:
1665 return collect(
1666 options.swarming,
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001667 task_ids,
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001668 options.timeout,
Vadim Shtayura86a2cef2014-04-18 11:13:39 -07001669 options.decorate,
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001670 options.print_status_updates,
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001671 options.task_summary_json,
maruel9531ce02016-04-13 06:11:23 -07001672 options.task_output_dir,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001673 options.task_output_stdout,
Takuto Ikuta1e6072c2018-11-06 20:42:43 +00001674 options.perf,
1675 options.filepath_filter)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001676 except Failure:
1677 on_error.report(None)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001678 return 1
maruel@chromium.org0437a732013-08-27 16:05:52 +00001679
1680
maruel18122c62015-10-23 06:31:23 -07001681@subcommand.usage('task_id -- <extra_args>')
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001682def CMDreproduce(parser, args):
1683 """Runs a task locally that was triggered on the server.
1684
1685 This running locally the same commands that have been run on the bot. The data
1686 downloaded will be in a subdirectory named 'work' of the current working
1687 directory.
maruel18122c62015-10-23 06:31:23 -07001688
1689 You can pass further additional arguments to the target command by passing
1690 them after --.
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001691 """
maruelc070e672016-02-22 17:32:57 -08001692 parser.add_option(
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001693 '--output', metavar='DIR', default='out',
maruelc070e672016-02-22 17:32:57 -08001694 help='Directory that will have results stored into')
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001695 parser.add_option(
1696 '--work', metavar='DIR', default='work',
1697 help='Directory to map the task input files into')
1698 parser.add_option(
1699 '--cache', metavar='DIR', default='cache',
1700 help='Directory that contains the input cache')
1701 parser.add_option(
1702 '--leak', action='store_true',
1703 help='Do not delete the working directory after execution')
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001704 options, args = parser.parse_args(args)
maruel18122c62015-10-23 06:31:23 -07001705 extra_args = []
1706 if not args:
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001707 parser.error('Must specify exactly one task id.')
maruel18122c62015-10-23 06:31:23 -07001708 if len(args) > 1:
1709 if args[1] == '--':
1710 if len(args) > 2:
1711 extra_args = args[2:]
1712 else:
1713 extra_args = args[1:]
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001714
smut281c3902018-05-30 17:50:05 -07001715 url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0]
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001716 request = net.url_read_json(url)
1717 if not request:
1718 print >> sys.stderr, 'Failed to retrieve request data for the task'
1719 return 1
1720
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001721 workdir = unicode(os.path.abspath(options.work))
maruele7cd38e2016-03-01 19:12:48 -08001722 if fs.isdir(workdir):
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001723 parser.error('Please delete the directory %r first' % options.work)
maruele7cd38e2016-03-01 19:12:48 -08001724 fs.mkdir(workdir)
iannucci31ab9192017-05-02 19:11:56 -07001725 cachedir = unicode(os.path.abspath('cipd_cache'))
1726 if not fs.exists(cachedir):
1727 fs.mkdir(cachedir)
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001728
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001729 properties = request['properties']
iannucci31ab9192017-05-02 19:11:56 -07001730 env = os.environ.copy()
1731 env['SWARMING_BOT_ID'] = 'reproduce'
1732 env['SWARMING_TASK_ID'] = 'reproduce'
maruel29ab2fd2015-10-16 11:44:01 -07001733 if properties.get('env'):
Marc-Antoine Ruel119b0842014-12-19 15:27:58 -05001734 logging.info('env: %r', properties['env'])
maruelb76604c2015-11-11 11:53:44 -08001735 for i in properties['env']:
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001736 key = i['key']
maruelb76604c2015-11-11 11:53:44 -08001737 if not i['value']:
1738 env.pop(key, None)
1739 else:
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001740 env[key] = i['value']
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001741
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001742 if properties.get('env_prefixes'):
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001743 env_prefixes = properties['env_prefixes']
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001744 logging.info('env_prefixes: %r', env_prefixes)
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001745 for i in env_prefixes:
1746 key = i['key']
1747 paths = [os.path.normpath(os.path.join(workdir, p)) for p in i['value']]
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001748 cur = env.get(key)
1749 if cur:
1750 paths.append(cur)
1751 env[key] = os.path.pathsep.join(paths)
1752
iannucci31ab9192017-05-02 19:11:56 -07001753 command = []
nodir152cba62016-05-12 16:08:56 -07001754 if (properties.get('inputs_ref') or {}).get('isolated'):
maruel29ab2fd2015-10-16 11:44:01 -07001755 # Create the tree.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001756 server_ref = isolate_storage.ServerRef(
maruel29ab2fd2015-10-16 11:44:01 -07001757 properties['inputs_ref']['isolatedserver'],
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001758 properties['inputs_ref']['namespace'])
1759 with isolateserver.get_storage(server_ref) as storage:
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001760 # Do not use MemoryContentAddressedCache here, as on 32-bits python,
1761 # inputs larger than ~1GiB will not fit in memory. This is effectively a
1762 # leak.
1763 policies = local_caching.CachePolicies(0, 0, 0, 0)
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001764 cache = local_caching.DiskContentAddressedCache(
Marc-Antoine Ruel79d42192019-02-06 19:24:16 +00001765 unicode(os.path.abspath(options.cache)), policies, False)
maruel29ab2fd2015-10-16 11:44:01 -07001766 bundle = isolateserver.fetch_isolated(
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001767 properties['inputs_ref']['isolated'], storage, cache, workdir, False)
maruel29ab2fd2015-10-16 11:44:01 -07001768 command = bundle.command
1769 if bundle.relative_cwd:
1770 workdir = os.path.join(workdir, bundle.relative_cwd)
maruela1b9e552016-01-06 12:42:03 -08001771 command.extend(properties.get('extra_args') or [])
iannucci31ab9192017-05-02 19:11:56 -07001772
1773 if properties.get('command'):
1774 command.extend(properties['command'])
1775
Marc-Antoine Ruelc7243592018-05-24 17:04:04 -04001776 # https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Magic-Values.md
Robert Iannucci24ae76a2018-02-26 12:51:18 -08001777 command = tools.fix_python_cmd(command, env)
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001778 if not options.output:
Marc-Antoine Ruel88229872018-01-10 16:35:29 -05001779 new_command = run_isolated.process_command(command, 'invalid', None)
1780 if new_command != command:
Marc-Antoine Ruel29ba75c2018-01-10 15:04:14 -05001781 parser.error('The task has outputs, you must use --output-dir')
Marc-Antoine Ruel88229872018-01-10 16:35:29 -05001782 else:
1783 # Make the path absolute, as the process will run from a subdirectory.
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001784 options.output = os.path.abspath(options.output)
Marc-Antoine Ruel88229872018-01-10 16:35:29 -05001785 new_command = run_isolated.process_command(
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001786 command, options.output, None)
1787 if not os.path.isdir(options.output):
1788 os.makedirs(options.output)
iannucci31ab9192017-05-02 19:11:56 -07001789 command = new_command
1790 file_path.ensure_command_has_abs_path(command, workdir)
1791
1792 if properties.get('cipd_input'):
1793 ci = properties['cipd_input']
1794 cp = ci['client_package']
1795 client_manager = cipd.get_client(
1796 ci['server'], cp['package_name'], cp['version'], cachedir)
1797
1798 with client_manager as client:
1799 by_path = collections.defaultdict(list)
1800 for pkg in ci['packages']:
1801 path = pkg['path']
1802 # cipd deals with 'root' as ''
1803 if path == '.':
1804 path = ''
1805 by_path[path].append((pkg['package_name'], pkg['version']))
1806 client.ensure(workdir, by_path, cache_dir=cachedir)
1807
maruel77f720b2015-09-15 12:35:22 -07001808 try:
Marc-Antoine Ruel95c21872018-01-10 14:24:28 -05001809 return subprocess42.call(command + extra_args, env=env, cwd=workdir)
maruel77f720b2015-09-15 12:35:22 -07001810 except OSError as e:
maruel29ab2fd2015-10-16 11:44:01 -07001811 print >> sys.stderr, 'Failed to run: %s' % ' '.join(command)
maruel77f720b2015-09-15 12:35:22 -07001812 print >> sys.stderr, str(e)
1813 return 1
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001814 finally:
1815 # Do not delete options.cache.
1816 if not options.leak:
1817 file_path.rmtree(workdir)
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001818
1819
maruel0eb1d1b2015-10-02 14:48:21 -07001820@subcommand.usage('bot_id')
1821def CMDterminate(parser, args):
1822 """Tells a bot to gracefully shut itself down as soon as it can.
1823
1824 This is done by completing whatever current task there is then exiting the bot
1825 process.
1826 """
1827 parser.add_option(
1828 '--wait', action='store_true', help='Wait for the bot to terminate')
1829 options, args = parser.parse_args(args)
1830 if len(args) != 1:
1831 parser.error('Please provide the bot id')
smut281c3902018-05-30 17:50:05 -07001832 url = options.swarming + '/_ah/api/swarming/v1/bot/%s/terminate' % args[0]
maruel0eb1d1b2015-10-02 14:48:21 -07001833 request = net.url_read_json(url, data={})
1834 if not request:
1835 print >> sys.stderr, 'Failed to ask for termination'
1836 return 1
1837 if options.wait:
1838 return collect(
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001839 options.swarming,
1840 [request['task_id']],
1841 0.,
1842 False,
1843 False,
1844 None,
1845 None,
1846 [],
Takuto Ikuta1e6072c2018-11-06 20:42:43 +00001847 False,
1848 None)
maruelbfc5f872017-06-10 16:43:17 -07001849 else:
1850 print request['task_id']
maruel0eb1d1b2015-10-02 14:48:21 -07001851 return 0
1852
1853
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001854@subcommand.usage("(hash|isolated) [-- extra_args|raw command]")
maruel@chromium.org0437a732013-08-27 16:05:52 +00001855def CMDtrigger(parser, args):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001856 """Triggers a Swarming task.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001857
Vadim Shtayuraae8085b2014-05-02 17:13:10 -07001858 Passes all extra arguments provided after '--' as additional command line
1859 arguments for an isolated command specified in *.isolate file.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001860 """
1861 add_trigger_options(parser)
Vadim Shtayurab450c602014-05-12 19:23:25 -07001862 add_sharding_options(parser)
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001863 parser.add_option(
1864 '--dump-json',
1865 metavar='FILE',
1866 help='Dump details about the triggered task(s) to this file as json')
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001867 options, args = parser.parse_args(args)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001868 task_request = process_trigger_options(parser, options, args)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001869 try:
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001870 tasks = trigger_task_shards(
1871 options.swarming, task_request, options.shards)
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001872 if tasks:
maruel0a25f6c2017-05-10 10:43:23 -07001873 print('Triggered task: %s' % task_request.name)
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -04001874 tasks_sorted = sorted(
1875 tasks.itervalues(), key=lambda x: x['shard_index'])
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001876 if options.dump_json:
1877 data = {
maruel0a25f6c2017-05-10 10:43:23 -07001878 'base_task_name': task_request.name,
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001879 'tasks': tasks,
Vadim Shtayura2d83a942017-08-14 17:41:24 -07001880 'request': task_request_to_raw_request(task_request),
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001881 }
maruel46b015f2015-10-13 18:40:35 -07001882 tools.write_json(unicode(options.dump_json), data, True)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001883 print('To collect results, use:')
Bruce Dawsonf0a5ae42018-09-04 20:06:46 +00001884 print(' tools/swarming_client/swarming.py collect -S %s --json %s' %
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001885 (options.swarming, options.dump_json))
1886 else:
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001887 print('To collect results, use:')
Bruce Dawsonf0a5ae42018-09-04 20:06:46 +00001888 print(' tools/swarming_client/swarming.py collect -S %s %s' %
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -04001889 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted)))
1890 print('Or visit:')
1891 for t in tasks_sorted:
1892 print(' ' + t['view_url'])
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001893 return int(not tasks)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001894 except Failure:
1895 on_error.report(None)
vadimsh@chromium.orgd908a542013-10-30 01:36:17 +00001896 return 1
maruel@chromium.org0437a732013-08-27 16:05:52 +00001897
1898
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001899class OptionParserSwarming(logging_utils.OptionParserWithLogging):
maruel@chromium.org0437a732013-08-27 16:05:52 +00001900 def __init__(self, **kwargs):
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001901 logging_utils.OptionParserWithLogging.__init__(
maruel@chromium.org0437a732013-08-27 16:05:52 +00001902 self, prog='swarming.py', **kwargs)
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001903 self.server_group = optparse.OptionGroup(self, 'Server')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001904 self.server_group.add_option(
maruel@chromium.orge9403ab2013-09-20 18:03:49 +00001905 '-S', '--swarming',
Kevin Graney5346c162014-01-24 12:20:01 -05001906 metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
maruel@chromium.orge9403ab2013-09-20 18:03:49 +00001907 help='Swarming server to use')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001908 self.add_option_group(self.server_group)
Vadim Shtayurae34e13a2014-02-02 11:23:26 -08001909 auth.add_auth_options(self)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001910
1911 def parse_args(self, *args, **kwargs):
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001912 options, args = logging_utils.OptionParserWithLogging.parse_args(
maruel@chromium.org0437a732013-08-27 16:05:52 +00001913 self, *args, **kwargs)
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001914 auth.process_auth_options(self, options)
1915 user = self._process_swarming(options)
1916 if hasattr(options, 'user') and not options.user:
1917 options.user = user
1918 return options, args
1919
1920 def _process_swarming(self, options):
1921 """Processes the --swarming option and aborts if not specified.
1922
1923 Returns the identity as determined by the server.
1924 """
maruel@chromium.org0437a732013-08-27 16:05:52 +00001925 if not options.swarming:
1926 self.error('--swarming is required.')
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001927 try:
1928 options.swarming = net.fix_url(options.swarming)
1929 except ValueError as e:
1930 self.error('--swarming %s' % e)
1931 on_error.report_on_exception_exit(options.swarming)
Marc-Antoine Ruelf7d737d2014-12-10 15:36:29 -05001932 try:
1933 user = auth.ensure_logged_in(options.swarming)
1934 except ValueError as e:
1935 self.error(str(e))
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001936 return user
maruel@chromium.org0437a732013-08-27 16:05:52 +00001937
1938
1939def main(args):
1940 dispatcher = subcommand.CommandDispatcher(__name__)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001941 return dispatcher.execute(OptionParserSwarming(version=__version__), args)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001942
1943
1944if __name__ == '__main__':
maruel8e4e40c2016-05-30 06:21:07 -07001945 subprocess42.inhibit_os_error_reporting()
maruel@chromium.org0437a732013-08-27 16:05:52 +00001946 fix_encoding.fix_encoding()
1947 tools.disable_buffering()
1948 colorama.init()
1949 sys.exit(main(sys.argv[1:]))