blob: 46df1b06bf93cc18e597092eca34886bfbd0a1ea [file] [log] [blame]
maruel@chromium.org0437a732013-08-27 16:05:52 +00001#!/usr/bin/env python
maruelea586f32016-04-05 11:11:33 -07002# Copyright 2013 The LUCI Authors. All rights reserved.
maruelf1f5e2a2016-05-25 17:10:39 -07003# Use of this source code is governed under the Apache License, Version 2.0
4# that can be found in the LICENSE file.
maruel@chromium.org0437a732013-08-27 16:05:52 +00005"""Client tool to trigger tasks or retrieve results from a Swarming server."""
6
Lei Leife202df2019-06-11 17:33:34 +00007from __future__ import print_function
8
9__version__ = '1.0'
maruel@chromium.org0437a732013-08-27 16:05:52 +000010
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050011import collections
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -040012import datetime
maruel@chromium.org0437a732013-08-27 16:05:52 +000013import json
14import logging
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -040015import optparse
maruel@chromium.org0437a732013-08-27 16:05:52 +000016import os
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +100017import re
maruel@chromium.org0437a732013-08-27 16:05:52 +000018import sys
maruel11e31af2017-02-15 07:30:50 -080019import textwrap
Vadim Shtayurab19319e2014-04-27 08:50:06 -070020import threading
maruel@chromium.org0437a732013-08-27 16:05:52 +000021import time
Takuto Ikuta35250172020-01-31 09:33:46 +000022import uuid
maruel@chromium.org0437a732013-08-27 16:05:52 +000023
vadimsh@chromium.org6b706212013-08-28 15:03:46 +000024from utils import tools
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000025tools.force_local_third_party()
maruel@chromium.org0437a732013-08-27 16:05:52 +000026
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000027# third_party/
28import colorama
29from chromium import natsort
30from depot_tools import fix_encoding
31from depot_tools import subcommand
Takuto Ikuta6e2ff962019-10-29 12:35:27 +000032import six
Marc-Antoine Ruelad8cabe2019-10-10 23:24:26 +000033from six.moves import urllib
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000034
35# pylint: disable=ungrouped-imports
Vadim Shtayurae34e13a2014-02-02 11:23:26 -080036import auth
iannucci31ab9192017-05-02 19:11:56 -070037import cipd
maruel@chromium.org7b844a62013-09-17 13:04:59 +000038import isolateserver
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +000039import isolate_storage
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -040040import local_caching
maruelc070e672016-02-22 17:32:57 -080041import run_isolated
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000042from utils import file_path
43from utils import fs
44from utils import logging_utils
45from utils import net
46from utils import on_error
47from utils import subprocess42
48from utils import threading_utils
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050049
50
51class Failure(Exception):
52 """Generic failure."""
53 pass
54
55
maruel0a25f6c2017-05-10 10:43:23 -070056def default_task_name(options):
57 """Returns a default task name if not specified."""
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050058 if not options.task_name:
Junji Watanabe38b28b02020-04-23 10:23:30 +000059 task_name = u'%s/%s' % (options.user, '_'.join(
60 '%s=%s' % (k, v) for k, v in options.dimensions))
maruel0a25f6c2017-05-10 10:43:23 -070061 if options.isolated:
62 task_name += u'/' + options.isolated
63 return task_name
64 return options.task_name
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -050065
66
67### Triggering.
68
maruel77f720b2015-09-15 12:35:22 -070069# See ../appengine/swarming/swarming_rpcs.py.
Junji Watanabe38b28b02020-04-23 10:23:30 +000070CipdPackage = collections.namedtuple('CipdPackage', [
71 'package_name',
72 'path',
73 'version',
74])
borenet02f772b2016-06-22 12:42:19 -070075
borenet02f772b2016-06-22 12:42:19 -070076# See ../appengine/swarming/swarming_rpcs.py.
Junji Watanabe38b28b02020-04-23 10:23:30 +000077CipdInput = collections.namedtuple('CipdInput', [
78 'client_package',
79 'packages',
80 'server',
81])
borenet02f772b2016-06-22 12:42:19 -070082
83# See ../appengine/swarming/swarming_rpcs.py.
Junji Watanabe38b28b02020-04-23 10:23:30 +000084FilesRef = collections.namedtuple('FilesRef', [
85 'isolated',
86 'isolatedserver',
87 'namespace',
88])
maruel77f720b2015-09-15 12:35:22 -070089
90# See ../appengine/swarming/swarming_rpcs.py.
Robert Iannuccibf5f84c2017-11-22 12:56:50 -080091StringListPair = collections.namedtuple(
Junji Watanabe38b28b02020-04-23 10:23:30 +000092 'StringListPair',
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +000093 [
Junji Watanabe38b28b02020-04-23 10:23:30 +000094 'key',
95 'value', # repeated string
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +000096 ])
97
Junji Watanabe38b28b02020-04-23 10:23:30 +000098# See ../appengine/swarming/swarming_rpcs.py.
99Containment = collections.namedtuple('Containment', [
100 'lower_priority',
101 'containment_type',
102])
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800103
104# See ../appengine/swarming/swarming_rpcs.py.
Junji Watanabe38b28b02020-04-23 10:23:30 +0000105TaskProperties = collections.namedtuple('TaskProperties', [
106 'caches',
107 'cipd_input',
108 'command',
109 'containment',
110 'relative_cwd',
111 'dimensions',
112 'env',
113 'env_prefixes',
114 'execution_timeout_secs',
115 'extra_args',
116 'grace_period_secs',
117 'idempotent',
118 'inputs_ref',
119 'io_timeout_secs',
120 'outputs',
121 'secret_bytes',
122])
maruel77f720b2015-09-15 12:35:22 -0700123
Junji Watanabecb054042020-07-21 08:43:26 +0000124# See ../appengine/swarming/swarming_rpcs.py.
125TaskSlice = collections.namedtuple('TaskSlice', [
126 'expiration_secs',
127 'properties',
128 'wait_for_capacity',
129])
maruel77f720b2015-09-15 12:35:22 -0700130
131# See ../appengine/swarming/swarming_rpcs.py.
Junji Watanabecb054042020-07-21 08:43:26 +0000132NewTaskRequest = collections.namedtuple('NewTaskRequest', [
133 'name',
134 'parent_task_id',
135 'priority',
Junji Watanabe71bbaef2020-07-21 08:55:37 +0000136 'realm',
Scott Lee44c13d72020-09-14 06:09:50 +0000137 'resultdb',
Junji Watanabecb054042020-07-21 08:43:26 +0000138 'task_slices',
139 'service_account',
140 'tags',
141 'user',
142 'pool_task_template',
143])
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500144
145
maruel77f720b2015-09-15 12:35:22 -0700146def namedtuple_to_dict(value):
147 """Recursively converts a namedtuple to a dict."""
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -0400148 if hasattr(value, '_asdict'):
149 return namedtuple_to_dict(value._asdict())
150 if isinstance(value, (list, tuple)):
151 return [namedtuple_to_dict(v) for v in value]
152 if isinstance(value, dict):
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +0000153 return {k: namedtuple_to_dict(v) for k, v in value.items()}
Lei Lei73a5f732020-03-23 20:36:14 +0000154 # json.dumps in Python3 doesn't support bytes.
155 if isinstance(value, bytes):
156 return six.ensure_str(value)
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -0400157 return value
maruel77f720b2015-09-15 12:35:22 -0700158
159
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700160def task_request_to_raw_request(task_request):
maruel71c61c82016-02-22 06:52:05 -0800161 """Returns the json-compatible dict expected by the server for new request.
maruelaf6269b2015-09-10 14:37:51 -0700162
163 This is for the v1 client Swarming API.
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500164 """
maruel77f720b2015-09-15 12:35:22 -0700165 out = namedtuple_to_dict(task_request)
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700166 # Don't send 'service_account' if it is None to avoid confusing older
167 # version of the server that doesn't know about 'service_account' and don't
168 # use it at all.
169 if not out['service_account']:
170 out.pop('service_account')
Brad Hallf78187a2018-10-19 17:08:55 +0000171 for task_slice in out['task_slices']:
Junji Watanabecb054042020-07-21 08:43:26 +0000172 task_slice['properties']['env'] = [{
173 'key': k,
174 'value': v
175 } for k, v in task_slice['properties']['env'].items()]
Brad Hallf78187a2018-10-19 17:08:55 +0000176 task_slice['properties']['env'].sort(key=lambda x: x['key'])
Takuto Ikuta35250172020-01-31 09:33:46 +0000177 out['request_uuid'] = str(uuid.uuid4())
maruel77f720b2015-09-15 12:35:22 -0700178 return out
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500179
180
maruel77f720b2015-09-15 12:35:22 -0700181def swarming_trigger(swarming, raw_request):
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500182 """Triggers a request on the Swarming server and returns the json data.
183
184 It's the low-level function.
185
186 Returns:
187 {
188 'request': {
189 'created_ts': u'2010-01-02 03:04:05',
190 'name': ..
191 },
192 'task_id': '12300',
193 }
194 """
195 logging.info('Triggering: %s', raw_request['name'])
196
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500197 result = net.url_read_json(
smut281c3902018-05-30 17:50:05 -0700198 swarming + '/_ah/api/swarming/v1/tasks/new', data=raw_request)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500199 if not result:
200 on_error.report('Failed to trigger task %s' % raw_request['name'])
201 return None
maruele557bce2015-11-17 09:01:27 -0800202 if result.get('error'):
marueld4d15312015-11-16 17:22:59 -0800203 # The reply is an error.
maruele557bce2015-11-17 09:01:27 -0800204 msg = 'Failed to trigger task %s' % raw_request['name']
205 if result['error'].get('errors'):
206 for err in result['error']['errors']:
207 if err.get('message'):
208 msg += '\nMessage: %s' % err['message']
209 if err.get('debugInfo'):
210 msg += '\nDebug info:\n%s' % err['debugInfo']
211 elif result['error'].get('message'):
maruelbf53e042015-12-01 15:00:51 -0800212 msg += '\nMessage: %s' % result['error']['message']
maruele557bce2015-11-17 09:01:27 -0800213
214 on_error.report(msg)
marueld4d15312015-11-16 17:22:59 -0800215 return None
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500216 return result
217
218
219def setup_googletest(env, shards, index):
220 """Sets googletest specific environment variables."""
221 if shards > 1:
maruel77f720b2015-09-15 12:35:22 -0700222 assert not any(i['key'] == 'GTEST_SHARD_INDEX' for i in env), env
223 assert not any(i['key'] == 'GTEST_TOTAL_SHARDS' for i in env), env
224 env = env[:]
225 env.append({'key': 'GTEST_SHARD_INDEX', 'value': str(index)})
226 env.append({'key': 'GTEST_TOTAL_SHARDS', 'value': str(shards)})
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500227 return env
228
229
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000230def trigger_task_shards(swarming, task_request):
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500231 """Triggers one or many subtasks of a sharded task.
232
233 Returns:
234 Dict with task details, returned to caller as part of --dump-json output.
235 None in case of failure.
236 """
Junji Watanabecb054042020-07-21 08:43:26 +0000237
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000238 def convert_request():
Erik Chend50a88f2019-02-16 01:22:07 +0000239 """
240 Args:
241 index: The index of the task request.
242
243 Returns:
244 raw_request: A swarming compatible JSON dictionary of the request.
245 shard_index: The index of the shard, which may be different than the index
246 of the task request.
247 """
Vadim Shtayura2d83a942017-08-14 17:41:24 -0700248 req = task_request_to_raw_request(task_request)
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000249 shard_index = 0
Erik Chend50a88f2019-02-16 01:22:07 +0000250
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000251 task_slices = req['task_slices']
252 total_shards = 1
253 # Multiple tasks slices might exist if there are optional "slices", e.g.
254 # multiple ways of dispatching the task that should be equivalent. These
255 # should be functionally equivalent but we have cannot guarantee that. If
256 # we see the GTEST_SHARD_INDEX env var, we assume that it applies to all
257 # slices.
258 for task_slice in task_slices:
259 for env_var in task_slice['properties']['env']:
260 if env_var['key'] == 'GTEST_SHARD_INDEX':
261 shard_index = int(env_var['value'])
262 if env_var['key'] == 'GTEST_TOTAL_SHARDS':
263 total_shards = int(env_var['value'])
264 if total_shards > 1:
265 req['name'] += ':%s:%s' % (shard_index, total_shards)
266 if shard_index and total_shards:
267 req['tags'] += [
268 'shard_index:%d' % shard_index,
269 'total_shards:%d' % total_shards,
270 ]
Erik Chend50a88f2019-02-16 01:22:07 +0000271
272 return req, shard_index
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500273
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000274 request, shard_index = convert_request()
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500275 tasks = {}
276 priority_warning = False
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000277 task = swarming_trigger(swarming, request)
278 if task is not None:
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500279 logging.info('Request result: %s', task)
280 if (not priority_warning and
Marc-Antoine Ruelb1216762017-08-17 10:07:49 -0400281 int(task['request']['priority']) != task_request.priority):
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500282 priority_warning = True
Junji Watanabecb054042020-07-21 08:43:26 +0000283 print(
284 'Priority was reset to %s' % task['request']['priority'],
285 file=sys.stderr)
Ted Pudlikadc55e92020-09-28 23:25:49 +0000286 view_url = '%s/user/task/%s' % (swarming, task['task_id'])
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500287 tasks[request['name']] = {
Junji Watanabecb054042020-07-21 08:43:26 +0000288 'shard_index': shard_index,
289 'task_id': task['task_id'],
Ted Pudlikadc55e92020-09-28 23:25:49 +0000290 'view_url': view_url,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500291 }
Ted Pudlikadc55e92020-09-28 23:25:49 +0000292 logging.info('Task UI: %s', view_url)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500293
Ye Kuang3f3f2f72020-10-21 10:14:59 +0000294 if not tasks:
295 print('Task did not trigger successfully', file=sys.stderr)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500296 return None
297
298 return tasks
299
300
301### Collection.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000302
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700303# How often to print status updates to stdout in 'collect'.
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000304STATUS_UPDATE_INTERVAL = 5 * 60.
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700305
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400306
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000307class TaskState(object):
308 """Represents the current task state.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000309
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000310 For documentation, see the comments in the swarming_rpcs.TaskState enum, which
311 is the source of truth for these values:
312 https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_rpcs.py?q=TaskState\(
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400313
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000314 It's in fact an enum.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400315 """
316 RUNNING = 0x10
317 PENDING = 0x20
318 EXPIRED = 0x30
319 TIMED_OUT = 0x40
320 BOT_DIED = 0x50
321 CANCELED = 0x60
322 COMPLETED = 0x70
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -0400323 KILLED = 0x80
Marc-Antoine Ruelfc708352018-05-04 20:25:43 -0400324 NO_RESOURCE = 0x100
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400325
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000326 STATES_RUNNING = ('PENDING', 'RUNNING')
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400327
maruel77f720b2015-09-15 12:35:22 -0700328 _ENUMS = {
Junji Watanabecb054042020-07-21 08:43:26 +0000329 'RUNNING': RUNNING,
330 'PENDING': PENDING,
331 'EXPIRED': EXPIRED,
332 'TIMED_OUT': TIMED_OUT,
333 'BOT_DIED': BOT_DIED,
334 'CANCELED': CANCELED,
335 'COMPLETED': COMPLETED,
336 'KILLED': KILLED,
337 'NO_RESOURCE': NO_RESOURCE,
maruel77f720b2015-09-15 12:35:22 -0700338 }
339
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400340 @classmethod
maruel77f720b2015-09-15 12:35:22 -0700341 def from_enum(cls, state):
342 """Returns int value based on the string."""
343 if state not in cls._ENUMS:
344 raise ValueError('Invalid state %s' % state)
345 return cls._ENUMS[state]
346
maruel@chromium.org0437a732013-08-27 16:05:52 +0000347
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700348class TaskOutputCollector(object):
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700349 """Assembles task execution summary (for --task-summary-json output).
350
351 Optionally fetches task outputs from isolate server to local disk (used when
352 --task-output-dir is passed).
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700353
354 This object is shared among multiple threads running 'retrieve_results'
355 function, in particular they call 'process_shard_result' method in parallel.
356 """
357
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000358 def __init__(self, task_output_dir, task_output_stdout, shard_count,
359 filter_cb):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700360 """Initializes TaskOutputCollector, ensures |task_output_dir| exists.
361
362 Args:
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700363 task_output_dir: (optional) local directory to put fetched files to.
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700364 shard_count: expected number of task shards.
365 """
maruel12e30012015-10-09 11:55:35 -0700366 self.task_output_dir = (
Takuto Ikuta6e2ff962019-10-29 12:35:27 +0000367 six.text_type(os.path.abspath(task_output_dir))
maruel12e30012015-10-09 11:55:35 -0700368 if task_output_dir else task_output_dir)
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000369 self.task_output_stdout = task_output_stdout
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700370 self.shard_count = shard_count
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000371 self.filter_cb = filter_cb
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700372
373 self._lock = threading.Lock()
374 self._per_shard_results = {}
375 self._storage = None
376
nodire5028a92016-04-29 14:38:21 -0700377 if self.task_output_dir:
378 file_path.ensure_tree(self.task_output_dir)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700379
Vadim Shtayurab450c602014-05-12 19:23:25 -0700380 def process_shard_result(self, shard_index, result):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700381 """Stores results of a single task shard, fetches output files if necessary.
382
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400383 Modifies |result| in place.
384
maruel77f720b2015-09-15 12:35:22 -0700385 shard_index is 0-based.
386
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700387 Called concurrently from multiple threads.
388 """
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700389 # Sanity check index is in expected range.
Vadim Shtayurab450c602014-05-12 19:23:25 -0700390 assert isinstance(shard_index, int)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700391 if shard_index < 0 or shard_index >= self.shard_count:
Junji Watanabe38b28b02020-04-23 10:23:30 +0000392 logging.warning('Shard index %d is outside of expected range: [0; %d]',
393 shard_index, self.shard_count - 1)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700394 return
395
maruel77f720b2015-09-15 12:35:22 -0700396 if result.get('outputs_ref'):
397 ref = result['outputs_ref']
398 result['outputs_ref']['view_url'] = '%s/browse?%s' % (
399 ref['isolatedserver'],
Marc-Antoine Ruelad8cabe2019-10-10 23:24:26 +0000400 urllib.parse.urlencode([('namespace', ref['namespace']),
401 ('hash', ref['isolated'])]))
Kevin Graneyc2c3b9e2014-08-26 09:04:17 -0400402
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700403 # Store result dict of that shard, ignore results we've already seen.
404 with self._lock:
405 if shard_index in self._per_shard_results:
406 logging.warning('Ignoring duplicate shard index %d', shard_index)
407 return
408 self._per_shard_results[shard_index] = result
409
410 # Fetch output files if necessary.
maruel77f720b2015-09-15 12:35:22 -0700411 if self.task_output_dir and result.get('outputs_ref'):
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000412 server_ref = isolate_storage.ServerRef(
Junji Watanabecb054042020-07-21 08:43:26 +0000413 result['outputs_ref']['isolatedserver'],
414 result['outputs_ref']['namespace'])
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000415 storage = self._get_storage(server_ref)
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400416 if storage:
417 # Output files are supposed to be small and they are not reused across
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -0400418 # tasks. So use MemoryContentAddressedCache for them instead of on-disk
419 # cache. Make files writable, so that calling script can delete them.
Marc-Antoine Ruele4dcbb82014-10-01 09:30:56 -0400420 isolateserver.fetch_isolated(
Junji Watanabecb054042020-07-21 08:43:26 +0000421 result['outputs_ref']['isolated'], storage,
Lei Leife202df2019-06-11 17:33:34 +0000422 local_caching.MemoryContentAddressedCache(file_mode_mask=0o700),
Junji Watanabecb054042020-07-21 08:43:26 +0000423 os.path.join(self.task_output_dir, str(shard_index)), False,
424 self.filter_cb)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700425
426 def finalize(self):
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700427 """Assembles and returns task summary JSON, shutdowns underlying Storage."""
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700428 with self._lock:
429 # Write an array of shard results with None for missing shards.
430 summary = {
Marc-Antoine Ruel0fdee222019-10-10 14:42:40 +0000431 'shards': [
432 self._per_shard_results.get(i) for i in range(self.shard_count)
433 ],
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700434 }
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000435
436 # Don't store stdout in the summary if not requested too.
437 if "json" not in self.task_output_stdout:
438 for shard_json in summary['shards']:
439 if not shard_json:
440 continue
441 if "output" in shard_json:
442 del shard_json["output"]
443 if "outputs" in shard_json:
444 del shard_json["outputs"]
445
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700446 # Write summary.json to task_output_dir as well.
447 if self.task_output_dir:
448 tools.write_json(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000449 os.path.join(self.task_output_dir, u'summary.json'), summary, False)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700450 if self._storage:
451 self._storage.close()
452 self._storage = None
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700453 return summary
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700454
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000455 def _get_storage(self, server_ref):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700456 """Returns isolateserver.Storage to use to fetch files."""
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700457 assert self.task_output_dir
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700458 with self._lock:
459 if not self._storage:
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000460 self._storage = isolateserver.get_storage(server_ref)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700461 else:
462 # Shards must all use exact same isolate server and namespace.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000463 if self._storage.server_ref.url != server_ref.url:
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700464 logging.error(
465 'Task shards are using multiple isolate servers: %s and %s',
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000466 self._storage.server_ref.url, server_ref.url)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700467 return None
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000468 if self._storage.server_ref.namespace != server_ref.namespace:
Junji Watanabe38b28b02020-04-23 10:23:30 +0000469 logging.error('Task shards are using multiple namespaces: %s and %s',
470 self._storage.server_ref.namespace,
471 server_ref.namespace)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700472 return None
473 return self._storage
474
475
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500476def now():
477 """Exists so it can be mocked easily."""
478 return time.time()
479
480
maruel77f720b2015-09-15 12:35:22 -0700481def parse_time(value):
482 """Converts serialized time from the API to datetime.datetime."""
483 # When microseconds are 0, the '.123456' suffix is elided. This means the
484 # serialized format is not consistent, which confuses the hell out of python.
485 for fmt in ('%Y-%m-%dT%H:%M:%S.%f', '%Y-%m-%dT%H:%M:%S'):
486 try:
487 return datetime.datetime.strptime(value, fmt)
488 except ValueError:
489 pass
490 raise ValueError('Failed to parse %s' % value)
491
492
Junji Watanabe38b28b02020-04-23 10:23:30 +0000493def retrieve_results(base_url, shard_index, task_id, timeout, should_stop,
494 output_collector, include_perf, fetch_stdout):
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400495 """Retrieves results for a single task ID.
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700496
Vadim Shtayurab450c602014-05-12 19:23:25 -0700497 Returns:
498 <result dict> on success.
499 None on failure.
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700500 """
maruel71c61c82016-02-22 06:52:05 -0800501 assert timeout is None or isinstance(timeout, float), timeout
smut281c3902018-05-30 17:50:05 -0700502 result_url = '%s/_ah/api/swarming/v1/task/%s/result' % (base_url, task_id)
maruel9531ce02016-04-13 06:11:23 -0700503 if include_perf:
504 result_url += '?include_performance_stats=true'
smut281c3902018-05-30 17:50:05 -0700505 output_url = '%s/_ah/api/swarming/v1/task/%s/stdout' % (base_url, task_id)
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700506 started = now()
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400507 deadline = started + timeout if timeout > 0 else None
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700508 attempt = 0
509
510 while not should_stop.is_set():
511 attempt += 1
512
513 # Waiting for too long -> give up.
514 current_time = now()
515 if deadline and current_time >= deadline:
Junji Watanabecb054042020-07-21 08:43:26 +0000516 logging.error('retrieve_results(%s) timed out on attempt %d', base_url,
517 attempt)
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700518 return None
519
520 # Do not spin too fast. Spin faster at the beginning though.
521 # Start with 1 sec delay and for each 30 sec of waiting add another second
522 # of delay, until hitting 15 sec ceiling.
523 if attempt > 1:
524 max_delay = min(15, 1 + (current_time - started) / 30.0)
525 delay = min(max_delay, deadline - current_time) if deadline else max_delay
526 if delay > 0:
527 logging.debug('Waiting %.1f sec before retrying', delay)
528 should_stop.wait(delay)
529 if should_stop.is_set():
530 return None
531
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400532 # Disable internal retries in net.url_read_json, since we are doing retries
533 # ourselves.
534 # TODO(maruel): We'd need to know if it's a 404 and not retry at all.
maruel0eb1d1b2015-10-02 14:48:21 -0700535 # TODO(maruel): Sadly, we currently have to poll here. Use hanging HTTP
536 # request on GAE v2.
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400537 # Retry on 500s only if no timeout is specified.
538 result = net.url_read_json(result_url, retry_50x=bool(timeout == -1))
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400539 if not result:
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400540 if timeout == -1:
541 return None
Marc-Antoine Ruel200b3952014-08-14 11:07:44 -0400542 continue
maruel77f720b2015-09-15 12:35:22 -0700543
maruelbf53e042015-12-01 15:00:51 -0800544 if result.get('error'):
545 # An error occurred.
546 if result['error'].get('errors'):
547 for err in result['error']['errors']:
Junji Watanabe38b28b02020-04-23 10:23:30 +0000548 logging.warning('Error while reading task: %s; %s',
549 err.get('message'), err.get('debugInfo'))
maruelbf53e042015-12-01 15:00:51 -0800550 elif result['error'].get('message'):
Junji Watanabecb054042020-07-21 08:43:26 +0000551 logging.warning('Error while reading task: %s',
552 result['error']['message'])
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400553 if timeout == -1:
554 return result
maruelbf53e042015-12-01 15:00:51 -0800555 continue
556
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -0400557 # When timeout == -1, always return on first attempt. 500s are already
558 # retried in this case.
Marc-Antoine Ruel20b764d2018-06-22 18:08:37 +0000559 if result['state'] not in TaskState.STATES_RUNNING or timeout == -1:
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000560 if fetch_stdout:
561 out = net.url_read_json(output_url)
Vadim Shtayura6fd3c7b2017-11-03 15:32:51 -0700562 result['output'] = out.get('output', '') if out else ''
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700563 # Record the result, try to fetch attached output files (if any).
564 if output_collector:
565 # TODO(vadimsh): Respect |should_stop| and |deadline| when fetching.
Vadim Shtayurab450c602014-05-12 19:23:25 -0700566 output_collector.process_shard_result(shard_index, result)
maruel77f720b2015-09-15 12:35:22 -0700567 if result.get('internal_failure'):
568 logging.error('Internal error!')
569 elif result['state'] == 'BOT_DIED':
570 logging.error('Bot died!')
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700571 return result
maruel@chromium.org0437a732013-08-27 16:05:52 +0000572
573
Junji Watanabecb054042020-07-21 08:43:26 +0000574def yield_results(swarm_base_url, task_ids, timeout, max_threads,
575 print_status_updates, output_collector, include_perf,
576 fetch_stdout):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500577 """Yields swarming task results from the swarming server as (index, result).
maruel@chromium.org0437a732013-08-27 16:05:52 +0000578
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700579 Duplicate shards are ignored. Shards are yielded in order of completion.
580 Timed out shards are NOT yielded at all. Caller can compare number of yielded
581 shards with len(task_keys) to verify all shards completed.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000582
583 max_threads is optional and is used to limit the number of parallel fetches
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500584 done. Since in general the number of task_keys is in the range <=10, it's not
maruel@chromium.org0437a732013-08-27 16:05:52 +0000585 worth normally to limit the number threads. Mostly used for testing purposes.
Marc-Antoine Ruel5c720342014-02-21 14:46:14 -0500586
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700587 output_collector is an optional instance of TaskOutputCollector that will be
588 used to fetch files produced by a task from isolate server to the local disk.
589
Marc-Antoine Ruel5c720342014-02-21 14:46:14 -0500590 Yields:
591 (index, result). In particular, 'result' is defined as the
592 GetRunnerResults() function in services/swarming/server/test_runner.py.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000593 """
maruel@chromium.org0437a732013-08-27 16:05:52 +0000594 number_threads = (
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400595 min(max_threads, len(task_ids)) if max_threads else len(task_ids))
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700596 should_stop = threading.Event()
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700597 results_channel = threading_utils.TaskChannel()
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700598
maruel@chromium.org0437a732013-08-27 16:05:52 +0000599 with threading_utils.ThreadPool(number_threads, number_threads, 0) as pool:
600 try:
Vadim Shtayurab450c602014-05-12 19:23:25 -0700601 # Adds a task to the thread pool to call 'retrieve_results' and return
602 # the results together with shard_index that produced them (as a tuple).
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400603 def enqueue_retrieve_results(shard_index, task_id):
Marc-Antoine Ruel486c9b52018-07-23 19:30:47 +0000604 # pylint: disable=no-value-for-parameter
Vadim Shtayurab450c602014-05-12 19:23:25 -0700605 task_fn = lambda *args: (shard_index, retrieve_results(*args))
Junji Watanabecb054042020-07-21 08:43:26 +0000606 pool.add_task(0, results_channel.wrap_task(task_fn), swarm_base_url,
607 shard_index, task_id, timeout, should_stop,
608 output_collector, include_perf, fetch_stdout)
Vadim Shtayurab450c602014-05-12 19:23:25 -0700609
610 # Enqueue 'retrieve_results' calls for each shard key to run in parallel.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400611 for shard_index, task_id in enumerate(task_ids):
612 enqueue_retrieve_results(shard_index, task_id)
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700613
614 # Wait for all of them to finish.
Lei Lei73a5f732020-03-23 20:36:14 +0000615 # Convert to list, since range in Python3 doesn't have remove.
616 shards_remaining = list(range(len(task_ids)))
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400617 active_task_count = len(task_ids)
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700618 while active_task_count:
Vadim Shtayurab450c602014-05-12 19:23:25 -0700619 shard_index, result = None, None
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700620 try:
Marc-Antoine Ruel4494b6c2018-11-28 21:00:41 +0000621 shard_index, result = results_channel.next(
Vadim Shtayurab450c602014-05-12 19:23:25 -0700622 timeout=STATUS_UPDATE_INTERVAL)
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700623 except threading_utils.TaskChannel.Timeout:
624 if print_status_updates:
Jao-ke Chin-Leeba184e62018-11-19 17:04:41 +0000625 time_now = str(datetime.datetime.now())
626 _, time_now = time_now.split(' ')
Junji Watanabe38b28b02020-04-23 10:23:30 +0000627 print('%s '
628 'Waiting for results from the following shards: %s' %
629 (time_now, ', '.join(map(str, shards_remaining))))
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700630 sys.stdout.flush()
631 continue
632 except Exception:
633 logging.exception('Unexpected exception in retrieve_results')
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700634
635 # A call to 'retrieve_results' finished (successfully or not).
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700636 active_task_count -= 1
maruel@chromium.org0437a732013-08-27 16:05:52 +0000637 if not result:
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -0500638 logging.error('Failed to retrieve the results for a swarming key')
maruel@chromium.org0437a732013-08-27 16:05:52 +0000639 continue
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700640
Vadim Shtayurab450c602014-05-12 19:23:25 -0700641 # Yield back results to the caller.
642 assert shard_index in shards_remaining
643 shards_remaining.remove(shard_index)
644 yield shard_index, result
Vadim Shtayurab19319e2014-04-27 08:50:06 -0700645
maruel@chromium.org0437a732013-08-27 16:05:52 +0000646 finally:
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700647 # Done or aborted with Ctrl+C, kill the remaining threads.
maruel@chromium.org0437a732013-08-27 16:05:52 +0000648 should_stop.set()
649
650
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000651def decorate_shard_output(swarming, shard_index, metadata, include_stdout):
maruel@chromium.org0437a732013-08-27 16:05:52 +0000652 """Returns wrapped output for swarming task shard."""
maruel77f720b2015-09-15 12:35:22 -0700653 if metadata.get('started_ts') and not metadata.get('deduped_from'):
Junji Watanabecb054042020-07-21 08:43:26 +0000654 pending = '%.1fs' % (parse_time(metadata['started_ts']) -
655 parse_time(metadata['created_ts'])).total_seconds()
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400656 elif (metadata.get('state') in ('BOT_DIED', 'CANCELED', 'EXPIRED') and
657 metadata.get('abandoned_ts')):
Junji Watanabecb054042020-07-21 08:43:26 +0000658 pending = '%.1fs' % (parse_time(metadata['abandoned_ts']) -
659 parse_time(metadata['created_ts'])).total_seconds()
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400660 else:
661 pending = 'N/A'
662
maruel77f720b2015-09-15 12:35:22 -0700663 if metadata.get('duration') is not None:
664 duration = '%.1fs' % metadata['duration']
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400665 else:
666 duration = 'N/A'
667
maruel77f720b2015-09-15 12:35:22 -0700668 if metadata.get('exit_code') is not None:
669 # Integers are encoded as string to not loose precision.
670 exit_code = '%s' % metadata['exit_code']
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400671 else:
672 exit_code = 'N/A'
673
674 bot_id = metadata.get('bot_id') or 'N/A'
675
maruel77f720b2015-09-15 12:35:22 -0700676 url = '%s/user/task/%s' % (swarming, metadata['task_id'])
Marc-Antoine Ruel4e6b73d2014-10-03 18:00:05 -0400677 tag_header = 'Shard %d %s' % (shard_index, url)
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000678 tag_footer1 = 'End of shard %d' % (shard_index)
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400679 if metadata.get('state') == 'CANCELED':
680 tag_footer2 = ' Pending: %s CANCELED' % pending
681 elif metadata.get('state') == 'EXPIRED':
682 tag_footer2 = ' Pending: %s EXPIRED (lack of capacity)' % pending
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -0400683 elif metadata.get('state') in ('BOT_DIED', 'TIMED_OUT', 'KILLED'):
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400684 tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s %s' % (
685 pending, duration, bot_id, exit_code, metadata['state'])
686 else:
687 tag_footer2 = ' Pending: %s Duration: %s Bot: %s Exit: %s' % (
688 pending, duration, bot_id, exit_code)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400689
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000690 tag_len = max(len(x) for x in [tag_header, tag_footer1, tag_footer2])
691 dash_pad = '+-%s-+' % ('-' * tag_len)
692 tag_header = '| %s |' % tag_header.ljust(tag_len)
693 tag_footer1 = '| %s |' % tag_footer1.ljust(tag_len)
694 tag_footer2 = '| %s |' % tag_footer2.ljust(tag_len)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400695
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000696 if include_stdout:
697 return '\n'.join([
698 dash_pad,
699 tag_header,
700 dash_pad,
Marc-Antoine Ruel3f9931a2017-11-03 14:34:49 -0400701 (metadata.get('output') or '').rstrip(),
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000702 dash_pad,
703 tag_footer1,
704 tag_footer2,
705 dash_pad,
Junji Watanabe38b28b02020-04-23 10:23:30 +0000706 ])
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +0000707 return '\n'.join([
708 dash_pad,
709 tag_header,
710 tag_footer2,
711 dash_pad,
Junji Watanabe38b28b02020-04-23 10:23:30 +0000712 ])
maruel@chromium.org0437a732013-08-27 16:05:52 +0000713
714
Junji Watanabecb054042020-07-21 08:43:26 +0000715def collect(swarming, task_ids, timeout, decorate, print_status_updates,
716 task_summary_json, task_output_dir, task_output_stdout,
717 include_perf, filepath_filter):
maruela5490782015-09-30 10:56:59 -0700718 """Retrieves results of a Swarming task.
719
720 Returns:
721 process exit code that should be returned to the user.
722 """
Takuto Ikuta1e6072c2018-11-06 20:42:43 +0000723
724 filter_cb = None
725 if filepath_filter:
726 filter_cb = re.compile(filepath_filter).match
727
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700728 # Collect summary JSON and output files (if task_output_dir is not None).
Junji Watanabecb054042020-07-21 08:43:26 +0000729 output_collector = TaskOutputCollector(task_output_dir, task_output_stdout,
730 len(task_ids), filter_cb)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700731
Vadim Shtayura86a2cef2014-04-18 11:13:39 -0700732 seen_shards = set()
maruela5490782015-09-30 10:56:59 -0700733 exit_code = None
Marc-Antoine Rueld59e8072014-10-21 18:54:45 -0400734 total_duration = 0
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700735 try:
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400736 for index, metadata in yield_results(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000737 swarming,
738 task_ids,
739 timeout,
740 None,
741 print_status_updates,
742 output_collector,
743 include_perf,
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000744 (len(task_output_stdout) > 0),
Junji Watanabe38b28b02020-04-23 10:23:30 +0000745 ):
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700746 seen_shards.add(index)
Vadim Shtayura473455a2014-05-14 15:22:35 -0700747
Marc-Antoine Ruel5e6ccdb2015-04-02 15:55:13 -0400748 # Default to failure if there was no process that even started.
maruel77f720b2015-09-15 12:35:22 -0700749 shard_exit_code = metadata.get('exit_code')
750 if shard_exit_code:
maruela5490782015-09-30 10:56:59 -0700751 # It's encoded as a string, so bool('0') is True.
maruel77f720b2015-09-15 12:35:22 -0700752 shard_exit_code = int(shard_exit_code)
maruela5490782015-09-30 10:56:59 -0700753 if shard_exit_code or exit_code is None:
Marc-Antoine Ruel4e6b73d2014-10-03 18:00:05 -0400754 exit_code = shard_exit_code
maruel77f720b2015-09-15 12:35:22 -0700755 total_duration += metadata.get('duration', 0)
Vadim Shtayura473455a2014-05-14 15:22:35 -0700756
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700757 if decorate:
Lei Lei805a75d2020-10-08 16:31:55 +0000758 s = decorate_shard_output(swarming, index, metadata,
Lei Lei73a5f732020-03-23 20:36:14 +0000759 "console" in task_output_stdout).encode(
Lei Lei805a75d2020-10-08 16:31:55 +0000760 'utf-8', 'replace')
761
762 # The default system encoding is ascii, which can not handle non-ascii
763 # characters, switch to use sys.stdout.buffer.write in Python3 to
764 # send utf-8 to stdout regardless of the console's encoding.
765 if six.PY3:
766 sys.stdout.buffer.write(s)
767 else:
768 print(s)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400769 if len(seen_shards) < len(task_ids):
770 print('')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700771 else:
Junji Watanabecb054042020-07-21 08:43:26 +0000772 print('%s: %s %s' % (metadata.get(
773 'bot_id', 'N/A'), metadata['task_id'], shard_exit_code))
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +1000774 if "console" in task_output_stdout and metadata['output']:
maruel77f720b2015-09-15 12:35:22 -0700775 output = metadata['output'].rstrip()
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400776 if output:
777 print(''.join(' %s\n' % l for l in output.splitlines()))
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700778 finally:
Vadim Shtayurac8437bf2014-07-09 19:45:36 -0700779 summary = output_collector.finalize()
780 if task_summary_json:
781 tools.write_json(task_summary_json, summary, False)
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700782
Marc-Antoine Rueld59e8072014-10-21 18:54:45 -0400783 if decorate and total_duration:
784 print('Total duration: %.1fs' % total_duration)
785
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -0400786 if len(seen_shards) != len(task_ids):
787 missing_shards = [x for x in range(len(task_ids)) if x not in seen_shards]
Junji Watanabecb054042020-07-21 08:43:26 +0000788 print(
789 'Results from some shards are missing: %s' %
790 ', '.join(map(str, missing_shards)),
791 file=sys.stderr)
Vadim Shtayurac524f512014-05-15 09:54:56 -0700792 return 1
Vadim Shtayurae3fbd102014-04-29 17:05:21 -0700793
maruela5490782015-09-30 10:56:59 -0700794 return exit_code if exit_code is not None else 1
maruel@chromium.org0437a732013-08-27 16:05:52 +0000795
796
maruel77f720b2015-09-15 12:35:22 -0700797### API management.
798
799
800class APIError(Exception):
801 pass
802
803
804def endpoints_api_discovery_apis(host):
805 """Uses Cloud Endpoints' API Discovery Service to returns metadata about all
806 the APIs exposed by a host.
807
808 https://developers.google.com/discovery/v1/reference/apis/list
809 """
maruel380e3262016-08-31 16:10:06 -0700810 # Uses the real Cloud Endpoints. This needs to be fixed once the Cloud
811 # Endpoints version is turned down.
maruel77f720b2015-09-15 12:35:22 -0700812 data = net.url_read_json(host + '/_ah/api/discovery/v1/apis')
813 if data is None:
814 raise APIError('Failed to discover APIs on %s' % host)
815 out = {}
816 for api in data['items']:
817 if api['id'] == 'discovery:v1':
818 continue
819 # URL is of the following form:
820 # url = host + (
821 # '/_ah/api/discovery/v1/apis/%s/%s/rest' % (api['id'], api['version'])
822 api_data = net.url_read_json(api['discoveryRestUrl'])
823 if api_data is None:
824 raise APIError('Failed to discover %s on %s' % (api['id'], host))
825 out[api['id']] = api_data
826 return out
827
828
maruelaf6b06c2017-06-08 06:26:53 -0700829def get_yielder(base_url, limit):
830 """Returns the first query and a function that yields following items."""
831 CHUNK_SIZE = 250
832
833 url = base_url
834 if limit:
835 url += '%slimit=%d' % ('&' if '?' in url else '?', min(CHUNK_SIZE, limit))
836 data = net.url_read_json(url)
837 if data is None:
838 # TODO(maruel): Do basic diagnostic.
839 raise Failure('Failed to access %s' % url)
840 org_cursor = data.pop('cursor', None)
841 org_total = len(data.get('items') or [])
842 logging.info('get_yielder(%s) returning %d items', base_url, org_total)
843 if not org_cursor or not org_total:
844 # This is not an iterable resource.
845 return data, lambda: []
846
847 def yielder():
848 cursor = org_cursor
849 total = org_total
850 # Some items support cursors. Try to get automatically if cursors are needed
851 # by looking at the 'cursor' items.
852 while cursor and (not limit or total < limit):
853 merge_char = '&' if '?' in base_url else '?'
Marc-Antoine Ruelad8cabe2019-10-10 23:24:26 +0000854 url = base_url + '%scursor=%s' % (merge_char, urllib.parse.quote(cursor))
maruelaf6b06c2017-06-08 06:26:53 -0700855 if limit:
856 url += '&limit=%d' % min(CHUNK_SIZE, limit - total)
857 new = net.url_read_json(url)
858 if new is None:
859 raise Failure('Failed to access %s' % url)
860 cursor = new.get('cursor')
861 new_items = new.get('items')
862 nb_items = len(new_items or [])
863 total += nb_items
864 logging.info('get_yielder(%s) yielding %d items', base_url, nb_items)
865 yield new_items
866
867 return data, yielder
868
869
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500870### Commands.
871
872
873def abort_task(_swarming, _manifest):
874 """Given a task manifest that was triggered, aborts its execution."""
875 # TODO(vadimsh): No supported by the server yet.
876
877
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400878def add_filter_options(parser):
maruel681d6802017-01-17 16:56:03 -0800879 parser.filter_group = optparse.OptionGroup(parser, 'Bot selection')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -0500880 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +0000881 '-d',
882 '--dimension',
883 default=[],
884 action='append',
885 nargs=2,
886 dest='dimensions',
887 metavar='FOO bar',
Marc-Antoine Ruel92f32422013-11-06 18:12:13 -0500888 help='dimension to filter on')
Brad Hallf78187a2018-10-19 17:08:55 +0000889 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +0000890 '--optional-dimension',
891 default=[],
892 action='append',
893 nargs=3,
894 dest='optional_dimensions',
895 metavar='key value expiration',
Brad Hallf78187a2018-10-19 17:08:55 +0000896 help='optional dimensions which will result in additional task slices ')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -0500897 parser.add_option_group(parser.filter_group)
898
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400899
Brad Hallf78187a2018-10-19 17:08:55 +0000900def _validate_filter_option(parser, key, value, expiration, argname):
901 if ':' in key:
902 parser.error('%s key cannot contain ":"' % argname)
903 if key.strip() != key:
904 parser.error('%s key has whitespace' % argname)
905 if not key:
906 parser.error('%s key is empty' % argname)
907
908 if value.strip() != value:
909 parser.error('%s value has whitespace' % argname)
910 if not value:
911 parser.error('%s value is empty' % argname)
912
913 if expiration is not None:
914 try:
915 expiration = int(expiration)
916 except ValueError:
917 parser.error('%s expiration is not an integer' % argname)
918 if expiration <= 0:
919 parser.error('%s expiration should be positive' % argname)
920 if expiration % 60 != 0:
921 parser.error('%s expiration is not divisible by 60' % argname)
922
923
maruelaf6b06c2017-06-08 06:26:53 -0700924def process_filter_options(parser, options):
925 for key, value in options.dimensions:
Brad Hallf78187a2018-10-19 17:08:55 +0000926 _validate_filter_option(parser, key, value, None, 'dimension')
927 for key, value, exp in options.optional_dimensions:
928 _validate_filter_option(parser, key, value, exp, 'optional-dimension')
maruelaf6b06c2017-06-08 06:26:53 -0700929 options.dimensions.sort()
930
931
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400932def add_trigger_options(parser):
933 """Adds all options to trigger a task on Swarming."""
Marc-Antoine Ruelf7d737d2014-12-10 15:36:29 -0500934 isolateserver.add_isolate_server_options(parser)
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -0400935 add_filter_options(parser)
936
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -0400937 group = optparse.OptionGroup(parser, 'TaskSlice properties')
maruel681d6802017-01-17 16:56:03 -0800938 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000939 '-s',
940 '--isolated',
941 metavar='HASH',
Marc-Antoine Ruel185ded42015-01-28 20:49:18 -0500942 help='Hash of the .isolated to grab from the isolate server')
maruel681d6802017-01-17 16:56:03 -0800943 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000944 '-e',
945 '--env',
946 default=[],
947 action='append',
948 nargs=2,
949 metavar='FOO bar',
Vadim Shtayurab450c602014-05-12 19:23:25 -0700950 help='Environment variables to set')
maruel681d6802017-01-17 16:56:03 -0800951 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000952 '--env-prefix',
953 default=[],
954 action='append',
955 nargs=2,
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800956 metavar='VAR local/path',
957 help='Prepend task-relative `local/path` to the task\'s VAR environment '
Junji Watanabe38b28b02020-04-23 10:23:30 +0000958 'variable using os-appropriate pathsep character. Can be specified '
959 'multiple times for the same VAR to add multiple paths.')
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800960 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000961 '--idempotent',
962 action='store_true',
963 default=False,
Marc-Antoine Ruel02196392014-10-17 16:29:43 -0400964 help='When set, the server will actively try to find a previous task '
Junji Watanabe38b28b02020-04-23 10:23:30 +0000965 'with the same parameter and return this result instead if possible')
maruel681d6802017-01-17 16:56:03 -0800966 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000967 '--secret-bytes-path',
968 metavar='FILE',
Stephen Martinisf391c772019-02-01 01:22:12 +0000969 help='The optional path to a file containing the secret_bytes to use '
Junji Watanabe38b28b02020-04-23 10:23:30 +0000970 'with this task.')
maruel681d6802017-01-17 16:56:03 -0800971 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000972 '--hard-timeout',
973 type='int',
974 default=60 * 60,
975 metavar='SECS',
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400976 help='Seconds to allow the task to complete.')
maruel681d6802017-01-17 16:56:03 -0800977 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000978 '--io-timeout',
979 type='int',
980 default=20 * 60,
981 metavar='SECS',
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -0400982 help='Seconds to allow the task to be silent.')
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +0000983 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000984 '--lower-priority',
985 action='store_true',
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +0000986 help='Lowers the child process priority')
Marc-Antoine Ruel7f61a4d2019-05-22 20:10:07 +0000987 containment_choices = ('NONE', 'AUTO', 'JOB_OBJECT')
988 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000989 '--containment-type',
990 default='NONE',
991 metavar='NONE',
Marc-Antoine Ruel7f61a4d2019-05-22 20:10:07 +0000992 choices=containment_choices,
993 help='Containment to use; one of: %s' % ', '.join(containment_choices))
maruel681d6802017-01-17 16:56:03 -0800994 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +0000995 '--raw-cmd',
996 action='store_true',
997 default=False,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -0500998 help='When set, the command after -- is used as-is without run_isolated. '
Junji Watanabe38b28b02020-04-23 10:23:30 +0000999 'In this case, the .isolated file is expected to not have a command')
maruel681d6802017-01-17 16:56:03 -08001000 group.add_option(
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001001 '--relative-cwd',
1002 help='Ignore the isolated \'relative_cwd\' and use this one instead; '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001003 'requires --raw-cmd')
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001004 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001005 '--cipd-package',
1006 action='append',
1007 default=[],
1008 metavar='PKG',
maruel5475ba62017-05-31 15:35:47 -07001009 help='CIPD packages to install on the Swarming bot. Uses the format: '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001010 'path:package_name:version')
maruel681d6802017-01-17 16:56:03 -08001011 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001012 '--named-cache',
1013 action='append',
1014 nargs=2,
1015 default=[],
maruel5475ba62017-05-31 15:35:47 -07001016 metavar='NAME RELPATH',
maruel681d6802017-01-17 16:56:03 -08001017 help='"<name> <relpath>" items to keep a persistent bot managed cache')
1018 group.add_option(
vadimsh93d167c2016-09-13 11:31:51 -07001019 '--service-account',
Vadim Shtayura2d83a942017-08-14 17:41:24 -07001020 help='Email of a service account to run the task as, or literal "bot" '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001021 'string to indicate that the task should use the same account the '
1022 'bot itself is using to authenticate to Swarming. Don\'t use task '
1023 'service accounts if not given (default).')
maruel681d6802017-01-17 16:56:03 -08001024 group.add_option(
Robert Iannuccifafa7352018-06-13 17:08:17 +00001025 '--pool-task-template',
1026 choices=('AUTO', 'CANARY_PREFER', 'CANARY_NEVER', 'SKIP'),
1027 default='AUTO',
1028 help='Set how you want swarming to apply the pool\'s TaskTemplate. '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001029 'By default, the pool\'s TaskTemplate is automatically selected, '
1030 'according the pool configuration on the server. Choices are: '
1031 'AUTO, CANARY_PREFER, CANARY_NEVER, and SKIP (default: AUTO).')
Robert Iannuccifafa7352018-06-13 17:08:17 +00001032 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001033 '-o',
1034 '--output',
1035 action='append',
1036 default=[],
1037 metavar='PATH',
maruel5475ba62017-05-31 15:35:47 -07001038 help='A list of files to return in addition to those written to '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001039 '${ISOLATED_OUTDIR}. An error will occur if a file specified by'
1040 'this option is also written directly to ${ISOLATED_OUTDIR}.')
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001041 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001042 '--wait-for-capacity',
1043 action='store_true',
1044 default=False,
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001045 help='Instructs to leave the task PENDING even if there\'s no known bot '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001046 'that could run this task, otherwise the task will be denied with '
1047 'NO_RESOURCE')
maruel681d6802017-01-17 16:56:03 -08001048 parser.add_option_group(group)
1049
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001050 group = optparse.OptionGroup(parser, 'TaskRequest details')
maruel681d6802017-01-17 16:56:03 -08001051 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001052 '--priority',
1053 type='int',
1054 default=200,
maruel681d6802017-01-17 16:56:03 -08001055 help='The lower value, the more important the task is')
1056 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001057 '-T',
1058 '--task-name',
1059 metavar='NAME',
maruel681d6802017-01-17 16:56:03 -08001060 help='Display name of the task. Defaults to '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001061 '<base_name>/<dimensions>/<isolated hash>/<timestamp> if an '
1062 'isolated file is provided, if a hash is provided, it defaults to '
1063 '<user>/<dimensions>/<isolated hash>/<timestamp>')
maruel681d6802017-01-17 16:56:03 -08001064 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001065 '--tags',
1066 action='append',
1067 default=[],
1068 metavar='FOO:BAR',
maruel681d6802017-01-17 16:56:03 -08001069 help='Tags to assign to the task.')
1070 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001071 '--user',
1072 default='',
maruel681d6802017-01-17 16:56:03 -08001073 help='User associated with the task. Defaults to authenticated user on '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001074 'the server.')
maruel681d6802017-01-17 16:56:03 -08001075 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001076 '--expiration',
1077 type='int',
1078 default=6 * 60 * 60,
1079 metavar='SECS',
maruel681d6802017-01-17 16:56:03 -08001080 help='Seconds to allow the task to be pending for a bot to run before '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001081 'this task request expires.')
maruel681d6802017-01-17 16:56:03 -08001082 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001083 '--deadline', type='int', dest='expiration', help=optparse.SUPPRESS_HELP)
Junji Watanabe71bbaef2020-07-21 08:55:37 +00001084 group.add_option(
1085 '--realm',
1086 dest='realm',
1087 metavar='REALM',
1088 help='Realm associated with the task.')
Scott Lee44c13d72020-09-14 06:09:50 +00001089 group.add_option(
1090 '--resultdb',
1091 action='store_true',
1092 default=False,
1093 help='When set, the task is created with ResultDB enabled.')
maruel681d6802017-01-17 16:56:03 -08001094 parser.add_option_group(group)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001095
1096
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001097def process_trigger_options(parser, options, args):
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001098 """Processes trigger options and does preparatory steps.
1099
1100 Returns:
1101 NewTaskRequest instance.
1102 """
maruelaf6b06c2017-06-08 06:26:53 -07001103 process_filter_options(parser, options)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001104 options.env = dict(options.env)
maruel0a25f6c2017-05-10 10:43:23 -07001105 if args and args[0] == '--':
1106 args = args[1:]
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001107
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001108 if not options.dimensions:
1109 parser.error('Please at least specify one --dimension')
Marc-Antoine Ruel33d198c2018-11-27 21:12:16 +00001110 if not any(k == 'pool' for k, _v in options.dimensions):
1111 parser.error('You must specify --dimension pool <value>')
maruel0a25f6c2017-05-10 10:43:23 -07001112 if not all(len(t.split(':', 1)) == 2 for t in options.tags):
1113 parser.error('--tags must be in the format key:value')
1114 if options.raw_cmd and not args:
1115 parser.error(
1116 'Arguments with --raw-cmd should be passed after -- as command '
1117 'delimiter.')
1118 if options.isolate_server and not options.namespace:
1119 parser.error(
1120 '--namespace must be a valid value when --isolate-server is used')
1121 if not options.isolated and not options.raw_cmd:
1122 parser.error('Specify at least one of --raw-cmd or --isolated or both')
1123
1124 # Isolated
1125 # --isolated is required only if --raw-cmd wasn't provided.
1126 # TODO(maruel): --isolate-server may be optional as Swarming may have its own
1127 # preferred server.
Takuto Ikutaae767b32020-05-11 01:22:19 +00001128 isolateserver.process_isolate_server_options(parser, options,
1129 not options.raw_cmd)
maruel0a25f6c2017-05-10 10:43:23 -07001130 inputs_ref = None
1131 if options.isolate_server:
1132 inputs_ref = FilesRef(
1133 isolated=options.isolated,
1134 isolatedserver=options.isolate_server,
1135 namespace=options.namespace)
1136
1137 # Command
1138 command = None
1139 extra_args = None
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001140 if options.raw_cmd:
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001141 command = args
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001142 if options.relative_cwd:
1143 a = os.path.normpath(os.path.abspath(options.relative_cwd))
1144 if not a.startswith(os.getcwd()):
1145 parser.error(
1146 '--relative-cwd must not try to escape the working directory')
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001147 else:
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001148 if options.relative_cwd:
1149 parser.error('--relative-cwd requires --raw-cmd')
maruel0a25f6c2017-05-10 10:43:23 -07001150 extra_args = args
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001151
maruel0a25f6c2017-05-10 10:43:23 -07001152 # CIPD
borenet02f772b2016-06-22 12:42:19 -07001153 cipd_packages = []
1154 for p in options.cipd_package:
1155 split = p.split(':', 2)
1156 if len(split) != 3:
1157 parser.error('CIPD packages must take the form: path:package:version')
Junji Watanabe38b28b02020-04-23 10:23:30 +00001158 cipd_packages.append(
1159 CipdPackage(package_name=split[1], path=split[0], version=split[2]))
borenet02f772b2016-06-22 12:42:19 -07001160 cipd_input = None
1161 if cipd_packages:
1162 cipd_input = CipdInput(
Junji Watanabecb054042020-07-21 08:43:26 +00001163 client_package=None, packages=cipd_packages, server=None)
borenet02f772b2016-06-22 12:42:19 -07001164
maruel0a25f6c2017-05-10 10:43:23 -07001165 # Secrets
iannuccidc80dfb2016-10-28 12:50:20 -07001166 secret_bytes = None
1167 if options.secret_bytes_path:
Marc-Antoine Ruel5c98fa72018-05-18 12:19:59 -04001168 with open(options.secret_bytes_path, 'rb') as f:
iannuccidc80dfb2016-10-28 12:50:20 -07001169 secret_bytes = f.read().encode('base64')
1170
maruel0a25f6c2017-05-10 10:43:23 -07001171 # Named caches
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001172 caches = [{
1173 u'name': six.text_type(i[0]),
1174 u'path': six.text_type(i[1])
1175 } for i in options.named_cache]
maruel0a25f6c2017-05-10 10:43:23 -07001176
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001177 env_prefixes = {}
1178 for k, v in options.env_prefix:
1179 env_prefixes.setdefault(k, []).append(v)
1180
Brad Hallf78187a2018-10-19 17:08:55 +00001181 # Get dimensions into the key/value format we can manipulate later.
Junji Watanabecb054042020-07-21 08:43:26 +00001182 orig_dims = [{
1183 'key': key,
1184 'value': value
1185 } for key, value in options.dimensions]
Brad Hallf78187a2018-10-19 17:08:55 +00001186 orig_dims.sort(key=lambda x: (x['key'], x['value']))
1187
1188 # Construct base properties that we will use for all the slices, adding in
1189 # optional dimensions for the fallback slices.
maruel77f720b2015-09-15 12:35:22 -07001190 properties = TaskProperties(
maruel681d6802017-01-17 16:56:03 -08001191 caches=caches,
borenet02f772b2016-06-22 12:42:19 -07001192 cipd_input=cipd_input,
maruel0a25f6c2017-05-10 10:43:23 -07001193 command=command,
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +00001194 containment=Containment(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001195 lower_priority=bool(options.lower_priority),
1196 containment_type=options.containment_type,
Marc-Antoine Ruel89669dc2019-05-01 14:01:08 +00001197 ),
Marc-Antoine Ruelba1bf222017-12-21 21:41:01 -05001198 relative_cwd=options.relative_cwd,
Brad Hallf78187a2018-10-19 17:08:55 +00001199 dimensions=orig_dims,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001200 env=options.env,
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001201 env_prefixes=[StringListPair(k, v) for k, v in env_prefixes.items()],
maruel77f720b2015-09-15 12:35:22 -07001202 execution_timeout_secs=options.hard_timeout,
maruel0a25f6c2017-05-10 10:43:23 -07001203 extra_args=extra_args,
maruel77f720b2015-09-15 12:35:22 -07001204 grace_period_secs=30,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001205 idempotent=options.idempotent,
maruel77f720b2015-09-15 12:35:22 -07001206 inputs_ref=inputs_ref,
aludwincc5524e2016-10-28 10:25:24 -07001207 io_timeout_secs=options.io_timeout,
iannuccidc80dfb2016-10-28 12:50:20 -07001208 outputs=options.output,
1209 secret_bytes=secret_bytes)
Brad Hallf78187a2018-10-19 17:08:55 +00001210
1211 slices = []
1212
1213 # Group the optional dimensions by expiration.
1214 dims_by_exp = {}
1215 for key, value, exp_secs in options.optional_dimensions:
Junji Watanabecb054042020-07-21 08:43:26 +00001216 dims_by_exp.setdefault(int(exp_secs), []).append({
1217 'key': key,
1218 'value': value
1219 })
Brad Hallf78187a2018-10-19 17:08:55 +00001220
1221 # Create the optional slices with expiration deltas, we fix up the properties
1222 # below.
1223 last_exp = 0
1224 for expiration_secs in sorted(dims_by_exp):
1225 t = TaskSlice(
1226 expiration_secs=expiration_secs - last_exp,
1227 properties=properties,
1228 wait_for_capacity=False)
1229 slices.append(t)
1230 last_exp = expiration_secs
1231
1232 # Add back in the default slice (the last one).
1233 exp = max(int(options.expiration) - last_exp, 60)
1234 base_task_slice = TaskSlice(
1235 expiration_secs=exp,
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001236 properties=properties,
1237 wait_for_capacity=options.wait_for_capacity)
Brad Hallf78187a2018-10-19 17:08:55 +00001238 slices.append(base_task_slice)
1239
Brad Hall7f463e62018-11-16 16:13:30 +00001240 # Add optional dimensions to the task slices, replacing a dimension that
1241 # has the same key if it is a dimension where repeating isn't valid (otherwise
1242 # we append it). Currently the only dimension we can repeat is "caches"; the
1243 # rest (os, cpu, etc) shouldn't be repeated.
Brad Hallf78187a2018-10-19 17:08:55 +00001244 extra_dims = []
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001245 for i, (_, kvs) in enumerate(sorted(dims_by_exp.items(), reverse=True)):
Brad Hallf78187a2018-10-19 17:08:55 +00001246 dims = list(orig_dims)
Brad Hall7f463e62018-11-16 16:13:30 +00001247 # Replace or append the key/value pairs for this expiration in extra_dims;
1248 # we keep extra_dims around because we are iterating backwards and filling
1249 # in slices with shorter expirations. Dimensions expire as time goes on so
1250 # the slices that expire earlier will generally have more dimensions.
1251 for kv in kvs:
1252 if kv['key'] == 'caches':
1253 extra_dims.append(kv)
1254 else:
1255 extra_dims = [x for x in extra_dims if x['key'] != kv['key']] + [kv]
1256 # Then, add all the optional dimensions to the original dimension set, again
1257 # replacing if needed.
1258 for kv in extra_dims:
1259 if kv['key'] == 'caches':
1260 dims.append(kv)
1261 else:
1262 dims = [x for x in dims if x['key'] != kv['key']] + [kv]
Brad Hallf78187a2018-10-19 17:08:55 +00001263 dims.sort(key=lambda x: (x['key'], x['value']))
1264 slice_properties = properties._replace(dimensions=dims)
1265 slices[-2 - i] = slices[-2 - i]._replace(properties=slice_properties)
1266
maruel77f720b2015-09-15 12:35:22 -07001267 return NewTaskRequest(
maruel0a25f6c2017-05-10 10:43:23 -07001268 name=default_task_name(options),
maruel77f720b2015-09-15 12:35:22 -07001269 parent_task_id=os.environ.get('SWARMING_TASK_ID', ''),
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001270 priority=options.priority,
Brad Hallf78187a2018-10-19 17:08:55 +00001271 task_slices=slices,
Vadim Shtayura2d83a942017-08-14 17:41:24 -07001272 service_account=options.service_account,
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001273 tags=options.tags,
Robert Iannuccifafa7352018-06-13 17:08:17 +00001274 user=options.user,
Junji Watanabe71bbaef2020-07-21 08:55:37 +00001275 pool_task_template=options.pool_task_template,
Scott Lee44c13d72020-09-14 06:09:50 +00001276 realm=options.realm,
1277 resultdb={'enable': options.resultdb})
maruel@chromium.org0437a732013-08-27 16:05:52 +00001278
1279
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001280class TaskOutputStdoutOption(optparse.Option):
1281 """Where to output the each task's console output (stderr/stdout).
1282
1283 The output will be;
1284 none - not be downloaded.
1285 json - stored in summary.json file *only*.
1286 console - shown on stdout *only*.
1287 all - stored in summary.json and shown on stdout.
1288 """
1289
1290 choices = ['all', 'json', 'console', 'none']
1291
1292 def __init__(self, *args, **kw):
1293 optparse.Option.__init__(
1294 self,
1295 *args,
1296 choices=self.choices,
Marc-Antoine Ruel28488842017-09-12 18:09:17 -04001297 default=['console', 'json'],
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +00001298 help=re.sub(r'\s\s*', ' ', self.__doc__),
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001299 **kw)
1300
1301 def convert_value(self, opt, value):
1302 if value not in self.choices:
Junji Watanabecb054042020-07-21 08:43:26 +00001303 raise optparse.OptionValueError(
1304 "%s must be one of %s not %r" %
1305 (self.get_opt_string(), self.choices, value))
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001306 stdout_to = []
1307 if value == 'all':
Marc-Antoine Ruel28488842017-09-12 18:09:17 -04001308 stdout_to = ['console', 'json']
Tim 'mithro' Ansell5e8001d2017-09-08 09:32:52 +10001309 elif value != 'none':
1310 stdout_to = [value]
1311 return stdout_to
1312
1313
maruel@chromium.org0437a732013-08-27 16:05:52 +00001314def add_collect_options(parser):
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001315 parser.server_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001316 '-t',
1317 '--timeout',
1318 type='float',
1319 default=0.,
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001320 help='Timeout to wait for result, set to -1 for no timeout and get '
Junji Watanabecb054042020-07-21 08:43:26 +00001321 'current state; defaults to waiting until the task completes')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001322 parser.group_logging.add_option(
1323 '--decorate', action='store_true', help='Decorate output')
Vadim Shtayura86a2cef2014-04-18 11:13:39 -07001324 parser.group_logging.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001325 '--print-status-updates',
1326 action='store_true',
Vadim Shtayura86a2cef2014-04-18 11:13:39 -07001327 help='Print periodic status updates')
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001328 parser.task_output_group = optparse.OptionGroup(parser, 'Task output')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001329 parser.task_output_group.add_option(
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001330 '--task-summary-json',
1331 metavar='FILE',
1332 help='Dump a summary of task results to this file as json. It contains '
Junji Watanabecb054042020-07-21 08:43:26 +00001333 'only shards statuses as know to server directly. Any output files '
1334 'emitted by the task can be collected by using --task-output-dir')
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001335 parser.task_output_group.add_option(
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001336 '--task-output-dir',
Vadim Shtayurac8437bf2014-07-09 19:45:36 -07001337 metavar='DIR',
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001338 help='Directory to put task results into. When the task finishes, this '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001339 'directory contains per-shard directory with output files produced '
1340 'by shards: <task-output-dir>/<zero-based-shard-index>/.')
maruel9531ce02016-04-13 06:11:23 -07001341 parser.task_output_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001342 TaskOutputStdoutOption('--task-output-stdout'))
Takuto Ikuta1e6072c2018-11-06 20:42:43 +00001343 parser.task_output_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001344 '--filepath-filter',
1345 help='This is regexp filter used to specify downloaded filepath when '
1346 'collecting isolated output.')
1347 parser.task_output_group.add_option(
1348 '--perf',
1349 action='store_true',
1350 default=False,
maruel9531ce02016-04-13 06:11:23 -07001351 help='Includes performance statistics')
Vadim Shtayurae3fbd102014-04-29 17:05:21 -07001352 parser.add_option_group(parser.task_output_group)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001353
1354
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001355def process_collect_options(parser, options):
1356 # Only negative -1 is allowed, disallow other negative values.
1357 if options.timeout != -1 and options.timeout < 0:
1358 parser.error('Invalid --timeout value')
1359
1360
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001361@subcommand.usage('bots...')
1362def CMDbot_delete(parser, args):
1363 """Forcibly deletes bots from the Swarming server."""
1364 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001365 '-f',
1366 '--force',
1367 action='store_true',
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001368 help='Do not prompt for confirmation')
1369 options, args = parser.parse_args(args)
1370 if not args:
maruelfd0a90c2016-06-10 11:51:10 -07001371 parser.error('Please specify bots to delete')
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001372
1373 bots = sorted(args)
1374 if not options.force:
1375 print('Delete the following bots?')
1376 for bot in bots:
1377 print(' %s' % bot)
1378 if raw_input('Continue? [y/N] ') not in ('y', 'Y'):
1379 print('Goodbye.')
1380 return 1
1381
1382 result = 0
1383 for bot in bots:
smut281c3902018-05-30 17:50:05 -07001384 url = '%s/_ah/api/swarming/v1/bot/%s/delete' % (options.swarming, bot)
vadimshe4c0e242015-09-30 11:53:54 -07001385 if net.url_read_json(url, data={}, method='POST') is None:
1386 print('Deleting %s failed. Probably already gone' % bot)
Marc-Antoine Ruel13e7c882015-03-26 18:19:10 -04001387 result = 1
1388 return result
1389
1390
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001391def CMDbots(parser, args):
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001392 """Returns information about the bots connected to the Swarming server."""
1393 add_filter_options(parser)
1394 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001395 '--dead-only',
1396 action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001397 help='Filter out bots alive, useful to reap them and reimage broken bots')
Marc-Antoine Ruel28083112014-03-13 16:34:04 -04001398 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001399 '-k',
1400 '--keep-dead',
1401 action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001402 help='Keep both dead and alive bots')
1403 parser.filter_group.add_option(
1404 '--busy', action='store_true', help='Keep only busy bots')
1405 parser.filter_group.add_option(
1406 '--idle', action='store_true', help='Keep only idle bots')
1407 parser.filter_group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001408 '--mp',
1409 action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001410 help='Keep only Machine Provider managed bots')
1411 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001412 '--non-mp',
1413 action='store_true',
maruelaf6b06c2017-06-08 06:26:53 -07001414 help='Keep only non Machine Provider managed bots')
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001415 parser.filter_group.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001416 '-b', '--bare', action='store_true', help='Do not print out dimensions')
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001417 options, args = parser.parse_args(args)
maruelaf6b06c2017-06-08 06:26:53 -07001418 process_filter_options(parser, options)
Marc-Antoine Ruel28083112014-03-13 16:34:04 -04001419
1420 if options.keep_dead and options.dead_only:
maruelaf6b06c2017-06-08 06:26:53 -07001421 parser.error('Use only one of --keep-dead or --dead-only')
1422 if options.busy and options.idle:
1423 parser.error('Use only one of --busy or --idle')
1424 if options.mp and options.non_mp:
1425 parser.error('Use only one of --mp or --non-mp')
Vadim Shtayura6b555c12014-07-23 16:22:18 -07001426
smut281c3902018-05-30 17:50:05 -07001427 url = options.swarming + '/_ah/api/swarming/v1/bots/list?'
maruelaf6b06c2017-06-08 06:26:53 -07001428 values = []
1429 if options.dead_only:
1430 values.append(('is_dead', 'TRUE'))
1431 elif options.keep_dead:
1432 values.append(('is_dead', 'NONE'))
1433 else:
1434 values.append(('is_dead', 'FALSE'))
Marc-Antoine Ruelc6c579e2014-09-08 18:43:45 -04001435
maruelaf6b06c2017-06-08 06:26:53 -07001436 if options.busy:
1437 values.append(('is_busy', 'TRUE'))
1438 elif options.idle:
1439 values.append(('is_busy', 'FALSE'))
1440 else:
1441 values.append(('is_busy', 'NONE'))
1442
1443 if options.mp:
1444 values.append(('is_mp', 'TRUE'))
1445 elif options.non_mp:
1446 values.append(('is_mp', 'FALSE'))
1447 else:
1448 values.append(('is_mp', 'NONE'))
1449
1450 for key, value in options.dimensions:
1451 values.append(('dimensions', '%s:%s' % (key, value)))
Marc-Antoine Ruelad8cabe2019-10-10 23:24:26 +00001452 url += urllib.parse.urlencode(values)
maruelaf6b06c2017-06-08 06:26:53 -07001453 try:
1454 data, yielder = get_yielder(url, 0)
1455 bots = data.get('items') or []
1456 for items in yielder():
1457 if items:
1458 bots.extend(items)
1459 except Failure as e:
1460 sys.stderr.write('\n%s\n' % e)
1461 return 1
maruel77f720b2015-09-15 12:35:22 -07001462 for bot in natsort.natsorted(bots, key=lambda x: x['bot_id']):
Lei Leife202df2019-06-11 17:33:34 +00001463 print(bot['bot_id'])
maruelaf6b06c2017-06-08 06:26:53 -07001464 if not options.bare:
1465 dimensions = {i['key']: i.get('value') for i in bot.get('dimensions', {})}
Lei Leife202df2019-06-11 17:33:34 +00001466 print(' %s' % json.dumps(dimensions, sort_keys=True))
maruelaf6b06c2017-06-08 06:26:53 -07001467 if bot.get('task_id'):
Lei Leife202df2019-06-11 17:33:34 +00001468 print(' task: %s' % bot['task_id'])
Marc-Antoine Ruel819fb162014-03-12 16:38:26 -04001469 return 0
1470
1471
maruelfd0a90c2016-06-10 11:51:10 -07001472@subcommand.usage('task_id')
1473def CMDcancel(parser, args):
1474 """Cancels a task."""
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001475 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001476 '-k',
1477 '--kill-running',
1478 action='store_true',
1479 default=False,
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001480 help='Kill the task even if it was running')
maruelfd0a90c2016-06-10 11:51:10 -07001481 options, args = parser.parse_args(args)
1482 if not args:
1483 parser.error('Please specify the task to cancel')
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001484 data = {'kill_running': options.kill_running}
maruelfd0a90c2016-06-10 11:51:10 -07001485 for task_id in args:
smut281c3902018-05-30 17:50:05 -07001486 url = '%s/_ah/api/swarming/v1/task/%s/cancel' % (options.swarming, task_id)
Marc-Antoine Ruel2e52c552018-03-26 19:27:36 -04001487 resp = net.url_read_json(url, data=data, method='POST')
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001488 if resp is None:
maruelfd0a90c2016-06-10 11:51:10 -07001489 print('Deleting %s failed. Probably already gone' % task_id)
1490 return 1
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001491 logging.info('%s', resp)
maruelfd0a90c2016-06-10 11:51:10 -07001492 return 0
1493
1494
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001495@subcommand.usage('--json file | task_id...')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001496def CMDcollect(parser, args):
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001497 """Retrieves results of one or multiple Swarming task by its ID.
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001498
1499 The result can be in multiple part if the execution was sharded. It can
1500 potentially have retries.
1501 """
1502 add_collect_options(parser)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001503 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001504 '-j',
1505 '--json',
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001506 help='Load the task ids from .json as saved by trigger --dump-json')
maruel77f720b2015-09-15 12:35:22 -07001507 options, args = parser.parse_args(args)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001508 process_collect_options(parser, options)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001509 if not args and not options.json:
1510 parser.error('Must specify at least one task id or --json.')
1511 if args and options.json:
1512 parser.error('Only use one of task id or --json.')
1513
1514 if options.json:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001515 options.json = six.text_type(os.path.abspath(options.json))
Marc-Antoine Ruel9025a782015-03-17 16:42:59 -04001516 try:
maruel1ceb3872015-10-14 06:10:44 -07001517 with fs.open(options.json, 'rb') as f:
maruel71c61c82016-02-22 06:52:05 -08001518 data = json.load(f)
1519 except (IOError, ValueError):
1520 parser.error('Failed to open %s' % options.json)
1521 try:
Junji Watanabe38b28b02020-04-23 10:23:30 +00001522 tasks = sorted(data['tasks'].values(), key=lambda x: x['shard_index'])
maruel71c61c82016-02-22 06:52:05 -08001523 args = [t['task_id'] for t in tasks]
1524 except (KeyError, TypeError):
1525 parser.error('Failed to process %s' % options.json)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001526 if not options.timeout:
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -04001527 # Take in account all the task slices.
1528 offset = 0
1529 for s in data['request']['task_slices']:
Junji Watanabecb054042020-07-21 08:43:26 +00001530 m = (
1531 offset + s['properties']['execution_timeout_secs'] +
1532 s['expiration_secs'])
Marc-Antoine Ruelb73066b2018-04-19 20:16:55 -04001533 if m > options.timeout:
1534 options.timeout = m
1535 offset += s['expiration_secs']
Marc-Antoine Ruel9fc42612018-04-20 08:34:22 -04001536 options.timeout += 10.
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001537 else:
1538 valid = frozenset('0123456789abcdef')
1539 if any(not valid.issuperset(task_id) for task_id in args):
1540 parser.error('Task ids are 0-9a-f.')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001541
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001542 try:
Junji Watanabe38b28b02020-04-23 10:23:30 +00001543 return collect(options.swarming, args, options.timeout, options.decorate,
1544 options.print_status_updates, options.task_summary_json,
1545 options.task_output_dir, options.task_output_stdout,
1546 options.perf, options.filepath_filter)
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001547 except Failure:
1548 on_error.report(None)
1549 return 1
1550
1551
maruel77f720b2015-09-15 12:35:22 -07001552@subcommand.usage('[method name]')
Marc-Antoine Ruel833f5eb2018-04-25 16:49:40 -04001553def CMDpost(parser, args):
1554 """Sends a JSON RPC POST to one API endpoint and prints out the raw result.
1555
1556 Input data must be sent to stdin, result is printed to stdout.
1557
1558 If HTTP response code >= 400, returns non-zero.
1559 """
1560 options, args = parser.parse_args(args)
1561 if len(args) != 1:
1562 parser.error('Must specify only API name')
smut281c3902018-05-30 17:50:05 -07001563 url = options.swarming + '/_ah/api/swarming/v1/' + args[0]
Marc-Antoine Ruel833f5eb2018-04-25 16:49:40 -04001564 data = sys.stdin.read()
1565 try:
1566 resp = net.url_read(url, data=data, method='POST')
1567 except net.TimeoutError:
1568 sys.stderr.write('Timeout!\n')
1569 return 1
1570 if not resp:
1571 sys.stderr.write('No response!\n')
1572 return 1
1573 sys.stdout.write(resp)
1574 return 0
1575
1576
1577@subcommand.usage('[method name]')
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001578def CMDquery(parser, args):
maruel77f720b2015-09-15 12:35:22 -07001579 """Returns raw JSON information via an URL endpoint. Use 'query-list' to
1580 gather the list of API methods from the server.
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001581
1582 Examples:
maruelaf6b06c2017-06-08 06:26:53 -07001583 Raw task request and results:
1584 swarming.py query -S server-url.com task/123456/request
1585 swarming.py query -S server-url.com task/123456/result
1586
maruel77f720b2015-09-15 12:35:22 -07001587 Listing all bots:
maruel84e77aa2015-10-21 06:37:24 -07001588 swarming.py query -S server-url.com bots/list
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001589
maruelaf6b06c2017-06-08 06:26:53 -07001590 Listing last 10 tasks on a specific bot named 'bot1':
1591 swarming.py query -S server-url.com --limit 10 bot/bot1/tasks
maruel84e77aa2015-10-21 06:37:24 -07001592
maruelaf6b06c2017-06-08 06:26:53 -07001593 Listing last 10 tasks with tags os:Ubuntu-14.04 and pool:Chrome. Note that
maruel84e77aa2015-10-21 06:37:24 -07001594 quoting is important!:
1595 swarming.py query -S server-url.com --limit 10 \\
maruelaf6b06c2017-06-08 06:26:53 -07001596 'tasks/list?tags=os:Ubuntu-14.04&tags=pool:Chrome'
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001597 """
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001598 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001599 '-L',
1600 '--limit',
1601 type='int',
1602 default=200,
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001603 help='Limit to enforce on limitless items (like number of tasks); '
Junji Watanabecb054042020-07-21 08:43:26 +00001604 'default=%default')
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001605 parser.add_option(
1606 '--json', help='Path to JSON output file (otherwise prints to stdout)')
maruel77f720b2015-09-15 12:35:22 -07001607 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001608 '--progress',
1609 action='store_true',
maruel77f720b2015-09-15 12:35:22 -07001610 help='Prints a dot at each request to show progress')
1611 options, args = parser.parse_args(args)
marueld8aba222015-09-03 12:21:19 -07001612 if len(args) != 1:
maruel77f720b2015-09-15 12:35:22 -07001613 parser.error(
1614 'Must specify only method name and optionally query args properly '
1615 'escaped.')
smut281c3902018-05-30 17:50:05 -07001616 base_url = options.swarming + '/_ah/api/swarming/v1/' + args[0]
maruelaf6b06c2017-06-08 06:26:53 -07001617 try:
1618 data, yielder = get_yielder(base_url, options.limit)
1619 for items in yielder():
1620 if items:
1621 data['items'].extend(items)
maruel77f720b2015-09-15 12:35:22 -07001622 if options.progress:
maruelaf6b06c2017-06-08 06:26:53 -07001623 sys.stderr.write('.')
1624 sys.stderr.flush()
1625 except Failure as e:
1626 sys.stderr.write('\n%s\n' % e)
1627 return 1
maruel77f720b2015-09-15 12:35:22 -07001628 if options.progress:
maruelaf6b06c2017-06-08 06:26:53 -07001629 sys.stderr.write('\n')
1630 sys.stderr.flush()
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001631 if options.json:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001632 options.json = six.text_type(os.path.abspath(options.json))
maruel1ceb3872015-10-14 06:10:44 -07001633 tools.write_json(options.json, data, True)
Paweł Hajdan, Jr53ef0132015-03-20 17:49:18 +01001634 else:
Marc-Antoine Ruelcda90ee2015-03-23 15:13:20 -04001635 try:
maruel77f720b2015-09-15 12:35:22 -07001636 tools.write_json(sys.stdout, data, False)
Marc-Antoine Ruelcda90ee2015-03-23 15:13:20 -04001637 sys.stdout.write('\n')
1638 except IOError:
1639 pass
Marc-Antoine Ruel79940ae2014-09-23 17:55:41 -04001640 return 0
1641
1642
maruel77f720b2015-09-15 12:35:22 -07001643def CMDquery_list(parser, args):
1644 """Returns list of all the Swarming APIs that can be used with command
1645 'query'.
1646 """
1647 parser.add_option(
1648 '--json', help='Path to JSON output file (otherwise prints to stdout)')
1649 options, args = parser.parse_args(args)
1650 if args:
1651 parser.error('No argument allowed.')
1652
1653 try:
1654 apis = endpoints_api_discovery_apis(options.swarming)
1655 except APIError as e:
1656 parser.error(str(e))
1657 if options.json:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001658 options.json = six.text_type(os.path.abspath(options.json))
maruel1ceb3872015-10-14 06:10:44 -07001659 with fs.open(options.json, 'wb') as f:
maruel77f720b2015-09-15 12:35:22 -07001660 json.dump(apis, f)
1661 else:
1662 help_url = (
Junji Watanabecb054042020-07-21 08:43:26 +00001663 'https://apis-explorer.appspot.com/apis-explorer/?base=%s/_ah/api#p/' %
1664 options.swarming)
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001665 for i, (api_id, api) in enumerate(sorted(apis.items())):
maruel11e31af2017-02-15 07:30:50 -08001666 if i:
1667 print('')
Lei Leife202df2019-06-11 17:33:34 +00001668 print(api_id)
1669 print(' ' + api['description'].strip())
maruel11e31af2017-02-15 07:30:50 -08001670 if 'resources' in api:
1671 # Old.
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +00001672 # TODO(maruel): Remove.
1673 # pylint: disable=too-many-nested-blocks
Junji Watanabecb054042020-07-21 08:43:26 +00001674 for j, (resource_name,
1675 resource) in enumerate(sorted(api['resources'].items())):
maruel11e31af2017-02-15 07:30:50 -08001676 if j:
1677 print('')
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001678 for method_name, method in sorted(resource['methods'].items()):
maruel11e31af2017-02-15 07:30:50 -08001679 # Only list the GET ones.
1680 if method['httpMethod'] != 'GET':
1681 continue
Junji Watanabecb054042020-07-21 08:43:26 +00001682 print('- %s.%s: %s' % (resource_name, method_name, method['path']))
1683 print('\n'.join(' ' + l for l in textwrap.wrap(
1684 method.get('description', 'No description'), 78)))
Lei Leife202df2019-06-11 17:33:34 +00001685 print(' %s%s%s' % (help_url, api['servicePath'], method['id']))
maruel11e31af2017-02-15 07:30:50 -08001686 else:
1687 # New.
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001688 for method_name, method in sorted(api['methods'].items()):
maruel77f720b2015-09-15 12:35:22 -07001689 # Only list the GET ones.
1690 if method['httpMethod'] != 'GET':
1691 continue
Lei Leife202df2019-06-11 17:33:34 +00001692 print('- %s: %s' % (method['id'], method['path']))
maruel11e31af2017-02-15 07:30:50 -08001693 print('\n'.join(
1694 ' ' + l for l in textwrap.wrap(method['description'], 78)))
Lei Leife202df2019-06-11 17:33:34 +00001695 print(' %s%s%s' % (help_url, api['servicePath'], method['id']))
maruel77f720b2015-09-15 12:35:22 -07001696 return 0
1697
1698
Vadim Shtayuraae8085b2014-05-02 17:13:10 -07001699@subcommand.usage('(hash|isolated) [-- extra_args]')
maruel@chromium.org0437a732013-08-27 16:05:52 +00001700def CMDrun(parser, args):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001701 """Triggers a task and wait for the results.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001702
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001703 Basically, does everything to run a command remotely.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001704 """
1705 add_trigger_options(parser)
1706 add_collect_options(parser)
1707 options, args = parser.parse_args(args)
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001708 process_collect_options(parser, options)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001709 task_request = process_trigger_options(parser, options, args)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001710 try:
Ye Kuang3f3f2f72020-10-21 10:14:59 +00001711 tasks = trigger_task_shards(options.swarming, task_request)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001712 except Failure as e:
Junji Watanabecb054042020-07-21 08:43:26 +00001713 on_error.report('Failed to trigger %s(%s): %s' %
1714 (task_request.name, args[0], e.args[0]))
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001715 return 1
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001716 if not tasks:
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001717 on_error.report('Failed to trigger the task.')
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001718 return 1
maruel0a25f6c2017-05-10 10:43:23 -07001719 print('Triggered task: %s' % task_request.name)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001720 task_ids = [
Junji Watanabe38b28b02020-04-23 10:23:30 +00001721 t['task_id']
1722 for t in sorted(tasks.values(), key=lambda x: x['shard_index'])
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001723 ]
Caleb Rouleau779c4f02019-05-22 21:18:49 +00001724 for task_id in task_ids:
1725 print('Task: {server}/task?id={task}'.format(
1726 server=options.swarming, task=task_id))
Marc-Antoine Ruelf24f09c2018-03-23 16:06:18 -04001727 if not options.timeout:
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001728 offset = 0
1729 for s in task_request.task_slices:
Junji Watanabecb054042020-07-21 08:43:26 +00001730 m = (offset + s.properties.execution_timeout_secs + s.expiration_secs)
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001731 if m > options.timeout:
1732 options.timeout = m
Marc-Antoine Ruel1f835c72018-05-25 12:29:42 -04001733 offset += s.expiration_secs
Marc-Antoine Ruel3a030bc2018-04-23 10:31:25 -04001734 options.timeout += 10.
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001735 try:
Junji Watanabecb054042020-07-21 08:43:26 +00001736 return collect(options.swarming, task_ids, options.timeout,
1737 options.decorate, options.print_status_updates,
1738 options.task_summary_json, options.task_output_dir,
1739 options.task_output_stdout, options.perf,
1740 options.filepath_filter)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001741 except Failure:
1742 on_error.report(None)
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001743 return 1
maruel@chromium.org0437a732013-08-27 16:05:52 +00001744
1745
maruel18122c62015-10-23 06:31:23 -07001746@subcommand.usage('task_id -- <extra_args>')
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001747def CMDreproduce(parser, args):
1748 """Runs a task locally that was triggered on the server.
1749
1750 This running locally the same commands that have been run on the bot. The data
1751 downloaded will be in a subdirectory named 'work' of the current working
1752 directory.
maruel18122c62015-10-23 06:31:23 -07001753
1754 You can pass further additional arguments to the target command by passing
1755 them after --.
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001756 """
maruelc070e672016-02-22 17:32:57 -08001757 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001758 '--output',
1759 metavar='DIR',
1760 default='out',
maruelc070e672016-02-22 17:32:57 -08001761 help='Directory that will have results stored into')
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001762 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001763 '--work',
1764 metavar='DIR',
1765 default='work',
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001766 help='Directory to map the task input files into')
1767 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001768 '--cache',
1769 metavar='DIR',
1770 default='cache',
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001771 help='Directory that contains the input cache')
1772 parser.add_option(
Junji Watanabecb054042020-07-21 08:43:26 +00001773 '--leak',
1774 action='store_true',
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001775 help='Do not delete the working directory after execution')
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001776 options, args = parser.parse_args(args)
maruel18122c62015-10-23 06:31:23 -07001777 extra_args = []
1778 if not args:
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001779 parser.error('Must specify exactly one task id.')
maruel18122c62015-10-23 06:31:23 -07001780 if len(args) > 1:
1781 if args[1] == '--':
1782 if len(args) > 2:
1783 extra_args = args[2:]
1784 else:
1785 extra_args = args[1:]
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001786
smut281c3902018-05-30 17:50:05 -07001787 url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0]
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001788 request = net.url_read_json(url)
1789 if not request:
Lei Leife202df2019-06-11 17:33:34 +00001790 print('Failed to retrieve request data for the task', file=sys.stderr)
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001791 return 1
1792
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001793 workdir = six.text_type(os.path.abspath(options.work))
maruele7cd38e2016-03-01 19:12:48 -08001794 if fs.isdir(workdir):
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001795 parser.error('Please delete the directory %r first' % options.work)
maruele7cd38e2016-03-01 19:12:48 -08001796 fs.mkdir(workdir)
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001797 cachedir = six.text_type(os.path.abspath('cipd_cache'))
iannucci31ab9192017-05-02 19:11:56 -07001798 if not fs.exists(cachedir):
1799 fs.mkdir(cachedir)
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001800
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001801 properties = request['properties']
iannucci31ab9192017-05-02 19:11:56 -07001802 env = os.environ.copy()
1803 env['SWARMING_BOT_ID'] = 'reproduce'
1804 env['SWARMING_TASK_ID'] = 'reproduce'
maruel29ab2fd2015-10-16 11:44:01 -07001805 if properties.get('env'):
Marc-Antoine Ruel119b0842014-12-19 15:27:58 -05001806 logging.info('env: %r', properties['env'])
maruelb76604c2015-11-11 11:53:44 -08001807 for i in properties['env']:
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001808 key = i['key']
maruelb76604c2015-11-11 11:53:44 -08001809 if not i['value']:
1810 env.pop(key, None)
1811 else:
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001812 env[key] = i['value']
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001813
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001814 if properties.get('env_prefixes'):
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001815 env_prefixes = properties['env_prefixes']
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001816 logging.info('env_prefixes: %r', env_prefixes)
Marc-Antoine Ruel36e09792018-01-09 14:03:25 -05001817 for i in env_prefixes:
1818 key = i['key']
1819 paths = [os.path.normpath(os.path.join(workdir, p)) for p in i['value']]
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001820 cur = env.get(key)
1821 if cur:
1822 paths.append(cur)
1823 env[key] = os.path.pathsep.join(paths)
1824
iannucci31ab9192017-05-02 19:11:56 -07001825 command = []
nodir152cba62016-05-12 16:08:56 -07001826 if (properties.get('inputs_ref') or {}).get('isolated'):
maruel29ab2fd2015-10-16 11:44:01 -07001827 # Create the tree.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001828 server_ref = isolate_storage.ServerRef(
Junji Watanabecb054042020-07-21 08:43:26 +00001829 properties['inputs_ref']['isolatedserver'],
1830 properties['inputs_ref']['namespace'])
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001831 with isolateserver.get_storage(server_ref) as storage:
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001832 # Do not use MemoryContentAddressedCache here, as on 32-bits python,
1833 # inputs larger than ~1GiB will not fit in memory. This is effectively a
1834 # leak.
1835 policies = local_caching.CachePolicies(0, 0, 0, 0)
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001836 cache = local_caching.DiskContentAddressedCache(
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001837 six.text_type(os.path.abspath(options.cache)), policies, False)
maruel29ab2fd2015-10-16 11:44:01 -07001838 bundle = isolateserver.fetch_isolated(
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001839 properties['inputs_ref']['isolated'], storage, cache, workdir, False)
maruel29ab2fd2015-10-16 11:44:01 -07001840 command = bundle.command
1841 if bundle.relative_cwd:
1842 workdir = os.path.join(workdir, bundle.relative_cwd)
maruela1b9e552016-01-06 12:42:03 -08001843 command.extend(properties.get('extra_args') or [])
iannucci31ab9192017-05-02 19:11:56 -07001844
1845 if properties.get('command'):
1846 command.extend(properties['command'])
1847
Marc-Antoine Ruelc7243592018-05-24 17:04:04 -04001848 # https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Magic-Values.md
Brian Sheedy7a761172019-08-30 22:55:14 +00001849 command = tools.find_executable(command, env)
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001850 if not options.output:
Marc-Antoine Ruel88229872018-01-10 16:35:29 -05001851 new_command = run_isolated.process_command(command, 'invalid', None)
1852 if new_command != command:
Marc-Antoine Ruel29ba75c2018-01-10 15:04:14 -05001853 parser.error('The task has outputs, you must use --output-dir')
Marc-Antoine Ruel88229872018-01-10 16:35:29 -05001854 else:
1855 # Make the path absolute, as the process will run from a subdirectory.
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001856 options.output = os.path.abspath(options.output)
Junji Watanabecb054042020-07-21 08:43:26 +00001857 new_command = run_isolated.process_command(command, options.output, None)
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001858 if not os.path.isdir(options.output):
1859 os.makedirs(options.output)
iannucci31ab9192017-05-02 19:11:56 -07001860 command = new_command
1861 file_path.ensure_command_has_abs_path(command, workdir)
1862
1863 if properties.get('cipd_input'):
1864 ci = properties['cipd_input']
1865 cp = ci['client_package']
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001866 client_manager = cipd.get_client(cachedir, ci['server'], cp['package_name'],
1867 cp['version'])
iannucci31ab9192017-05-02 19:11:56 -07001868
1869 with client_manager as client:
1870 by_path = collections.defaultdict(list)
1871 for pkg in ci['packages']:
1872 path = pkg['path']
1873 # cipd deals with 'root' as ''
1874 if path == '.':
1875 path = ''
1876 by_path[path].append((pkg['package_name'], pkg['version']))
1877 client.ensure(workdir, by_path, cache_dir=cachedir)
1878
maruel77f720b2015-09-15 12:35:22 -07001879 try:
Marc-Antoine Ruel95c21872018-01-10 14:24:28 -05001880 return subprocess42.call(command + extra_args, env=env, cwd=workdir)
maruel77f720b2015-09-15 12:35:22 -07001881 except OSError as e:
Lei Leife202df2019-06-11 17:33:34 +00001882 print('Failed to run: %s' % ' '.join(command), file=sys.stderr)
1883 print(str(e), file=sys.stderr)
maruel77f720b2015-09-15 12:35:22 -07001884 return 1
Marc-Antoine Ruel5aeb3bb2018-06-16 13:11:02 +00001885 finally:
1886 # Do not delete options.cache.
1887 if not options.leak:
1888 file_path.rmtree(workdir)
Marc-Antoine Ruel13a81272014-10-07 20:16:43 -04001889
1890
maruel0eb1d1b2015-10-02 14:48:21 -07001891@subcommand.usage('bot_id')
1892def CMDterminate(parser, args):
1893 """Tells a bot to gracefully shut itself down as soon as it can.
1894
1895 This is done by completing whatever current task there is then exiting the bot
1896 process.
1897 """
1898 parser.add_option(
1899 '--wait', action='store_true', help='Wait for the bot to terminate')
1900 options, args = parser.parse_args(args)
1901 if len(args) != 1:
1902 parser.error('Please provide the bot id')
smut281c3902018-05-30 17:50:05 -07001903 url = options.swarming + '/_ah/api/swarming/v1/bot/%s/terminate' % args[0]
maruel0eb1d1b2015-10-02 14:48:21 -07001904 request = net.url_read_json(url, data={})
1905 if not request:
Lei Leife202df2019-06-11 17:33:34 +00001906 print('Failed to ask for termination', file=sys.stderr)
maruel0eb1d1b2015-10-02 14:48:21 -07001907 return 1
1908 if options.wait:
Junji Watanabecb054042020-07-21 08:43:26 +00001909 return collect(options.swarming, [request['task_id']], 0., False, False,
1910 None, None, [], False, None)
maruelbfc5f872017-06-10 16:43:17 -07001911 else:
Lei Leife202df2019-06-11 17:33:34 +00001912 print(request['task_id'])
maruel0eb1d1b2015-10-02 14:48:21 -07001913 return 0
1914
1915
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001916@subcommand.usage("(hash|isolated) [-- extra_args|raw command]")
maruel@chromium.org0437a732013-08-27 16:05:52 +00001917def CMDtrigger(parser, args):
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001918 """Triggers a Swarming task.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001919
Vadim Shtayuraae8085b2014-05-02 17:13:10 -07001920 Passes all extra arguments provided after '--' as additional command line
1921 arguments for an isolated command specified in *.isolate file.
maruel@chromium.org0437a732013-08-27 16:05:52 +00001922 """
1923 add_trigger_options(parser)
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001924 parser.add_option(
1925 '--dump-json',
1926 metavar='FILE',
1927 help='Dump details about the triggered task(s) to this file as json')
Marc-Antoine Ruel7c543272013-11-26 13:26:15 -05001928 options, args = parser.parse_args(args)
Marc-Antoine Ruelefdc5282014-12-12 19:31:00 -05001929 task_request = process_trigger_options(parser, options, args)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001930 try:
Ye Kuang3f3f2f72020-10-21 10:14:59 +00001931 tasks = trigger_task_shards(options.swarming, task_request)
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001932 if tasks:
maruel0a25f6c2017-05-10 10:43:23 -07001933 print('Triggered task: %s' % task_request.name)
Junji Watanabecb054042020-07-21 08:43:26 +00001934 tasks_sorted = sorted(tasks.values(), key=lambda x: x['shard_index'])
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001935 if options.dump_json:
1936 data = {
Junji Watanabe38b28b02020-04-23 10:23:30 +00001937 'base_task_name': task_request.name,
1938 'tasks': tasks,
1939 'request': task_request_to_raw_request(task_request),
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001940 }
maruel46b015f2015-10-13 18:40:35 -07001941 tools.write_json(unicode(options.dump_json), data, True)
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001942 print('To collect results, use:')
Bruce Dawsonf0a5ae42018-09-04 20:06:46 +00001943 print(' tools/swarming_client/swarming.py collect -S %s --json %s' %
Junji Watanabecb054042020-07-21 08:43:26 +00001944 (options.swarming, options.dump_json))
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001945 else:
Marc-Antoine Ruel12a7da42014-10-01 08:29:47 -04001946 print('To collect results, use:')
Bruce Dawsonf0a5ae42018-09-04 20:06:46 +00001947 print(' tools/swarming_client/swarming.py collect -S %s %s' %
Junji Watanabecb054042020-07-21 08:43:26 +00001948 (options.swarming, ' '.join(t['task_id'] for t in tasks_sorted)))
Marc-Antoine Ruel2f6581a2014-10-03 11:09:53 -04001949 print('Or visit:')
1950 for t in tasks_sorted:
1951 print(' ' + t['view_url'])
Marc-Antoine Rueld6dbe762014-06-18 13:49:42 -04001952 return int(not tasks)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04001953 except Failure:
1954 on_error.report(None)
vadimsh@chromium.orgd908a542013-10-30 01:36:17 +00001955 return 1
maruel@chromium.org0437a732013-08-27 16:05:52 +00001956
1957
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001958class OptionParserSwarming(logging_utils.OptionParserWithLogging):
Junji Watanabe38b28b02020-04-23 10:23:30 +00001959
maruel@chromium.org0437a732013-08-27 16:05:52 +00001960 def __init__(self, **kwargs):
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001961 logging_utils.OptionParserWithLogging.__init__(
maruel@chromium.org0437a732013-08-27 16:05:52 +00001962 self, prog='swarming.py', **kwargs)
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001963 self.server_group = optparse.OptionGroup(self, 'Server')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001964 self.server_group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001965 '-S',
1966 '--swarming',
1967 metavar='URL',
1968 default=os.environ.get('SWARMING_SERVER', ''),
maruel@chromium.orge9403ab2013-09-20 18:03:49 +00001969 help='Swarming server to use')
Marc-Antoine Ruel5471e3d2013-11-11 19:10:32 -05001970 self.add_option_group(self.server_group)
Vadim Shtayurae34e13a2014-02-02 11:23:26 -08001971 auth.add_auth_options(self)
maruel@chromium.org0437a732013-08-27 16:05:52 +00001972
1973 def parse_args(self, *args, **kwargs):
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001974 options, args = logging_utils.OptionParserWithLogging.parse_args(
maruel@chromium.org0437a732013-08-27 16:05:52 +00001975 self, *args, **kwargs)
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001976 auth.process_auth_options(self, options)
1977 user = self._process_swarming(options)
1978 if hasattr(options, 'user') and not options.user:
1979 options.user = user
1980 return options, args
1981
1982 def _process_swarming(self, options):
1983 """Processes the --swarming option and aborts if not specified.
1984
1985 Returns the identity as determined by the server.
1986 """
maruel@chromium.org0437a732013-08-27 16:05:52 +00001987 if not options.swarming:
1988 self.error('--swarming is required.')
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001989 try:
1990 options.swarming = net.fix_url(options.swarming)
1991 except ValueError as e:
1992 self.error('--swarming %s' % e)
Takuto Ikutaae767b32020-05-11 01:22:19 +00001993
Marc-Antoine Ruelf7d737d2014-12-10 15:36:29 -05001994 try:
1995 user = auth.ensure_logged_in(options.swarming)
1996 except ValueError as e:
1997 self.error(str(e))
Marc-Antoine Ruel012067b2014-12-10 15:45:42 -05001998 return user
maruel@chromium.org0437a732013-08-27 16:05:52 +00001999
2000
2001def main(args):
2002 dispatcher = subcommand.CommandDispatcher(__name__)
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -04002003 return dispatcher.execute(OptionParserSwarming(version=__version__), args)
maruel@chromium.org0437a732013-08-27 16:05:52 +00002004
2005
2006if __name__ == '__main__':
maruel8e4e40c2016-05-30 06:21:07 -07002007 subprocess42.inhibit_os_error_reporting()
maruel@chromium.org0437a732013-08-27 16:05:52 +00002008 fix_encoding.fix_encoding()
2009 tools.disable_buffering()
2010 colorama.init()
Takuto Ikuta7c843c82020-04-15 05:42:54 +00002011 net.set_user_agent('swarming.py/' + __version__)
maruel@chromium.org0437a732013-08-27 16:05:52 +00002012 sys.exit(main(sys.argv[1:]))