blob: 3af9105e06c46991bf01e06fde26aa04899d0a0b [file] [log] [blame]
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001#!/usr/bin/env python
maruelea586f32016-04-05 11:11:33 -07002# Copyright 2012 The LUCI Authors. All rights reserved.
maruelf1f5e2a2016-05-25 17:10:39 -07003# Use of this source code is governed under the Apache License, Version 2.0
4# that can be found in the LICENSE file.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00005
nodir55be77b2016-05-03 09:39:57 -07006"""Runs a command with optional isolated input/output.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00007
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +00008run_isolated takes cares of setting up a temporary environment, running a
9command, and tearing it down.
nodir55be77b2016-05-03 09:39:57 -070010
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +000011It handles downloading and uploading isolated files, mapping CIPD packages and
12reusing stateful named caches.
13
14The isolated files, CIPD packages and named caches are kept as a global LRU
15cache.
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -050016
Roberto Carrillo71ade6d2018-10-08 22:30:24 +000017Any ${EXECUTABLE_SUFFIX} on the command line or the environment variables passed
18with the --env option will be replaced with ".exe" string on Windows and "" on
19other platforms.
nodirbe642ff2016-06-09 15:51:51 -070020
Roberto Carrillo71ade6d2018-10-08 22:30:24 +000021Any ${ISOLATED_OUTDIR} on the command line or the environment variables passed
22with the --env option will be replaced by the location of a temporary directory
23upon execution of the command specified in the .isolated file. All content
24written to this directory will be uploaded upon termination and the .isolated
25file describing this directory will be printed to stdout.
bpastene447c1992016-06-20 15:21:47 -070026
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +000027Any ${SWARMING_BOT_FILE} on the command line or the environment variables passed
28with the --env option will be replaced by the value of the --bot-file parameter.
29This file is used by a swarming bot to communicate state of the host to tasks.
30It is written to by the swarming bot's on_before_task() hook in the swarming
31server's custom bot_config.py.
32
33See
34https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Magic-Values.md
35for all the variables.
36
37See
38https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/swarming_bot/config/bot_config.py
39for more information about bot_config.py.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000040"""
41
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +000042from __future__ import print_function
43
44__version__ = '1.0.1'
maruel@chromium.orgdedbf492013-09-12 20:42:11 +000045
aludwin7556e0c2016-10-26 08:46:10 -070046import argparse
maruel064c0a32016-04-05 11:47:15 -070047import base64
iannucci96fcccc2016-08-30 15:52:22 -070048import collections
vadimsh232f5a82017-01-20 19:23:44 -080049import contextlib
Sadaf Matinkhoo10743a62018-03-29 16:28:58 -040050import errno
aludwin7556e0c2016-10-26 08:46:10 -070051import json
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000052import logging
53import optparse
54import os
Takuto Ikuta5c59a842020-01-24 03:05:24 +000055import platform
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -040056import re
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000057import sys
58import tempfile
maruel064c0a32016-04-05 11:47:15 -070059import time
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000060
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000061from utils import tools
62tools.force_local_third_party()
vadimsh@chromium.orga4326472013-08-24 02:05:41 +000063
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000064# third_party/
65from depot_tools import fix_encoding
Takuto Ikuta6e2ff962019-10-29 12:35:27 +000066import six
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000067
68# pylint: disable=ungrouped-imports
69import auth
70import cipd
71import isolate_storage
72import isolateserver
73import local_caching
74from libs import luci_context
Vadim Shtayura6b555c12014-07-23 16:22:18 -070075from utils import file_path
maruel12e30012015-10-09 11:55:35 -070076from utils import fs
maruel064c0a32016-04-05 11:47:15 -070077from utils import large
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -040078from utils import logging_utils
Ye Kuang2dd17442020-04-22 08:45:52 +000079from utils import net
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -040080from utils import on_error
Marc-Antoine Ruelc44f5722015-01-08 16:10:01 -050081from utils import subprocess42
maruel@chromium.orgdedbf492013-09-12 20:42:11 +000082
vadimsh@chromium.orga4326472013-08-24 02:05:41 +000083
maruele2f2cb82016-07-13 14:41:03 -070084# Magic variables that can be found in the isolate task command line.
85ISOLATED_OUTDIR_PARAMETER = '${ISOLATED_OUTDIR}'
86EXECUTABLE_SUFFIX_PARAMETER = '${EXECUTABLE_SUFFIX}'
87SWARMING_BOT_FILE_PARAMETER = '${SWARMING_BOT_FILE}'
88
89
csharp@chromium.orgff2a4662012-11-21 20:49:32 +000090# The name of the log file to use.
91RUN_ISOLATED_LOG_FILE = 'run_isolated.log'
92
maruele2f2cb82016-07-13 14:41:03 -070093
csharp@chromium.orge217f302012-11-22 16:51:53 +000094# The name of the log to use for the run_test_cases.py command
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000095RUN_TEST_CASES_LOG = 'run_test_cases.log'
csharp@chromium.orge217f302012-11-22 16:51:53 +000096
vadimsh@chromium.org87d63262013-04-04 19:34:21 +000097
maruele2f2cb82016-07-13 14:41:03 -070098# Use short names for temporary directories. This is driven by Windows, which
99# imposes a relatively short maximum path length of 260 characters, often
100# referred to as MAX_PATH. It is relatively easy to create files with longer
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +0000101# path length. A use case is with recursive dependency trees like npm packages.
maruele2f2cb82016-07-13 14:41:03 -0700102#
103# It is recommended to start the script with a `root_dir` as short as
104# possible.
105# - ir stands for isolated_run
106# - io stands for isolated_out
107# - it stands for isolated_tmp
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000108# - ic stands for isolated_client
maruele2f2cb82016-07-13 14:41:03 -0700109ISOLATED_RUN_DIR = u'ir'
110ISOLATED_OUT_DIR = u'io'
111ISOLATED_TMP_DIR = u'it'
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000112ISOLATED_CLIENT_DIR = u'ic'
maruele2f2cb82016-07-13 14:41:03 -0700113
Takuto Ikuta02edca22019-11-29 10:04:51 +0000114# TODO(tikuta): take these parameter from luci-config?
Takuto Ikutac8c92e62020-04-01 07:07:29 +0000115# Update tag by `./client/update_isolated.sh`.
116# Or take revision from
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000117# https://ci.chromium.org/p/infra-internal/g/infra-packagers/console
Takuto Ikuta02edca22019-11-29 10:04:51 +0000118ISOLATED_PACKAGE = 'infra/tools/luci/isolated/${platform}'
Ye Kuang9123c9d2020-05-08 13:08:56 +0000119ISOLATED_REVISION = 'git_revision:f097334d4f2503ba7693d2cf068c958895282993'
maruele2f2cb82016-07-13 14:41:03 -0700120
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -0400121# Keep synced with task_request.py
Lei Leife202df2019-06-11 17:33:34 +0000122CACHE_NAME_RE = re.compile(r'^[a-z0-9_]{1,4096}$')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -0400123
124
marueld928c862017-06-08 08:20:04 -0700125OUTLIVING_ZOMBIE_MSG = """\
126*** Swarming tried multiple times to delete the %s directory and failed ***
127*** Hard failing the task ***
128
129Swarming detected that your testing script ran an executable, which may have
130started a child executable, and the main script returned early, leaving the
131children executables playing around unguided.
132
133You don't want to leave children processes outliving the task on the Swarming
134bot, do you? The Swarming bot doesn't.
135
136How to fix?
137- For any process that starts children processes, make sure all children
138 processes terminated properly before each parent process exits. This is
139 especially important in very deep process trees.
140 - This must be done properly both in normal successful task and in case of
141 task failure. Cleanup is very important.
142- The Swarming bot sends a SIGTERM in case of timeout.
143 - You have %s seconds to comply after the signal was sent to the process
144 before the process is forcibly killed.
145- To achieve not leaking children processes in case of signals on timeout, you
146 MUST handle signals in each executable / python script and propagate them to
147 children processes.
148 - When your test script (python or binary) receives a signal like SIGTERM or
149 CTRL_BREAK_EVENT on Windows), send it to all children processes and wait for
150 them to terminate before quitting.
151
152See
Marc-Antoine Ruelc7243592018-05-24 17:04:04 -0400153https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Bot.md#Graceful-termination_aka-the-SIGTERM-and-SIGKILL-dance
marueld928c862017-06-08 08:20:04 -0700154for more information.
155
156*** May the SIGKILL force be with you ***
157"""
158
159
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000160# Currently hardcoded. Eventually could be exposed as a flag once there's value.
161# 3 weeks
162MAX_AGE_SECS = 21*24*60*60
163
164
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500165TaskData = collections.namedtuple(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000166 'TaskData',
167 [
Takuto Ikuta9a319502019-11-26 07:40:14 +0000168 # List of strings; the command line to use, independent of what was
169 # specified in the isolated file.
170 'command',
171 # Relative directory to start command into.
172 'relative_cwd',
173 # List of strings; the arguments to add to the command specified in the
174 # isolated file.
175 'extra_args',
176 # Hash of the .isolated file that must be retrieved to recreate the tree
177 # of files to run the target executable. The command specified in the
178 # .isolated is executed. Mutually exclusive with command argument.
179 'isolated_hash',
180 # isolateserver.Storage instance to retrieve remote objects. This object
181 # has a reference to an isolateserver.StorageApi, which does the actual
182 # I/O.
183 'storage',
184 # isolateserver.LocalCache instance to keep from retrieving the same
185 # objects constantly by caching the objects retrieved. Can be on-disk or
186 # in-memory.
187 'isolate_cache',
188 # List of paths relative to root_dir to put into the output isolated
189 # bundle upon task completion (see link_outputs_to_outdir).
190 'outputs',
191 # Function (run_dir) => context manager that installs named caches into
192 # |run_dir|.
193 'install_named_caches',
194 # If True, the temporary directory will be deliberately leaked for later
195 # examination.
196 'leak_temp_dir',
197 # Path to the directory to use to create the temporary directory. If not
198 # specified, a random temporary directory is created.
199 'root_dir',
200 # Kills the process if it lasts more than this amount of seconds.
201 'hard_timeout',
202 # Number of seconds to wait between SIGTERM and SIGKILL.
203 'grace_period',
204 # Path to a file with bot state, used in place of ${SWARMING_BOT_FILE}
205 # task command line argument.
206 'bot_file',
207 # Logical account to switch LUCI_CONTEXT into.
208 'switch_to_account',
209 # Context manager dir => CipdInfo, see install_client_and_packages.
210 'install_packages_fn',
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000211 # Use go isolated client.
212 'use_go_isolated',
Takuto Ikuta057c5342019-12-03 04:05:05 +0000213 # Cache directory for go isolated client.
214 'go_cache_dir',
Takuto Ikuta879788c2020-01-10 08:00:26 +0000215 # Parameters passed to go isolated client.
216 'go_cache_policies',
Takuto Ikuta9a319502019-11-26 07:40:14 +0000217 # Environment variables to set.
218 'env',
219 # Environment variables to mutate with relative directories.
220 # Example: {"ENV_KEY": ['relative', 'paths', 'to', 'prepend']}
221 'env_prefix',
222 # Lowers the task process priority.
223 'lower_priority',
224 # subprocess42.Containment instance. Can be None.
225 'containment',
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000226 ])
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500227
228
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500229def _to_str(s):
230 """Downgrades a unicode instance to str. Pass str through as-is."""
231 if isinstance(s, str):
232 return s
233 # This is technically incorrect, especially on Windows. In theory
234 # sys.getfilesystemencoding() should be used to use the right 'ANSI code
235 # page' on Windows, but that causes other problems, as the character set
236 # is very limited.
237 return s.encode('utf-8')
238
239
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -0500240def _to_unicode(s):
241 """Upgrades a str instance to unicode. Pass unicode through as-is."""
Takuto Ikuta95459dd2019-10-29 12:39:47 +0000242 if isinstance(s, six.text_type) or s is None:
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -0500243 return s
244 return s.decode('utf-8')
245
246
maruel03e11842016-07-14 10:50:16 -0700247def make_temp_dir(prefix, root_dir):
248 """Returns a new unique temporary directory."""
Takuto Ikuta6e2ff962019-10-29 12:35:27 +0000249 return six.text_type(tempfile.mkdtemp(prefix=prefix, dir=root_dir))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000250
251
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500252def change_tree_read_only(rootdir, read_only):
253 """Changes the tree read-only bits according to the read_only specification.
254
255 The flag can be 0, 1 or 2, which will affect the possibility to modify files
256 and create or delete files.
257 """
258 if read_only == 2:
259 # Files and directories (except on Windows) are marked read only. This
260 # inhibits modifying, creating or deleting files in the test directory,
261 # except on Windows where creating and deleting files is still possible.
Marc-Antoine Ruele4ad07e2014-10-15 20:22:29 -0400262 file_path.make_tree_read_only(rootdir)
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500263 elif read_only == 1:
264 # Files are marked read only but not the directories. This inhibits
265 # modifying files but creating or deleting files is still possible.
Marc-Antoine Ruele4ad07e2014-10-15 20:22:29 -0400266 file_path.make_tree_files_read_only(rootdir)
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500267 elif read_only in (0, None):
Marc-Antoine Ruelf1d827c2014-11-24 15:22:25 -0500268 # Anything can be modified.
Marc-Antoine Ruel2666d9c2018-05-18 13:52:02 -0400269 # TODO(maruel): This is currently dangerous as long as
270 # DiskContentAddressedCache.touch() is not yet changed to verify the hash of
271 # the content of the files it is looking at, so that if a test modifies an
272 # input file, the file must be deleted.
Marc-Antoine Ruele4ad07e2014-10-15 20:22:29 -0400273 file_path.make_tree_writeable(rootdir)
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500274 else:
275 raise ValueError(
276 'change_tree_read_only(%s, %s): Unknown flag %s' %
277 (rootdir, read_only, read_only))
278
279
vadimsh9c54b2c2017-07-25 14:08:29 -0700280@contextlib.contextmanager
281def set_luci_context_account(account, tmp_dir):
282 """Sets LUCI_CONTEXT account to be used by the task.
283
284 If 'account' is None or '', does nothing at all. This happens when
285 run_isolated.py is called without '--switch-to-account' flag. In this case,
286 if run_isolated.py is running in some LUCI_CONTEXT environment, the task will
Takuto Ikuta33e2ff32019-09-30 12:44:03 +0000287 just inherit whatever account is already set. This may happen if users invoke
vadimsh9c54b2c2017-07-25 14:08:29 -0700288 run_isolated.py explicitly from their code.
289
290 If the requested account is not defined in the context, switches to
291 non-authenticated access. This happens for Swarming tasks that don't use
292 'task' service accounts.
293
294 If not using LUCI_CONTEXT-based auth, does nothing.
295 If already running as requested account, does nothing.
296 """
297 if not account:
298 # Not actually switching.
299 yield
300 return
301
302 local_auth = luci_context.read('local_auth')
303 if not local_auth:
304 # Not using LUCI_CONTEXT auth at all.
305 yield
306 return
307
308 # See LUCI_CONTEXT.md for the format of 'local_auth'.
309 if local_auth.get('default_account_id') == account:
310 # Already set, no need to switch.
311 yield
312 return
313
314 available = {a['id'] for a in local_auth.get('accounts') or []}
315 if account in available:
316 logging.info('Switching default LUCI_CONTEXT account to %r', account)
317 local_auth['default_account_id'] = account
318 else:
319 logging.warning(
320 'Requested LUCI_CONTEXT account %r is not available (have only %r), '
321 'disabling authentication', account, sorted(available))
322 local_auth.pop('default_account_id', None)
323
324 with luci_context.write(_tmpdir=tmp_dir, local_auth=local_auth):
325 yield
326
327
nodir90bc8dc2016-06-15 13:35:21 -0700328def process_command(command, out_dir, bot_file):
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000329 """Replaces parameters in a command line.
nodirbe642ff2016-06-09 15:51:51 -0700330
331 Raises:
332 ValueError if a parameter is requested in |command| but its value is not
333 provided.
334 """
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000335 return [replace_parameters(arg, out_dir, bot_file) for arg in command]
336
337
338def replace_parameters(arg, out_dir, bot_file):
339 """Replaces parameter tokens with appropriate values in a string.
340
341 Raises:
342 ValueError if a parameter is requested in |arg| but its value is not
343 provided.
344 """
345 arg = arg.replace(EXECUTABLE_SUFFIX_PARAMETER, cipd.EXECUTABLE_SUFFIX)
346 replace_slash = False
347 if ISOLATED_OUTDIR_PARAMETER in arg:
348 if not out_dir:
349 raise ValueError(
350 'output directory is requested in command or env var, but not '
351 'provided; please specify one')
352 arg = arg.replace(ISOLATED_OUTDIR_PARAMETER, out_dir)
353 replace_slash = True
354 if SWARMING_BOT_FILE_PARAMETER in arg:
355 if bot_file:
356 arg = arg.replace(SWARMING_BOT_FILE_PARAMETER, bot_file)
nodirbe642ff2016-06-09 15:51:51 -0700357 replace_slash = True
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000358 else:
359 logging.warning('SWARMING_BOT_FILE_PARAMETER found in command or env '
360 'var, but no bot_file specified. Leaving parameter '
361 'unchanged.')
362 if replace_slash:
363 # Replace slashes only if parameters are present
364 # because of arguments like '${ISOLATED_OUTDIR}/foo/bar'
365 arg = arg.replace('/', os.sep)
366 return arg
maruela9cfd6f2015-09-15 11:03:15 -0700367
368
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000369
370def get_command_env(tmp_dir, cipd_info, run_dir, env, env_prefixes, out_dir,
371 bot_file):
vadimsh232f5a82017-01-20 19:23:44 -0800372 """Returns full OS environment to run a command in.
373
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800374 Sets up TEMP, puts directory with cipd binary in front of PATH, exposes
375 CIPD_CACHE_DIR env var, and installs all env_prefixes.
vadimsh232f5a82017-01-20 19:23:44 -0800376
377 Args:
378 tmp_dir: temp directory.
379 cipd_info: CipdInfo object is cipd client is used, None if not.
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500380 run_dir: The root directory the isolated tree is mapped in.
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500381 env: environment variables to use
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800382 env_prefixes: {"ENV_KEY": ['cwd', 'relative', 'paths', 'to', 'prepend']}
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000383 out_dir: Isolated output directory. Required to be != None if any of the
384 env vars contain ISOLATED_OUTDIR_PARAMETER.
385 bot_file: Required to be != None if any of the env vars contain
386 SWARMING_BOT_FILE_PARAMETER.
vadimsh232f5a82017-01-20 19:23:44 -0800387 """
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500388 out = os.environ.copy()
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +0000389 for k, v in env.items():
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500390 if not v:
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500391 out.pop(k, None)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500392 else:
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000393 out[k] = replace_parameters(v, out_dir, bot_file)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500394
395 if cipd_info:
396 bin_dir = os.path.dirname(cipd_info.client.binary_path)
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500397 out['PATH'] = '%s%s%s' % (_to_str(bin_dir), os.pathsep, out['PATH'])
398 out['CIPD_CACHE_DIR'] = _to_str(cipd_info.cache_dir)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500399
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +0000400 for key, paths in env_prefixes.items():
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500401 assert isinstance(paths, list), paths
402 paths = [os.path.normpath(os.path.join(run_dir, p)) for p in paths]
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500403 cur = out.get(key)
404 if cur:
405 paths.append(cur)
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500406 out[key] = _to_str(os.path.pathsep.join(paths))
vadimsh232f5a82017-01-20 19:23:44 -0800407
Marc-Antoine Ruelefb30b12018-07-25 18:34:36 +0000408 tmp_dir = _to_str(tmp_dir)
409 # pylint: disable=line-too-long
410 # * python respects $TMPDIR, $TEMP, and $TMP in this order, regardless of
411 # platform. So $TMPDIR must be set on all platforms.
412 # https://github.com/python/cpython/blob/2.7/Lib/tempfile.py#L155
413 out['TMPDIR'] = tmp_dir
414 if sys.platform == 'win32':
415 # * chromium's base utils uses GetTempPath().
416 # https://cs.chromium.org/chromium/src/base/files/file_util_win.cc?q=GetTempPath
417 # * Go uses GetTempPath().
418 # * GetTempDir() uses %TMP%, then %TEMP%, then other stuff. So %TMP% must be
419 # set.
420 # https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-gettemppathw
421 out['TMP'] = tmp_dir
422 # https://blogs.msdn.microsoft.com/oldnewthing/20150417-00/?p=44213
423 out['TEMP'] = tmp_dir
424 elif sys.platform == 'darwin':
425 # * Chromium uses an hack on macOS before calling into
426 # NSTemporaryDirectory().
427 # https://cs.chromium.org/chromium/src/base/files/file_util_mac.mm?q=GetTempDir
428 # https://developer.apple.com/documentation/foundation/1409211-nstemporarydirectory
429 out['MAC_CHROMIUM_TMPDIR'] = tmp_dir
430 else:
431 # TMPDIR is specified as the POSIX standard envvar for the temp directory.
432 # http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html
433 # * mktemp on linux respects $TMPDIR.
434 # * Chromium respects $TMPDIR on linux.
435 # https://cs.chromium.org/chromium/src/base/files/file_util_posix.cc?q=GetTempDir
436 # * Go uses $TMPDIR.
437 # https://go.googlesource.com/go/+/go1.10.3/src/os/file_unix.go#307
438 pass
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500439 return out
vadimsh232f5a82017-01-20 19:23:44 -0800440
441
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000442def run_command(
443 command, cwd, env, hard_timeout, grace_period, lower_priority, containment):
maruel6be7f9e2015-10-01 12:25:30 -0700444 """Runs the command.
445
446 Returns:
447 tuple(process exit code, bool if had a hard timeout)
448 """
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000449 logging.info(
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000450 'run_command(%s, %s, %s, %s, %s, %s)',
451 command, cwd, hard_timeout, grace_period, lower_priority, containment)
marueleb5fbee2015-09-17 13:01:36 -0700452
maruel6be7f9e2015-10-01 12:25:30 -0700453 exit_code = None
454 had_hard_timeout = False
maruela9cfd6f2015-09-15 11:03:15 -0700455 with tools.Profiler('RunTest'):
maruel6be7f9e2015-10-01 12:25:30 -0700456 proc = None
457 had_signal = []
maruela9cfd6f2015-09-15 11:03:15 -0700458 try:
maruel6be7f9e2015-10-01 12:25:30 -0700459 # TODO(maruel): This code is imperfect. It doesn't handle well signals
460 # during the download phase and there's short windows were things can go
461 # wrong.
462 def handler(signum, _frame):
463 if proc and not had_signal:
464 logging.info('Received signal %d', signum)
465 had_signal.append(True)
maruel556d9052015-10-05 11:12:44 -0700466 raise subprocess42.TimeoutExpired(command, None)
maruel6be7f9e2015-10-01 12:25:30 -0700467
Marc-Antoine Ruel30b80fe2019-02-08 13:51:31 +0000468 proc = subprocess42.Popen(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000469 command, cwd=cwd, env=env, detached=True, close_fds=True,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000470 lower_priority=lower_priority, containment=containment)
maruel6be7f9e2015-10-01 12:25:30 -0700471 with subprocess42.set_signal_handler(subprocess42.STOP_SIGNALS, handler):
472 try:
John Budorickc398f092019-06-10 22:49:44 +0000473 exit_code = proc.wait(hard_timeout or None)
maruel6be7f9e2015-10-01 12:25:30 -0700474 except subprocess42.TimeoutExpired:
475 if not had_signal:
476 logging.warning('Hard timeout')
477 had_hard_timeout = True
478 logging.warning('Sending SIGTERM')
479 proc.terminate()
480
481 # Ignore signals in grace period. Forcibly give the grace period to the
482 # child process.
483 if exit_code is None:
484 ignore = lambda *_: None
485 with subprocess42.set_signal_handler(subprocess42.STOP_SIGNALS, ignore):
486 try:
487 exit_code = proc.wait(grace_period or None)
488 except subprocess42.TimeoutExpired:
489 # Now kill for real. The user can distinguish between the
490 # following states:
491 # - signal but process exited within grace period,
492 # hard_timed_out will be set but the process exit code will be
493 # script provided.
494 # - processed exited late, exit code will be -9 on posix.
495 logging.warning('Grace exhausted; sending SIGKILL')
496 proc.kill()
martiniss5c8043e2017-08-01 17:09:43 -0700497 logging.info('Waiting for process exit')
maruel6be7f9e2015-10-01 12:25:30 -0700498 exit_code = proc.wait()
Takuto Ikutaeccf0862020-03-19 03:05:55 +0000499 except OSError as e:
maruela9cfd6f2015-09-15 11:03:15 -0700500 # This is not considered to be an internal error. The executable simply
501 # does not exit.
maruela72f46e2016-02-24 11:05:45 -0800502 sys.stderr.write(
tikuta2d678212019-09-23 23:12:08 +0000503 '<The executable does not exist, a dependent library is missing or '
504 'the command line is too long>\n'
505 '<Check for missing .so/.dll in the .isolate or GN file or length of '
506 'command line args>\n'
Takuto Ikutaeccf0862020-03-19 03:05:55 +0000507 '<Command: %s, Exception: %s>\n' % (command, e))
maruela72f46e2016-02-24 11:05:45 -0800508 if os.environ.get('SWARMING_TASK_ID'):
509 # Give an additional hint when running as a swarming task.
510 sys.stderr.write(
511 '<See the task\'s page for commands to help diagnose this issue '
512 'by reproducing the task locally>\n')
maruela9cfd6f2015-09-15 11:03:15 -0700513 exit_code = 1
514 logging.info(
515 'Command finished with exit code %d (%s)',
516 exit_code, hex(0xffffffff & exit_code))
maruel6be7f9e2015-10-01 12:25:30 -0700517 return exit_code, had_hard_timeout
maruela9cfd6f2015-09-15 11:03:15 -0700518
519
Takuto Ikuta879788c2020-01-10 08:00:26 +0000520def _fetch_and_map_with_go(isolated_hash, storage, outdir, go_cache_dir,
521 policies, isolated_client):
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000522 """
523 Fetches an isolated tree using go client, create the tree and returns
524 (bundle, stats).
525 """
526 start = time.time()
527 server_ref = storage.server_ref
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000528 result_json_handle, result_json_path = tempfile.mkstemp(
529 prefix=u'fetch-and-map-result-', suffix=u'.json')
530 os.close(result_json_handle)
531 try:
Ye Kuanga98764c2020-04-09 03:17:37 +0000532 cmd = [
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000533 isolated_client,
534 'download',
535 '-isolate-server',
536 server_ref.url,
537 '-namespace',
538 server_ref.namespace,
539 '-isolated',
540 isolated_hash,
541
542 # flags for cache
543 '-cache-dir',
Takuto Ikuta057c5342019-12-03 04:05:05 +0000544 go_cache_dir,
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000545 '-cache-max-items',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000546 str(policies.max_items),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000547 '-cache-max-size',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000548 str(policies.max_cache_size),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000549 '-cache-min-free-space',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000550 str(policies.min_free_space),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000551
552 # flags for output
553 '-output-dir',
554 outdir,
555 '-fetch-and-map-result-json',
556 result_json_path,
Ye Kuanga98764c2020-04-09 03:17:37 +0000557 ]
558 proc = subprocess42.Popen(cmd)
559 cmd_str = ' '.join(cmd)
Takuto Ikuta3153e3b2020-02-18 06:11:47 +0000560
Ye Kuanga98764c2020-04-09 03:17:37 +0000561 exceeded_max_timeout = True
562 check_period_sec = 30
563 max_checks = 100
564 # max timeout = max_checks * check_period_sec = 50 minutes
565 for i in range(max_checks):
Takuto Ikuta3153e3b2020-02-18 06:11:47 +0000566 # This is to prevent I/O timeout error during isolated setup.
567 try:
Ye Kuanga98764c2020-04-09 03:17:37 +0000568 retcode = proc.wait(check_period_sec)
Takuto Ikutab64c5192020-02-19 02:46:34 +0000569 if retcode != 0:
Ye Kuanga98764c2020-04-09 03:17:37 +0000570 raise ValueError("retcode is not 0: %s (cmd=%s)" % (retcode, cmd_str))
571 exceeded_max_timeout = False
Takuto Ikuta3153e3b2020-02-18 06:11:47 +0000572 break
573 except subprocess42.TimeoutExpired:
Ye Kuanga98764c2020-04-09 03:17:37 +0000574 print('still running isolated (after %d seconds)' % (
575 (i + 1) * check_period_sec))
576
577 if exceeded_max_timeout:
578 proc.terminate()
579 try:
580 proc.wait(check_period_sec)
581 except subprocess42.TimeoutExpired:
Takuto Ikutac20d2b32020-04-28 06:05:14 +0000582 logging.exception(
583 "failed to terminate? timeout happened after %d seconds",
584 check_period_sec)
Ye Kuanga98764c2020-04-09 03:17:37 +0000585 proc.kill()
586 proc.wait()
587 # Raise unconditionally, because |proc| was forcefully terminated.
Takuto Ikuta09ed9b22020-04-24 00:52:50 +0000588 raise ValueError("timedout after %d seconds (cmd=%s)" %
Ye Kuanga98764c2020-04-09 03:17:37 +0000589 (check_period_sec * max_checks, cmd_str))
Takuto Ikuta3153e3b2020-02-18 06:11:47 +0000590
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000591 with open(result_json_path) as json_file:
592 result_json = json.load(json_file)
593
594 isolated = result_json['isolated']
595 bundle = isolateserver.IsolatedBundle(filter_cb=None)
596 # Only following properties are used in caller.
597 bundle.command = isolated.get('command')
598 bundle.read_only = isolated.get('read_only')
599 bundle.relative_cwd = isolated.get('relative_cwd')
600
601 return bundle, {
602 'duration': time.time() - start,
603 'items_cold': result_json['items_cold'],
604 'items_hot': result_json['items_hot'],
605 }
606 finally:
607 fs.remove(result_json_path)
608
609
610# TODO(crbug.com/932396): remove this function.
Takuto Ikuta16fac4b2019-12-09 04:57:18 +0000611def fetch_and_map(isolated_hash, storage, cache, outdir):
maruel4409e302016-07-19 14:25:51 -0700612 """Fetches an isolated tree, create the tree and returns (bundle, stats)."""
nodir6f801882016-04-29 14:41:50 -0700613 start = time.time()
614 bundle = isolateserver.fetch_isolated(
615 isolated_hash=isolated_hash,
616 storage=storage,
617 cache=cache,
maruel4409e302016-07-19 14:25:51 -0700618 outdir=outdir,
Takuto Ikuta16fac4b2019-12-09 04:57:18 +0000619 use_symlinks=False)
Takuto Ikuta2b9640e2019-06-19 00:53:23 +0000620 hot = (collections.Counter(cache.used) -
621 collections.Counter(cache.added)).elements()
nodir6f801882016-04-29 14:41:50 -0700622 return bundle, {
623 'duration': time.time() - start,
nodir6f801882016-04-29 14:41:50 -0700624 'items_cold': base64.b64encode(large.pack(sorted(cache.added))),
Takuto Ikuta2b9640e2019-06-19 00:53:23 +0000625 'items_hot': base64.b64encode(large.pack(sorted(hot))),
nodir6f801882016-04-29 14:41:50 -0700626 }
627
628
aludwin0a8e17d2016-10-27 15:57:39 -0700629def link_outputs_to_outdir(run_dir, out_dir, outputs):
630 """Links any named outputs to out_dir so they can be uploaded.
631
632 Raises an error if the file already exists in that directory.
633 """
634 if not outputs:
635 return
636 isolateserver.create_directories(out_dir, outputs)
637 for o in outputs:
Sadaf Matinkhoo10743a62018-03-29 16:28:58 -0400638 copy_recursively(os.path.join(run_dir, o), os.path.join(out_dir, o))
639
640
641def copy_recursively(src, dst):
642 """Efficiently copies a file or directory from src_dir to dst_dir.
643
644 `item` may be a file, directory, or a symlink to a file or directory.
645 All symlinks are replaced with their targets, so the resulting
646 directory structure in dst_dir will never have any symlinks.
647
648 To increase speed, copy_recursively hardlinks individual files into the
649 (newly created) directory structure if possible, unlike Python's
650 shutil.copytree().
651 """
652 orig_src = src
653 try:
654 # Replace symlinks with their final target.
655 while fs.islink(src):
656 res = fs.readlink(src)
657 src = os.path.join(os.path.dirname(src), res)
658 # TODO(sadafm): Explicitly handle cyclic symlinks.
659
660 # Note that fs.isfile (which is a wrapper around os.path.isfile) throws
661 # an exception if src does not exist. A warning will be logged in that case.
662 if fs.isfile(src):
663 file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK)
664 return
665
666 if not fs.exists(dst):
667 os.makedirs(dst)
668
669 for child in fs.listdir(src):
670 copy_recursively(os.path.join(src, child), os.path.join(dst, child))
671
672 except OSError as e:
673 if e.errno == errno.ENOENT:
674 logging.warning('Path %s does not exist or %s is a broken symlink',
675 src, orig_src)
676 else:
677 logging.info("Couldn't collect output file %s: %s", src, e)
aludwin0a8e17d2016-10-27 15:57:39 -0700678
679
Takuto Ikutab4aa8662019-09-17 05:54:36 +0000680def upload_then_delete(storage, out_dir, leak_temp_dir):
maruela9cfd6f2015-09-15 11:03:15 -0700681 """Deletes the temporary run directory and uploads results back.
682
683 Returns:
nodir6f801882016-04-29 14:41:50 -0700684 tuple(outputs_ref, success, stats)
maruel064c0a32016-04-05 11:47:15 -0700685 - outputs_ref: a dict referring to the results archived back to the isolated
686 server, if applicable.
687 - success: False if something occurred that means that the task must
688 forcibly be considered a failure, e.g. zombie processes were left
689 behind.
nodir6f801882016-04-29 14:41:50 -0700690 - stats: uploading stats.
maruela9cfd6f2015-09-15 11:03:15 -0700691 """
maruela9cfd6f2015-09-15 11:03:15 -0700692 # Upload out_dir and generate a .isolated file out of this directory. It is
693 # only done if files were written in the directory.
694 outputs_ref = None
maruel064c0a32016-04-05 11:47:15 -0700695 cold = []
696 hot = []
nodir6f801882016-04-29 14:41:50 -0700697 start = time.time()
698
maruel12e30012015-10-09 11:55:35 -0700699 if fs.isdir(out_dir) and fs.listdir(out_dir):
maruela9cfd6f2015-09-15 11:03:15 -0700700 with tools.Profiler('ArchiveOutput'):
701 try:
maruel064c0a32016-04-05 11:47:15 -0700702 results, f_cold, f_hot = isolateserver.archive_files_to_storage(
Takuto Ikuta26980872020-04-09 06:56:37 +0000703 storage, [out_dir], None, verify_push=True)
maruela9cfd6f2015-09-15 11:03:15 -0700704 outputs_ref = {
Marc-Antoine Rueld0868ec2018-11-28 20:47:29 +0000705 'isolated': results.values()[0],
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +0000706 'isolatedserver': storage.server_ref.url,
707 'namespace': storage.server_ref.namespace,
maruela9cfd6f2015-09-15 11:03:15 -0700708 }
maruel064c0a32016-04-05 11:47:15 -0700709 cold = sorted(i.size for i in f_cold)
710 hot = sorted(i.size for i in f_hot)
maruela9cfd6f2015-09-15 11:03:15 -0700711 except isolateserver.Aborted:
712 # This happens when a signal SIGTERM was received while uploading data.
713 # There is 2 causes:
714 # - The task was too slow and was about to be killed anyway due to
715 # exceeding the hard timeout.
716 # - The amount of data uploaded back is very large and took too much
717 # time to archive.
718 sys.stderr.write('Received SIGTERM while uploading')
719 # Re-raise, so it will be treated as an internal failure.
720 raise
nodir6f801882016-04-29 14:41:50 -0700721
722 success = False
maruela9cfd6f2015-09-15 11:03:15 -0700723 try:
maruel12e30012015-10-09 11:55:35 -0700724 if (not leak_temp_dir and fs.isdir(out_dir) and
maruel6eeea7d2015-09-16 12:17:42 -0700725 not file_path.rmtree(out_dir)):
maruela9cfd6f2015-09-15 11:03:15 -0700726 logging.error('Had difficulties removing out_dir %s', out_dir)
nodir6f801882016-04-29 14:41:50 -0700727 else:
728 success = True
maruela9cfd6f2015-09-15 11:03:15 -0700729 except OSError as e:
730 # When this happens, it means there's a process error.
maruel12e30012015-10-09 11:55:35 -0700731 logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e)
nodir6f801882016-04-29 14:41:50 -0700732 stats = {
733 'duration': time.time() - start,
734 'items_cold': base64.b64encode(large.pack(cold)),
735 'items_hot': base64.b64encode(large.pack(hot)),
736 }
737 return outputs_ref, success, stats
maruela9cfd6f2015-09-15 11:03:15 -0700738
739
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500740def map_and_run(data, constant_run_path):
nodir55be77b2016-05-03 09:39:57 -0700741 """Runs a command with optional isolated input/output.
742
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500743 Arguments:
744 - data: TaskData instance.
745 - constant_run_path: TODO
nodir55be77b2016-05-03 09:39:57 -0700746
747 Returns metadata about the result.
748 """
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +0000749
750 if data.isolate_cache:
751 download_stats = {
752 #'duration': 0.,
753 'initial_number_items': len(data.isolate_cache),
754 'initial_size': data.isolate_cache.total_size,
755 #'items_cold': '<large.pack()>',
756 #'items_hot': '<large.pack()>',
757 }
758 else:
759 # TODO(tikuta): take stats from state.json in this case too.
760 download_stats = {}
761
maruela9cfd6f2015-09-15 11:03:15 -0700762 result = {
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000763 'duration': None,
764 'exit_code': None,
765 'had_hard_timeout': False,
766 'internal_failure': 'run_isolated did not complete properly',
767 'stats': {
768 #'cipd': {
769 # 'duration': 0.,
770 # 'get_client_duration': 0.,
771 #},
772 'isolated': {
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +0000773 'download': download_stats,
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000774 #'upload': {
775 # 'duration': 0.,
776 # 'items_cold': '<large.pack()>',
777 # 'items_hot': '<large.pack()>',
778 #},
779 },
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000780 },
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000781 #'cipd_pins': {
782 # 'packages': [
783 # {'package_name': ..., 'version': ..., 'path': ...},
784 # ...
785 # ],
786 # 'client_package': {'package_name': ..., 'version': ...},
787 #},
788 'outputs_ref': None,
789 'version': 5,
maruela9cfd6f2015-09-15 11:03:15 -0700790 }
nodirbe642ff2016-06-09 15:51:51 -0700791
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500792 if data.root_dir:
Lei Leife202df2019-06-11 17:33:34 +0000793 file_path.ensure_tree(data.root_dir, 0o700)
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +0000794 elif data.use_go_isolated:
795 data = data._replace(root_dir=os.path.dirname(data.go_cache_dir))
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500796 elif data.isolate_cache.cache_dir:
797 data = data._replace(
798 root_dir=os.path.dirname(data.isolate_cache.cache_dir))
maruele2f2cb82016-07-13 14:41:03 -0700799 # See comment for these constants.
maruelcffa0542017-04-07 08:39:20 -0700800 # If root_dir is not specified, it is not constant.
801 # TODO(maruel): This is not obvious. Change this to become an error once we
802 # make the constant_run_path an exposed flag.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500803 if constant_run_path and data.root_dir:
804 run_dir = os.path.join(data.root_dir, ISOLATED_RUN_DIR)
maruel5c4eed82017-05-26 05:33:40 -0700805 if os.path.isdir(run_dir):
806 file_path.rmtree(run_dir)
Lei Leife202df2019-06-11 17:33:34 +0000807 os.mkdir(run_dir, 0o700)
maruelcffa0542017-04-07 08:39:20 -0700808 else:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500809 run_dir = make_temp_dir(ISOLATED_RUN_DIR, data.root_dir)
maruel03e11842016-07-14 10:50:16 -0700810 # storage should be normally set but don't crash if it is not. This can happen
811 # as Swarming task can run without an isolate server.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500812 out_dir = make_temp_dir(
813 ISOLATED_OUT_DIR, data.root_dir) if data.storage else None
814 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, data.root_dir)
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000815 isolated_client_dir = make_temp_dir(ISOLATED_CLIENT_DIR, data.root_dir)
nodir55be77b2016-05-03 09:39:57 -0700816 cwd = run_dir
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -0500817 if data.relative_cwd:
818 cwd = os.path.normpath(os.path.join(cwd, data.relative_cwd))
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500819 command = data.command
nodir55be77b2016-05-03 09:39:57 -0700820 try:
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000821 with data.install_packages_fn(run_dir, isolated_client_dir) as cipd_info:
vadimsh232f5a82017-01-20 19:23:44 -0800822 if cipd_info:
823 result['stats']['cipd'] = cipd_info.stats
824 result['cipd_pins'] = cipd_info.pins
nodir90bc8dc2016-06-15 13:35:21 -0700825
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500826 if data.isolated_hash:
vadimsh232f5a82017-01-20 19:23:44 -0800827 isolated_stats = result['stats'].setdefault('isolated', {})
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000828 if data.use_go_isolated:
Takuto Ikuta90397ca2020-01-08 10:07:55 +0000829 bundle, stats = _fetch_and_map_with_go(
830 isolated_hash=data.isolated_hash,
831 storage=data.storage,
Takuto Ikuta90397ca2020-01-08 10:07:55 +0000832 outdir=run_dir,
833 go_cache_dir=data.go_cache_dir,
Takuto Ikuta879788c2020-01-10 08:00:26 +0000834 policies=data.go_cache_policies,
Takuto Ikuta90397ca2020-01-08 10:07:55 +0000835 isolated_client=os.path.join(isolated_client_dir,
836 'isolated' + cipd.EXECUTABLE_SUFFIX))
837 else:
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000838 bundle, stats = fetch_and_map(
839 isolated_hash=data.isolated_hash,
840 storage=data.storage,
841 cache=data.isolate_cache,
Takuto Ikuta16fac4b2019-12-09 04:57:18 +0000842 outdir=run_dir)
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000843 isolated_stats['download'].update(stats)
vadimsh232f5a82017-01-20 19:23:44 -0800844 change_tree_read_only(run_dir, bundle.read_only)
maruelabec63c2017-04-26 11:53:24 -0700845 # Inject the command
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500846 if not command and bundle.command:
847 command = bundle.command + data.extra_args
Marc-Antoine Rueld704a1f2017-10-31 10:51:23 -0400848 # Only set the relative directory if the isolated file specified a
849 # command, and no raw command was specified.
850 if bundle.relative_cwd:
851 cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
maruelabec63c2017-04-26 11:53:24 -0700852
853 if not command:
854 # Handle this as a task failure, not an internal failure.
855 sys.stderr.write(
856 '<No command was specified!>\n'
857 '<Please secify a command when triggering your Swarming task>\n')
858 result['exit_code'] = 1
859 return result
nodirbe642ff2016-06-09 15:51:51 -0700860
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -0500861 if not cwd.startswith(run_dir):
862 # Handle this as a task failure, not an internal failure. This is a
863 # 'last chance' way to gate against directory escape.
864 sys.stderr.write('<Relative CWD is outside of run directory!>\n')
865 result['exit_code'] = 1
866 return result
867
868 if not os.path.isdir(cwd):
869 # Accepts relative_cwd that does not exist.
Lei Leife202df2019-06-11 17:33:34 +0000870 os.makedirs(cwd, 0o700)
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -0500871
vadimsh232f5a82017-01-20 19:23:44 -0800872 # If we have an explicit list of files to return, make sure their
873 # directories exist now.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500874 if data.storage and data.outputs:
875 isolateserver.create_directories(run_dir, data.outputs)
aludwin0a8e17d2016-10-27 15:57:39 -0700876
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500877 with data.install_named_caches(run_dir):
nodird6160682017-02-02 13:03:35 -0800878 sys.stdout.flush()
879 start = time.time()
880 try:
vadimsh9c54b2c2017-07-25 14:08:29 -0700881 # Need to switch the default account before 'get_command_env' call,
882 # so it can grab correct value of LUCI_CONTEXT env var.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500883 with set_luci_context_account(data.switch_to_account, tmp_dir):
884 env = get_command_env(
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000885 tmp_dir, cipd_info, run_dir, data.env, data.env_prefix, out_dir,
886 data.bot_file)
Brian Sheedy7a761172019-08-30 22:55:14 +0000887 command = tools.find_executable(command, env)
Robert Iannucci24ae76a2018-02-26 12:51:18 -0800888 command = process_command(command, out_dir, data.bot_file)
889 file_path.ensure_command_has_abs_path(command, cwd)
890
vadimsh9c54b2c2017-07-25 14:08:29 -0700891 result['exit_code'], result['had_hard_timeout'] = run_command(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000892 command, cwd, env, data.hard_timeout, data.grace_period,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000893 data.lower_priority, data.containment)
nodird6160682017-02-02 13:03:35 -0800894 finally:
895 result['duration'] = max(time.time() - start, 0)
Seth Koehler49139812017-12-19 13:59:33 -0500896
897 # We successfully ran the command, set internal_failure back to
898 # None (even if the command failed, it's not an internal error).
899 result['internal_failure'] = None
maruela9cfd6f2015-09-15 11:03:15 -0700900 except Exception as e:
nodir90bc8dc2016-06-15 13:35:21 -0700901 # An internal error occurred. Report accordingly so the swarming task will
902 # be retried automatically.
maruel12e30012015-10-09 11:55:35 -0700903 logging.exception('internal failure: %s', e)
maruela9cfd6f2015-09-15 11:03:15 -0700904 result['internal_failure'] = str(e)
905 on_error.report(None)
aludwin0a8e17d2016-10-27 15:57:39 -0700906
907 # Clean up
maruela9cfd6f2015-09-15 11:03:15 -0700908 finally:
909 try:
aludwin0a8e17d2016-10-27 15:57:39 -0700910 # Try to link files to the output directory, if specified.
911 if out_dir:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500912 link_outputs_to_outdir(run_dir, out_dir, data.outputs)
aludwin0a8e17d2016-10-27 15:57:39 -0700913
nodir32a1ec12016-10-26 18:34:07 -0700914 success = False
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500915 if data.leak_temp_dir:
nodir32a1ec12016-10-26 18:34:07 -0700916 success = True
maruela9cfd6f2015-09-15 11:03:15 -0700917 logging.warning(
918 'Deliberately leaking %s for later examination', run_dir)
marueleb5fbee2015-09-17 13:01:36 -0700919 else:
maruel84537cb2015-10-16 14:21:28 -0700920 # On Windows rmtree(run_dir) call above has a synchronization effect: it
921 # finishes only when all task child processes terminate (since a running
922 # process locks *.exe file). Examine out_dir only after that call
923 # completes (since child processes may write to out_dir too and we need
924 # to wait for them to finish).
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000925 for directory in (run_dir, tmp_dir, isolated_client_dir):
Takuto Ikuta69c0d662019-11-27 01:18:08 +0000926 if not fs.isdir(directory):
927 continue
maruel84537cb2015-10-16 14:21:28 -0700928 try:
Takuto Ikuta69c0d662019-11-27 01:18:08 +0000929 success = file_path.rmtree(directory)
maruel84537cb2015-10-16 14:21:28 -0700930 except OSError as e:
Takuto Ikuta69c0d662019-11-27 01:18:08 +0000931 logging.error('rmtree(%r) failed: %s', directory, e)
maruel84537cb2015-10-16 14:21:28 -0700932 success = False
933 if not success:
Takuto Ikuta69c0d662019-11-27 01:18:08 +0000934 sys.stderr.write(
935 OUTLIVING_ZOMBIE_MSG % (directory, data.grace_period))
maruel84537cb2015-10-16 14:21:28 -0700936 if result['exit_code'] == 0:
937 result['exit_code'] = 1
maruela9cfd6f2015-09-15 11:03:15 -0700938
marueleb5fbee2015-09-17 13:01:36 -0700939 # This deletes out_dir if leak_temp_dir is not set.
nodir9130f072016-05-27 13:59:08 -0700940 if out_dir:
nodir55715712016-06-03 12:28:19 -0700941 isolated_stats = result['stats'].setdefault('isolated', {})
942 result['outputs_ref'], success, isolated_stats['upload'] = (
Takuto Ikutab4aa8662019-09-17 05:54:36 +0000943 upload_then_delete(data.storage, out_dir, data.leak_temp_dir))
maruela9cfd6f2015-09-15 11:03:15 -0700944 if not success and result['exit_code'] == 0:
945 result['exit_code'] = 1
946 except Exception as e:
947 # Swallow any exception in the main finally clause.
nodir9130f072016-05-27 13:59:08 -0700948 if out_dir:
949 logging.exception('Leaking out_dir %s: %s', out_dir, e)
maruela9cfd6f2015-09-15 11:03:15 -0700950 result['internal_failure'] = str(e)
Takuto Ikutaa9a907b2020-04-17 08:50:50 +0000951 on_error.report(None)
maruela9cfd6f2015-09-15 11:03:15 -0700952 return result
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -0500953
954
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500955def run_tha_test(data, result_json):
nodir55be77b2016-05-03 09:39:57 -0700956 """Runs an executable and records execution metadata.
957
nodir55be77b2016-05-03 09:39:57 -0700958 If isolated_hash is specified, downloads the dependencies in the cache,
959 hardlinks them into a temporary directory and runs the command specified in
960 the .isolated.
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -0500961
962 A temporary directory is created to hold the output files. The content inside
963 this directory will be uploaded back to |storage| packaged as a .isolated
964 file.
965
966 Arguments:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500967 - data: TaskData instance.
968 - result_json: File path to dump result metadata into. If set, the process
969 exit code is always 0 unless an internal error occurred.
maruela9cfd6f2015-09-15 11:03:15 -0700970
971 Returns:
972 Process exit code that should be used.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000973 """
maruela76b9ee2015-12-15 06:18:08 -0800974 if result_json:
975 # Write a json output file right away in case we get killed.
976 result = {
977 'exit_code': None,
978 'had_hard_timeout': False,
979 'internal_failure': 'Was terminated before completion',
980 'outputs_ref': None,
nodirbe642ff2016-06-09 15:51:51 -0700981 'version': 5,
maruela76b9ee2015-12-15 06:18:08 -0800982 }
983 tools.write_json(result_json, result, dense=True)
984
maruela9cfd6f2015-09-15 11:03:15 -0700985 # run_isolated exit code. Depends on if result_json is used or not.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500986 result = map_and_run(data, True)
maruela9cfd6f2015-09-15 11:03:15 -0700987 logging.info('Result:\n%s', tools.format_json(result, dense=True))
bpastene3ae09522016-06-10 17:12:59 -0700988
maruela9cfd6f2015-09-15 11:03:15 -0700989 if result_json:
maruel05d5a882015-09-21 13:59:02 -0700990 # We've found tests to delete 'work' when quitting, causing an exception
991 # here. Try to recreate the directory if necessary.
nodire5028a92016-04-29 14:38:21 -0700992 file_path.ensure_tree(os.path.dirname(result_json))
maruela9cfd6f2015-09-15 11:03:15 -0700993 tools.write_json(result_json, result, dense=True)
994 # Only return 1 if there was an internal error.
995 return int(bool(result['internal_failure']))
maruel@chromium.org781ccf62013-09-17 19:39:47 +0000996
maruela9cfd6f2015-09-15 11:03:15 -0700997 # Marshall into old-style inline output.
998 if result['outputs_ref']:
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +0000999 # pylint: disable=unsubscriptable-object
maruela9cfd6f2015-09-15 11:03:15 -07001000 data = {
Junji Watanabe38b28b02020-04-23 10:23:30 +00001001 'hash': result['outputs_ref']['isolated'],
1002 'namespace': result['outputs_ref']['namespace'],
1003 'storage': result['outputs_ref']['isolatedserver'],
maruela9cfd6f2015-09-15 11:03:15 -07001004 }
Marc-Antoine Ruelc44f5722015-01-08 16:10:01 -05001005 sys.stdout.flush()
Junji Watanabe38b28b02020-04-23 10:23:30 +00001006 print('[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
1007 tools.format_json(data, dense=True))
maruelb76604c2015-11-11 11:53:44 -08001008 sys.stdout.flush()
maruela9cfd6f2015-09-15 11:03:15 -07001009 return result['exit_code'] or int(bool(result['internal_failure']))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001010
1011
iannuccib58d10d2017-03-18 02:00:25 -07001012# Yielded by 'install_client_and_packages'.
vadimsh232f5a82017-01-20 19:23:44 -08001013CipdInfo = collections.namedtuple('CipdInfo', [
1014 'client', # cipd.CipdClient object
1015 'cache_dir', # absolute path to bot-global cipd tag and instance cache
1016 'stats', # dict with stats to return to the server
1017 'pins', # dict with installed cipd pins to return to the server
1018])
1019
1020
1021@contextlib.contextmanager
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001022def noop_install_packages(_run_dir, _isolated_dir):
iannuccib58d10d2017-03-18 02:00:25 -07001023 """Placeholder for 'install_client_and_packages' if cipd is disabled."""
vadimsh232f5a82017-01-20 19:23:44 -08001024 yield None
1025
1026
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001027def _install_packages(run_dir, cipd_cache_dir, client, packages):
iannuccib58d10d2017-03-18 02:00:25 -07001028 """Calls 'cipd ensure' for packages.
1029
1030 Args:
1031 run_dir (str): root of installation.
1032 cipd_cache_dir (str): the directory to use for the cipd package cache.
1033 client (CipdClient): the cipd client to use
1034 packages: packages to install, list [(path, package_name, version), ...].
iannuccib58d10d2017-03-18 02:00:25 -07001035
1036 Returns: list of pinned packages. Looks like [
1037 {
1038 'path': 'subdirectory',
1039 'package_name': 'resolved/package/name',
1040 'version': 'deadbeef...',
1041 },
1042 ...
1043 ]
1044 """
1045 package_pins = [None]*len(packages)
1046 def insert_pin(path, name, version, idx):
1047 package_pins[idx] = {
1048 'package_name': name,
1049 # swarming deals with 'root' as '.'
1050 'path': path or '.',
1051 'version': version,
1052 }
1053
1054 by_path = collections.defaultdict(list)
1055 for i, (path, name, version) in enumerate(packages):
1056 # cipd deals with 'root' as ''
1057 if path == '.':
1058 path = ''
1059 by_path[path].append((name, version, i))
1060
1061 pins = client.ensure(
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001062 run_dir,
1063 {
1064 subdir: [(name, vers) for name, vers, _ in pkgs
1065 ] for subdir, pkgs in by_path.items()
1066 },
1067 cache_dir=cipd_cache_dir,
iannuccib58d10d2017-03-18 02:00:25 -07001068 )
1069
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001070 for subdir, pin_list in sorted(pins.items()):
iannuccib58d10d2017-03-18 02:00:25 -07001071 this_subdir = by_path[subdir]
1072 for i, (name, version) in enumerate(pin_list):
1073 insert_pin(subdir, name, version, this_subdir[i][2])
1074
Robert Iannucci461b30d2017-12-13 11:34:03 -08001075 assert None not in package_pins, (packages, pins, package_pins)
iannuccib58d10d2017-03-18 02:00:25 -07001076
1077 return package_pins
1078
1079
vadimsh232f5a82017-01-20 19:23:44 -08001080@contextlib.contextmanager
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001081def install_client_and_packages(run_dir, packages, service_url,
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001082 client_package_name, client_version, cache_dir,
1083 isolated_dir):
vadimsh902948e2017-01-20 15:57:32 -08001084 """Bootstraps CIPD client and installs CIPD packages.
iannucci96fcccc2016-08-30 15:52:22 -07001085
vadimsh232f5a82017-01-20 19:23:44 -08001086 Yields CipdClient, stats, client info and pins (as single CipdInfo object).
1087
1088 Pins and the CIPD client info are in the form of:
iannucci96fcccc2016-08-30 15:52:22 -07001089 [
1090 {
1091 "path": path, "package_name": package_name, "version": version,
1092 },
1093 ...
1094 ]
vadimsh902948e2017-01-20 15:57:32 -08001095 (the CIPD client info is a single dictionary instead of a list)
iannucci96fcccc2016-08-30 15:52:22 -07001096
1097 such that they correspond 1:1 to all input package arguments from the command
1098 line. These dictionaries make their all the way back to swarming, where they
1099 become the arguments of CipdPackage.
nodirbe642ff2016-06-09 15:51:51 -07001100
vadimsh902948e2017-01-20 15:57:32 -08001101 If 'packages' list is empty, will bootstrap CIPD client, but won't install
1102 any packages.
1103
1104 The bootstrapped client (regardless whether 'packages' list is empty or not),
vadimsh232f5a82017-01-20 19:23:44 -08001105 will be made available to the task via $PATH.
vadimsh902948e2017-01-20 15:57:32 -08001106
nodirbe642ff2016-06-09 15:51:51 -07001107 Args:
nodir90bc8dc2016-06-15 13:35:21 -07001108 run_dir (str): root of installation.
vadimsh902948e2017-01-20 15:57:32 -08001109 packages: packages to install, list [(path, package_name, version), ...].
nodirbe642ff2016-06-09 15:51:51 -07001110 service_url (str): CIPD server url, e.g.
1111 "https://chrome-infra-packages.appspot.com."
nodir90bc8dc2016-06-15 13:35:21 -07001112 client_package_name (str): CIPD package name of CIPD client.
1113 client_version (str): Version of CIPD client.
nodirbe642ff2016-06-09 15:51:51 -07001114 cache_dir (str): where to keep cache of cipd clients, packages and tags.
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001115 isolated_dir (str): where to download isolated client.
nodirbe642ff2016-06-09 15:51:51 -07001116 """
1117 assert cache_dir
nodir90bc8dc2016-06-15 13:35:21 -07001118
nodirbe642ff2016-06-09 15:51:51 -07001119 start = time.time()
nodirbe642ff2016-06-09 15:51:51 -07001120
vadimsh902948e2017-01-20 15:57:32 -08001121 cache_dir = os.path.abspath(cache_dir)
vadimsh232f5a82017-01-20 19:23:44 -08001122 cipd_cache_dir = os.path.join(cache_dir, 'cache') # tag and instance caches
nodir90bc8dc2016-06-15 13:35:21 -07001123 run_dir = os.path.abspath(run_dir)
vadimsh902948e2017-01-20 15:57:32 -08001124 packages = packages or []
nodir90bc8dc2016-06-15 13:35:21 -07001125
nodirbe642ff2016-06-09 15:51:51 -07001126 get_client_start = time.time()
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001127 client_manager = cipd.get_client(service_url, client_package_name,
1128 client_version, cache_dir)
iannucci96fcccc2016-08-30 15:52:22 -07001129
nodirbe642ff2016-06-09 15:51:51 -07001130 with client_manager as client:
1131 get_client_duration = time.time() - get_client_start
nodir90bc8dc2016-06-15 13:35:21 -07001132
iannuccib58d10d2017-03-18 02:00:25 -07001133 package_pins = []
1134 if packages:
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001135 package_pins = _install_packages(run_dir, cipd_cache_dir, client,
1136 packages)
iannuccib58d10d2017-03-18 02:00:25 -07001137
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001138 # Install isolated client to |isolated_dir|.
Takuto Ikuta02edca22019-11-29 10:04:51 +00001139 _install_packages(isolated_dir, cipd_cache_dir, client,
1140 [('', ISOLATED_PACKAGE, ISOLATED_REVISION)])
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001141
iannuccib58d10d2017-03-18 02:00:25 -07001142 file_path.make_tree_files_read_only(run_dir)
nodir90bc8dc2016-06-15 13:35:21 -07001143
vadimsh232f5a82017-01-20 19:23:44 -08001144 total_duration = time.time() - start
Junji Watanabe38b28b02020-04-23 10:23:30 +00001145 logging.info('Installing CIPD client and packages took %d seconds',
1146 total_duration)
nodir90bc8dc2016-06-15 13:35:21 -07001147
vadimsh232f5a82017-01-20 19:23:44 -08001148 yield CipdInfo(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001149 client=client,
1150 cache_dir=cipd_cache_dir,
1151 stats={
1152 'duration': total_duration,
1153 'get_client_duration': get_client_duration,
iannuccib58d10d2017-03-18 02:00:25 -07001154 },
Junji Watanabe38b28b02020-04-23 10:23:30 +00001155 pins={
1156 'client_package': {
1157 'package_name': client.package_name,
1158 'version': client.instance_id,
1159 },
1160 'packages': package_pins,
1161 })
nodirbe642ff2016-06-09 15:51:51 -07001162
1163
1164def create_option_parser():
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001165 parser = logging_utils.OptionParserWithLogging(
nodir55be77b2016-05-03 09:39:57 -07001166 usage='%prog <options> [command to run or extra args]',
maruel@chromium.orgdedbf492013-09-12 20:42:11 +00001167 version=__version__,
1168 log_file=RUN_ISOLATED_LOG_FILE)
maruela9cfd6f2015-09-15 11:03:15 -07001169 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001170 '--clean',
1171 action='store_true',
maruel36a963d2016-04-08 17:15:49 -07001172 help='Cleans the cache, trimming it necessary and remove corrupted items '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001173 'and returns without executing anything; use with -v to know what '
1174 'was done')
maruel36a963d2016-04-08 17:15:49 -07001175 parser.add_option(
maruela9cfd6f2015-09-15 11:03:15 -07001176 '--json',
1177 help='dump output metadata to json file. When used, run_isolated returns '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001178 'non-zero only on internal failure')
maruel6be7f9e2015-10-01 12:25:30 -07001179 parser.add_option(
maruel5c9e47b2015-12-18 13:02:30 -08001180 '--hard-timeout', type='float', help='Enforce hard timeout in execution')
maruel6be7f9e2015-10-01 12:25:30 -07001181 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001182 '--grace-period',
1183 type='float',
maruel6be7f9e2015-10-01 12:25:30 -07001184 help='Grace period between SIGTERM and SIGKILL')
bpastene3ae09522016-06-10 17:12:59 -07001185 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001186 '--raw-cmd',
1187 action='store_true',
Marc-Antoine Ruel49e347d2017-10-24 16:52:02 -07001188 help='Ignore the isolated command, use the one supplied at the command '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001189 'line')
Marc-Antoine Ruel49e347d2017-10-24 16:52:02 -07001190 parser.add_option(
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001191 '--relative-cwd',
1192 help='Ignore the isolated \'relative_cwd\' and use this one instead; '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001193 'requires --raw-cmd')
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001194 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001195 '--env',
1196 default=[],
1197 action='append',
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001198 help='Environment variables to set for the child process')
1199 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001200 '--env-prefix',
1201 default=[],
1202 action='append',
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001203 help='Specify a VAR=./path/fragment to put in the environment variable '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001204 'before executing the command. The path fragment must be relative '
1205 'to the isolated run directory, and must not contain a `..` token. '
1206 'The path will be made absolute and prepended to the indicated '
1207 '$VAR using the OS\'s path separator. Multiple items for the same '
1208 '$VAR will be prepended in order.')
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001209 parser.add_option(
bpastene3ae09522016-06-10 17:12:59 -07001210 '--bot-file',
1211 help='Path to a file describing the state of the host. The content is '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001212 'defined by on_before_task() in bot_config.')
aludwin7556e0c2016-10-26 08:46:10 -07001213 parser.add_option(
vadimsh9c54b2c2017-07-25 14:08:29 -07001214 '--switch-to-account',
1215 help='If given, switches LUCI_CONTEXT to given logical service account '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001216 '(e.g. "task" or "system") before launching the isolated process.')
vadimsh9c54b2c2017-07-25 14:08:29 -07001217 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001218 '--output',
1219 action='append',
aludwin0a8e17d2016-10-27 15:57:39 -07001220 help='Specifies an output to return. If no outputs are specified, all '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001221 'files located in $(ISOLATED_OUTDIR) will be returned; '
1222 'otherwise, outputs in both $(ISOLATED_OUTDIR) and those '
1223 'specified by --output option (there can be multiple) will be '
1224 'returned. Note that if a file in OUT_DIR has the same path '
1225 'as an --output option, the --output version will be returned.')
aludwin0a8e17d2016-10-27 15:57:39 -07001226 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001227 '-a',
1228 '--argsfile',
aludwin7556e0c2016-10-26 08:46:10 -07001229 # This is actually handled in parse_args; it's included here purely so it
1230 # can make it into the help text.
1231 help='Specify a file containing a JSON array of arguments to this '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001232 'script. If --argsfile is provided, no other argument may be '
1233 'provided on the command line.')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001234
1235 group = optparse.OptionGroup(parser, 'Data source')
1236 group.add_option(
Marc-Antoine Ruel185ded42015-01-28 20:49:18 -05001237 '-s', '--isolated',
nodir55be77b2016-05-03 09:39:57 -07001238 help='Hash of the .isolated to grab from the isolate server.')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001239 isolateserver.add_isolate_server_options(group)
1240 parser.add_option_group(group)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001241
Marc-Antoine Ruela57d7db2014-10-15 20:31:19 -04001242 isolateserver.add_cache_options(parser)
nodirbe642ff2016-06-09 15:51:51 -07001243
1244 cipd.add_cipd_options(parser)
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001245
1246 group = optparse.OptionGroup(parser, 'Named caches')
1247 group.add_option(
1248 '--named-cache',
1249 dest='named_caches',
1250 action='append',
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001251 nargs=3,
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001252 default=[],
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001253 help='A named cache to request. Accepts 3 arguments: name, path, hint. '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001254 'name identifies the cache, must match regex [a-z0-9_]{1,4096}. '
1255 'path is a path relative to the run dir where the cache directory '
1256 'must be put to. '
1257 'This option can be specified more than once.')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001258 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001259 '--named-cache-root',
1260 default='named_caches',
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001261 help='Cache root directory. Default=%default')
1262 parser.add_option_group(group)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001263
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001264 group = optparse.OptionGroup(parser, 'Process containment')
1265 parser.add_option(
1266 '--lower-priority', action='store_true',
1267 help='Lowers the child process priority')
1268 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001269 '--containment-type',
1270 choices=('NONE', 'AUTO', 'JOB_OBJECT'),
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001271 default='NONE',
1272 help='Type of container to use')
1273 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001274 '--limit-processes',
1275 type='int',
1276 default=0,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001277 help='Maximum number of active processes in the containment')
1278 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001279 '--limit-total-committed-memory',
1280 type='int',
1281 default=0,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001282 help='Maximum sum of committed memory in the containment')
1283 parser.add_option_group(group)
1284
1285 group = optparse.OptionGroup(parser, 'Debugging')
1286 group.add_option(
Kenneth Russell61d42352014-09-15 11:41:16 -07001287 '--leak-temp-dir',
1288 action='store_true',
nodirbe642ff2016-06-09 15:51:51 -07001289 help='Deliberately leak isolate\'s temp dir for later examination. '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001290 'Default: %default')
1291 group.add_option('--root-dir', help='Use a directory instead of a random one')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001292 parser.add_option_group(group)
Kenneth Russell61d42352014-09-15 11:41:16 -07001293
Vadim Shtayurae34e13a2014-02-02 11:23:26 -08001294 auth.add_auth_options(parser)
nodirbe642ff2016-06-09 15:51:51 -07001295
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001296 parser.set_defaults(cache='cache', cipd_cache='cipd_cache')
nodirbe642ff2016-06-09 15:51:51 -07001297 return parser
1298
1299
Marc-Antoine Ruel49f9f8d2018-05-24 15:57:06 -04001300def process_named_cache_options(parser, options, time_fn=None):
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001301 """Validates named cache options and returns a CacheManager."""
1302 if options.named_caches and not options.named_cache_root:
1303 parser.error('--named-cache is specified, but --named-cache-root is empty')
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001304 for name, path, hint in options.named_caches:
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001305 if not CACHE_NAME_RE.match(name):
1306 parser.error(
1307 'cache name %r does not match %r' % (name, CACHE_NAME_RE.pattern))
1308 if not path:
1309 parser.error('cache path cannot be empty')
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001310 try:
1311 long(hint)
1312 except ValueError:
1313 parser.error('cache hint must be a number')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001314 if options.named_cache_root:
1315 # Make these configurable later if there is use case but for now it's fairly
1316 # safe values.
1317 # In practice, a fair chunk of bots are already recycled on a daily schedule
1318 # so this code doesn't have any effect to them, unless they are preloaded
1319 # with a really old cache.
1320 policies = local_caching.CachePolicies(
1321 # 1TiB.
1322 max_cache_size=1024*1024*1024*1024,
1323 min_free_space=options.min_free_space,
1324 max_items=50,
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +00001325 max_age_secs=MAX_AGE_SECS)
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001326 root_dir = six.text_type(os.path.abspath(options.named_cache_root))
John Budorickc6186972020-02-26 00:58:14 +00001327 cache = local_caching.NamedCache(root_dir, policies, time_fn=time_fn)
1328 # Touch any named caches we're going to use to minimize thrashing
1329 # between tasks that request some (but not all) of the same named caches.
John Budorick0a4dab62020-03-02 22:23:35 +00001330 cache.touch(*[name for name, _, _ in options.named_caches])
John Budorickc6186972020-02-26 00:58:14 +00001331 return cache
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001332 return None
1333
1334
aludwin7556e0c2016-10-26 08:46:10 -07001335def parse_args(args):
1336 # Create a fake mini-parser just to get out the "-a" command. Note that
1337 # it's not documented here; instead, it's documented in create_option_parser
1338 # even though that parser will never actually get to parse it. This is
1339 # because --argsfile is exclusive with all other options and arguments.
1340 file_argparse = argparse.ArgumentParser(add_help=False)
1341 file_argparse.add_argument('-a', '--argsfile')
1342 (file_args, nonfile_args) = file_argparse.parse_known_args(args)
1343 if file_args.argsfile:
1344 if nonfile_args:
1345 file_argparse.error('Can\'t specify --argsfile with'
1346 'any other arguments (%s)' % nonfile_args)
1347 try:
1348 with open(file_args.argsfile, 'r') as f:
1349 args = json.load(f)
1350 except (IOError, OSError, ValueError) as e:
1351 # We don't need to error out here - "args" is now empty,
1352 # so the call below to parser.parse_args(args) will fail
1353 # and print the full help text.
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +00001354 print('Couldn\'t read arguments: %s' % e, file=sys.stderr)
aludwin7556e0c2016-10-26 08:46:10 -07001355
1356 # Even if we failed to read the args, just call the normal parser now since it
1357 # will print the correct help message.
nodirbe642ff2016-06-09 15:51:51 -07001358 parser = create_option_parser()
Marc-Antoine Ruel90c98162013-12-18 15:11:57 -05001359 options, args = parser.parse_args(args)
aludwin7556e0c2016-10-26 08:46:10 -07001360 return (parser, options, args)
1361
1362
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001363def _calc_named_cache_hint(named_cache, named_caches):
1364 """Returns the expected size of the missing named caches."""
1365 present = named_cache.available
1366 size = 0
1367 for name, _, hint in named_caches:
1368 if name not in present:
1369 hint = long(hint)
1370 if hint > 0:
1371 size += hint
1372 return size
1373
1374
aludwin7556e0c2016-10-26 08:46:10 -07001375def main(args):
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -05001376 # Warning: when --argsfile is used, the strings are unicode instances, when
1377 # parsed normally, the strings are str instances.
aludwin7556e0c2016-10-26 08:46:10 -07001378 (parser, options, args) = parse_args(args)
maruel36a963d2016-04-08 17:15:49 -07001379
Marc-Antoine Ruel5028ba22017-08-25 17:37:51 -04001380 if not file_path.enable_symlink():
Marc-Antoine Ruel5a024272019-01-15 20:11:16 +00001381 logging.warning('Symlink support is not enabled')
Marc-Antoine Ruel5028ba22017-08-25 17:37:51 -04001382
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001383 named_cache = process_named_cache_options(parser, options)
Marc-Antoine Ruel0d8b0f62018-09-10 14:40:35 +00001384 # hint is 0 if there's no named cache.
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001385 hint = _calc_named_cache_hint(named_cache, options.named_caches)
1386 if hint:
1387 # Increase the --min-free-space value by the hint, and recreate the
1388 # NamedCache instance so it gets the updated CachePolicy.
1389 options.min_free_space += hint
1390 named_cache = process_named_cache_options(parser, options)
1391
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001392 # TODO(crbug.com/932396): Remove this.
1393 use_go_isolated = (
1394 options.cipd_enabled and
Takuto Ikuta64a9c2c2020-02-06 06:10:07 +00001395 # TODO(crbug.com/1045281): windows other than win10 has flaky connection
1396 # issue.
1397 (sys.platform != 'win32' or platform.release() == '10'))
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001398
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001399 # TODO(maruel): CIPD caches should be defined at an higher level here too, so
1400 # they can be cleaned the same way.
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001401 if use_go_isolated:
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +00001402 isolate_cache = None
1403 else:
1404 isolate_cache = isolateserver.process_cache_options(options, trim=False)
1405
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001406 caches = []
1407 if isolate_cache:
1408 caches.append(isolate_cache)
1409 if named_cache:
1410 caches.append(named_cache)
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001411 root = caches[0].cache_dir if caches else six.text_type(os.getcwd())
maruel36a963d2016-04-08 17:15:49 -07001412 if options.clean:
1413 if options.isolated:
1414 parser.error('Can\'t use --isolated with --clean.')
1415 if options.isolate_server:
1416 parser.error('Can\'t use --isolate-server with --clean.')
1417 if options.json:
1418 parser.error('Can\'t use --json with --clean.')
nodirf33b8d62016-10-26 22:34:58 -07001419 if options.named_caches:
1420 parser.error('Can\t use --named-cache with --clean.')
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001421 # Trim first, then clean.
1422 local_caching.trim_caches(
1423 caches,
1424 root,
1425 min_free_space=options.min_free_space,
1426 max_age_secs=MAX_AGE_SECS)
1427 for c in caches:
Marc-Antoine Ruel87fc2222018-06-18 13:09:24 +00001428 c.cleanup()
maruel36a963d2016-04-08 17:15:49 -07001429 return 0
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001430
1431 # Trim must still be done for the following case:
1432 # - named-cache was used
1433 # - some entries, with a large hint, where missing
1434 # - --min-free-space was increased accordingly, thus trimming is needed
1435 # Otherwise, this will have no effect, as bot_main calls run_isolated with
1436 # --clean after each task.
1437 if hint:
1438 logging.info('Additional trimming of %d bytes', hint)
Marc-Antoine Ruel0d8b0f62018-09-10 14:40:35 +00001439 local_caching.trim_caches(
1440 caches,
1441 root,
1442 min_free_space=options.min_free_space,
1443 max_age_secs=MAX_AGE_SECS)
maruel36a963d2016-04-08 17:15:49 -07001444
nodir55be77b2016-05-03 09:39:57 -07001445 if not options.isolated and not args:
1446 parser.error('--isolated or command to run is required.')
1447
Vadim Shtayura5d1efce2014-02-04 10:55:43 -08001448 auth.process_auth_options(parser, options)
nodir55be77b2016-05-03 09:39:57 -07001449
Takuto Ikutaae767b32020-05-11 01:22:19 +00001450 isolateserver.process_isolate_server_options(parser, options, False)
nodir55be77b2016-05-03 09:39:57 -07001451 if not options.isolate_server:
1452 if options.isolated:
1453 parser.error('--isolated requires --isolate-server')
1454 if ISOLATED_OUTDIR_PARAMETER in args:
1455 parser.error(
1456 '%s in args requires --isolate-server' % ISOLATED_OUTDIR_PARAMETER)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001457
nodir90bc8dc2016-06-15 13:35:21 -07001458 if options.root_dir:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001459 options.root_dir = six.text_type(os.path.abspath(options.root_dir))
maruel12e30012015-10-09 11:55:35 -07001460 if options.json:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001461 options.json = six.text_type(os.path.abspath(options.json))
nodir55be77b2016-05-03 09:39:57 -07001462
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001463 if any('=' not in i for i in options.env):
1464 parser.error(
1465 '--env required key=value form. value can be skipped to delete '
1466 'the variable')
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -05001467 options.env = dict(i.split('=', 1) for i in options.env)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001468
1469 prefixes = {}
1470 cwd = os.path.realpath(os.getcwd())
1471 for item in options.env_prefix:
1472 if '=' not in item:
1473 parser.error(
1474 '--env-prefix %r is malformed, must be in the form `VAR=./path`'
1475 % item)
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -05001476 key, opath = item.split('=', 1)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001477 if os.path.isabs(opath):
1478 parser.error('--env-prefix %r path is bad, must be relative.' % opath)
1479 opath = os.path.normpath(opath)
1480 if not os.path.realpath(os.path.join(cwd, opath)).startswith(cwd):
1481 parser.error(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001482 '--env-prefix %r path is bad, must be relative and not contain `..`.'
1483 % opath)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001484 prefixes.setdefault(key, []).append(opath)
1485 options.env_prefix = prefixes
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001486
nodirbe642ff2016-06-09 15:51:51 -07001487 cipd.validate_cipd_options(parser, options)
1488
vadimsh232f5a82017-01-20 19:23:44 -08001489 install_packages_fn = noop_install_packages
vadimsh902948e2017-01-20 15:57:32 -08001490 if options.cipd_enabled:
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001491 install_packages_fn = (
1492 lambda run_dir, isolated_dir: install_client_and_packages(
vadimsh902948e2017-01-20 15:57:32 -08001493 run_dir, cipd.parse_package_args(options.cipd_packages),
1494 options.cipd_server, options.cipd_client_package,
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001495 options.cipd_client_version, cache_dir=options.cipd_cache,
1496 isolated_dir=isolated_dir))
nodirbe642ff2016-06-09 15:51:51 -07001497
nodird6160682017-02-02 13:03:35 -08001498 @contextlib.contextmanager
nodir0ae98b32017-05-11 13:21:53 -07001499 def install_named_caches(run_dir):
nodird6160682017-02-02 13:03:35 -08001500 # WARNING: this function depends on "options" variable defined in the outer
1501 # function.
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001502 assert six.text_type(run_dir), repr(run_dir)
Marc-Antoine Ruel49f9f8d2018-05-24 15:57:06 -04001503 assert os.path.isabs(run_dir), run_dir
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001504 named_caches = [(os.path.join(run_dir, six.text_type(relpath)), name)
1505 for name, relpath, _ in options.named_caches]
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001506 for path, name in named_caches:
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001507 named_cache.install(path, name)
nodird6160682017-02-02 13:03:35 -08001508 try:
1509 yield
1510 finally:
dnje289d132017-07-07 11:16:44 -07001511 # Uninstall each named cache, returning it to the cache pool. If an
1512 # uninstall fails for a given cache, it will remain in the task's
1513 # temporary space, get cleaned up by the Swarming bot, and be lost.
1514 #
1515 # If the Swarming bot cannot clean up the cache, it will handle it like
1516 # any other bot file that could not be removed.
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001517 for path, name in reversed(named_caches):
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001518 try:
Marc-Antoine Ruele9558372018-08-03 03:41:22 +00001519 # uninstall() doesn't trim but does call save() implicitly. Trimming
1520 # *must* be done manually via periodic 'run_isolated.py --clean'.
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001521 named_cache.uninstall(path, name)
1522 except local_caching.NamedCacheError:
1523 logging.exception('Error while removing named cache %r at %r. '
1524 'The cache will be lost.', path, name)
nodirf33b8d62016-10-26 22:34:58 -07001525
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001526 extra_args = []
1527 command = []
1528 if options.raw_cmd:
1529 command = args
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001530 if options.relative_cwd:
1531 a = os.path.normpath(os.path.abspath(options.relative_cwd))
1532 if not a.startswith(os.getcwd()):
1533 parser.error(
1534 '--relative-cwd must not try to escape the working directory')
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001535 else:
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001536 if options.relative_cwd:
1537 parser.error('--relative-cwd requires --raw-cmd')
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001538 extra_args = args
1539
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001540 containment_type = subprocess42.Containment.NONE
1541 if options.containment_type == 'AUTO':
1542 containment_type = subprocess42.Containment.AUTO
1543 if options.containment_type == 'JOB_OBJECT':
1544 containment_type = subprocess42.Containment.JOB_OBJECT
1545 containment = subprocess42.Containment(
1546 containment_type=containment_type,
1547 limit_processes=options.limit_processes,
1548 limit_total_committed_memory=options.limit_total_committed_memory)
1549
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001550 data = TaskData(
1551 command=command,
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001552 relative_cwd=options.relative_cwd,
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001553 extra_args=extra_args,
1554 isolated_hash=options.isolated,
1555 storage=None,
1556 isolate_cache=isolate_cache,
1557 outputs=options.output,
1558 install_named_caches=install_named_caches,
1559 leak_temp_dir=options.leak_temp_dir,
1560 root_dir=_to_unicode(options.root_dir),
1561 hard_timeout=options.hard_timeout,
1562 grace_period=options.grace_period,
1563 bot_file=options.bot_file,
1564 switch_to_account=options.switch_to_account,
1565 install_packages_fn=install_packages_fn,
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001566 use_go_isolated=use_go_isolated,
Takuto Ikuta10cae642020-01-08 08:12:07 +00001567 go_cache_dir=options.cache,
Takuto Ikuta879788c2020-01-10 08:00:26 +00001568 go_cache_policies=local_caching.CachePolicies(
1569 max_cache_size=options.max_cache_size,
1570 min_free_space=options.min_free_space,
1571 max_items=options.max_items,
1572 max_age_secs=None,
1573 ),
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001574 env=options.env,
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +00001575 env_prefix=options.env_prefix,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001576 lower_priority=bool(options.lower_priority),
1577 containment=containment)
nodirbe642ff2016-06-09 15:51:51 -07001578 try:
nodir90bc8dc2016-06-15 13:35:21 -07001579 if options.isolate_server:
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001580 server_ref = isolate_storage.ServerRef(
nodir90bc8dc2016-06-15 13:35:21 -07001581 options.isolate_server, options.namespace)
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001582 storage = isolateserver.get_storage(server_ref)
nodir90bc8dc2016-06-15 13:35:21 -07001583 with storage:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001584 data = data._replace(storage=storage)
nodirf33b8d62016-10-26 22:34:58 -07001585 # Hashing schemes used by |storage| and |isolate_cache| MUST match.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001586 assert storage.server_ref.hash_algo == server_ref.hash_algo
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001587 return run_tha_test(data, options.json)
1588 return run_tha_test(data, options.json)
Junji Watanabe38b28b02020-04-23 10:23:30 +00001589 except (cipd.Error, local_caching.NamedCacheError,
1590 local_caching.NoMoreSpace) as ex:
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +00001591 print(ex.message, file=sys.stderr)
nodirbe642ff2016-06-09 15:51:51 -07001592 return 1
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001593
1594
1595if __name__ == '__main__':
maruel8e4e40c2016-05-30 06:21:07 -07001596 subprocess42.inhibit_os_error_reporting()
csharp@chromium.orgbfb98742013-03-26 20:28:36 +00001597 # Ensure that we are always running with the correct encoding.
vadimsh@chromium.orga4326472013-08-24 02:05:41 +00001598 fix_encoding.fix_encoding()
Ye Kuang2dd17442020-04-22 08:45:52 +00001599 net.set_user_agent('run_isolated.py/' + __version__)
Marc-Antoine Ruel90c98162013-12-18 15:11:57 -05001600 sys.exit(main(sys.argv[1:]))