blob: 629db585775f15410e986d35ffc99a7992f7d7d4 [file] [log] [blame]
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001#!/usr/bin/env python
maruelea586f32016-04-05 11:11:33 -07002# Copyright 2012 The LUCI Authors. All rights reserved.
maruelf1f5e2a2016-05-25 17:10:39 -07003# Use of this source code is governed under the Apache License, Version 2.0
4# that can be found in the LICENSE file.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00005
nodir55be77b2016-05-03 09:39:57 -07006"""Runs a command with optional isolated input/output.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00007
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +00008run_isolated takes cares of setting up a temporary environment, running a
9command, and tearing it down.
nodir55be77b2016-05-03 09:39:57 -070010
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +000011It handles downloading and uploading isolated files, mapping CIPD packages and
12reusing stateful named caches.
13
14The isolated files, CIPD packages and named caches are kept as a global LRU
15cache.
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -050016
Roberto Carrillo71ade6d2018-10-08 22:30:24 +000017Any ${EXECUTABLE_SUFFIX} on the command line or the environment variables passed
18with the --env option will be replaced with ".exe" string on Windows and "" on
19other platforms.
nodirbe642ff2016-06-09 15:51:51 -070020
Roberto Carrillo71ade6d2018-10-08 22:30:24 +000021Any ${ISOLATED_OUTDIR} on the command line or the environment variables passed
22with the --env option will be replaced by the location of a temporary directory
23upon execution of the command specified in the .isolated file. All content
24written to this directory will be uploaded upon termination and the .isolated
25file describing this directory will be printed to stdout.
bpastene447c1992016-06-20 15:21:47 -070026
Marc-Antoine Rueleed2f3a2019-03-14 00:00:40 +000027Any ${SWARMING_BOT_FILE} on the command line or the environment variables passed
28with the --env option will be replaced by the value of the --bot-file parameter.
29This file is used by a swarming bot to communicate state of the host to tasks.
30It is written to by the swarming bot's on_before_task() hook in the swarming
31server's custom bot_config.py.
32
33See
34https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Magic-Values.md
35for all the variables.
36
37See
38https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/swarming_bot/config/bot_config.py
39for more information about bot_config.py.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000040"""
41
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +000042from __future__ import print_function
43
44__version__ = '1.0.1'
maruel@chromium.orgdedbf492013-09-12 20:42:11 +000045
aludwin7556e0c2016-10-26 08:46:10 -070046import argparse
maruel064c0a32016-04-05 11:47:15 -070047import base64
iannucci96fcccc2016-08-30 15:52:22 -070048import collections
vadimsh232f5a82017-01-20 19:23:44 -080049import contextlib
Ye Kuangfff1e502020-07-13 13:21:57 +000050import distutils
Sadaf Matinkhoo10743a62018-03-29 16:28:58 -040051import errno
aludwin7556e0c2016-10-26 08:46:10 -070052import json
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000053import logging
54import optparse
55import os
Takuto Ikuta5c59a842020-01-24 03:05:24 +000056import platform
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -040057import re
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000058import sys
59import tempfile
maruel064c0a32016-04-05 11:47:15 -070060import time
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +000061
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000062from utils import tools
63tools.force_local_third_party()
vadimsh@chromium.orga4326472013-08-24 02:05:41 +000064
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000065# third_party/
66from depot_tools import fix_encoding
Takuto Ikuta6e2ff962019-10-29 12:35:27 +000067import six
Marc-Antoine Ruel016c7602019-04-02 18:31:13 +000068
69# pylint: disable=ungrouped-imports
70import auth
71import cipd
72import isolate_storage
73import isolateserver
74import local_caching
75from libs import luci_context
Vadim Shtayura6b555c12014-07-23 16:22:18 -070076from utils import file_path
maruel12e30012015-10-09 11:55:35 -070077from utils import fs
maruel064c0a32016-04-05 11:47:15 -070078from utils import large
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -040079from utils import logging_utils
Ye Kuang2dd17442020-04-22 08:45:52 +000080from utils import net
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -040081from utils import on_error
Marc-Antoine Ruelc44f5722015-01-08 16:10:01 -050082from utils import subprocess42
maruel@chromium.orgdedbf492013-09-12 20:42:11 +000083
vadimsh@chromium.orga4326472013-08-24 02:05:41 +000084
maruele2f2cb82016-07-13 14:41:03 -070085# Magic variables that can be found in the isolate task command line.
86ISOLATED_OUTDIR_PARAMETER = '${ISOLATED_OUTDIR}'
87EXECUTABLE_SUFFIX_PARAMETER = '${EXECUTABLE_SUFFIX}'
88SWARMING_BOT_FILE_PARAMETER = '${SWARMING_BOT_FILE}'
89
90
csharp@chromium.orgff2a4662012-11-21 20:49:32 +000091# The name of the log file to use.
92RUN_ISOLATED_LOG_FILE = 'run_isolated.log'
93
maruele2f2cb82016-07-13 14:41:03 -070094
csharp@chromium.orge217f302012-11-22 16:51:53 +000095# The name of the log to use for the run_test_cases.py command
vadimsh@chromium.org8b9d56b2013-08-21 22:24:35 +000096RUN_TEST_CASES_LOG = 'run_test_cases.log'
csharp@chromium.orge217f302012-11-22 16:51:53 +000097
vadimsh@chromium.org87d63262013-04-04 19:34:21 +000098
maruele2f2cb82016-07-13 14:41:03 -070099# Use short names for temporary directories. This is driven by Windows, which
100# imposes a relatively short maximum path length of 260 characters, often
101# referred to as MAX_PATH. It is relatively easy to create files with longer
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +0000102# path length. A use case is with recursive dependency trees like npm packages.
maruele2f2cb82016-07-13 14:41:03 -0700103#
104# It is recommended to start the script with a `root_dir` as short as
105# possible.
106# - ir stands for isolated_run
107# - io stands for isolated_out
108# - it stands for isolated_tmp
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000109# - ic stands for isolated_client
maruele2f2cb82016-07-13 14:41:03 -0700110ISOLATED_RUN_DIR = u'ir'
111ISOLATED_OUT_DIR = u'io'
112ISOLATED_TMP_DIR = u'it'
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000113ISOLATED_CLIENT_DIR = u'ic'
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000114_CAS_CLIENT_DIR = u'cc'
maruele2f2cb82016-07-13 14:41:03 -0700115
Takuto Ikuta02edca22019-11-29 10:04:51 +0000116# TODO(tikuta): take these parameter from luci-config?
Takuto Ikuta9c4eb1d2020-10-05 03:40:14 +0000117# Update tag by `./client/update_go_clients.sh`.
Takuto Ikutac8c92e62020-04-01 07:07:29 +0000118# Or take revision from
Takuto Ikutab7ce0e32019-11-27 23:26:18 +0000119# https://ci.chromium.org/p/infra-internal/g/infra-packagers/console
Takuto Ikuta02edca22019-11-29 10:04:51 +0000120ISOLATED_PACKAGE = 'infra/tools/luci/isolated/${platform}'
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000121_CAS_PACKAGE = 'infra/tools/luci/cas/${platform}'
Takuto Ikuta10a54332021-01-14 23:50:01 +0000122_LUCI_GO_REVISION = 'git_revision:77944aa535e42e29faadf6cfa81aee252807d468'
maruele2f2cb82016-07-13 14:41:03 -0700123
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -0400124# Keep synced with task_request.py
Lei Leife202df2019-06-11 17:33:34 +0000125CACHE_NAME_RE = re.compile(r'^[a-z0-9_]{1,4096}$')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -0400126
Takuto Ikutaa010c532020-10-21 05:42:29 +0000127_FREE_SPACE_BUFFER_FOR_GO = 1024 * 1024 * 1024
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -0400128
marueld928c862017-06-08 08:20:04 -0700129OUTLIVING_ZOMBIE_MSG = """\
130*** Swarming tried multiple times to delete the %s directory and failed ***
131*** Hard failing the task ***
132
133Swarming detected that your testing script ran an executable, which may have
134started a child executable, and the main script returned early, leaving the
135children executables playing around unguided.
136
137You don't want to leave children processes outliving the task on the Swarming
138bot, do you? The Swarming bot doesn't.
139
140How to fix?
141- For any process that starts children processes, make sure all children
142 processes terminated properly before each parent process exits. This is
143 especially important in very deep process trees.
144 - This must be done properly both in normal successful task and in case of
145 task failure. Cleanup is very important.
146- The Swarming bot sends a SIGTERM in case of timeout.
147 - You have %s seconds to comply after the signal was sent to the process
148 before the process is forcibly killed.
149- To achieve not leaking children processes in case of signals on timeout, you
150 MUST handle signals in each executable / python script and propagate them to
151 children processes.
152 - When your test script (python or binary) receives a signal like SIGTERM or
153 CTRL_BREAK_EVENT on Windows), send it to all children processes and wait for
154 them to terminate before quitting.
155
156See
Marc-Antoine Ruelc7243592018-05-24 17:04:04 -0400157https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Bot.md#Graceful-termination_aka-the-SIGTERM-and-SIGKILL-dance
marueld928c862017-06-08 08:20:04 -0700158for more information.
159
160*** May the SIGKILL force be with you ***
161"""
162
163
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000164# Currently hardcoded. Eventually could be exposed as a flag once there's value.
165# 3 weeks
166MAX_AGE_SECS = 21*24*60*60
167
Ye Kuang72e6fe82020-08-05 06:30:04 +0000168# TODO(1099655): Enable this once all prod issues are gone.
169_USE_GO_ISOLATED_TO_UPLOAD = False
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000170
Takuto Ikuta7ff4b242020-12-03 08:07:06 +0000171_CAS_KVS_CACHE_THRESHOLD = 5 * 1024 * 1024 * 1024 # 5 GiB
172
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500173TaskData = collections.namedtuple(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000174 'TaskData',
175 [
Takuto Ikuta9a319502019-11-26 07:40:14 +0000176 # List of strings; the command line to use, independent of what was
177 # specified in the isolated file.
178 'command',
179 # Relative directory to start command into.
180 'relative_cwd',
Takuto Ikuta9a319502019-11-26 07:40:14 +0000181 # Hash of the .isolated file that must be retrieved to recreate the tree
182 # of files to run the target executable. The command specified in the
183 # .isolated is executed. Mutually exclusive with command argument.
184 'isolated_hash',
185 # isolateserver.Storage instance to retrieve remote objects. This object
186 # has a reference to an isolateserver.StorageApi, which does the actual
187 # I/O.
188 'storage',
189 # isolateserver.LocalCache instance to keep from retrieving the same
190 # objects constantly by caching the objects retrieved. Can be on-disk or
191 # in-memory.
192 'isolate_cache',
Junji Watanabe54925c32020-09-08 00:56:18 +0000193 # Digest of the input root on RBE-CAS.
194 'cas_digest',
195 # Full CAS instance name.
196 'cas_instance',
Takuto Ikuta9a319502019-11-26 07:40:14 +0000197 # List of paths relative to root_dir to put into the output isolated
198 # bundle upon task completion (see link_outputs_to_outdir).
199 'outputs',
200 # Function (run_dir) => context manager that installs named caches into
201 # |run_dir|.
202 'install_named_caches',
203 # If True, the temporary directory will be deliberately leaked for later
204 # examination.
205 'leak_temp_dir',
206 # Path to the directory to use to create the temporary directory. If not
207 # specified, a random temporary directory is created.
208 'root_dir',
209 # Kills the process if it lasts more than this amount of seconds.
210 'hard_timeout',
211 # Number of seconds to wait between SIGTERM and SIGKILL.
212 'grace_period',
213 # Path to a file with bot state, used in place of ${SWARMING_BOT_FILE}
214 # task command line argument.
215 'bot_file',
216 # Logical account to switch LUCI_CONTEXT into.
217 'switch_to_account',
218 # Context manager dir => CipdInfo, see install_client_and_packages.
219 'install_packages_fn',
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000220 # Use go isolated client.
221 'use_go_isolated',
Junji Watanabeb03450b2020-09-25 05:09:27 +0000222 # Cache directory for go `isolated` client.
Takuto Ikuta057c5342019-12-03 04:05:05 +0000223 'go_cache_dir',
Junji Watanabeb03450b2020-09-25 05:09:27 +0000224 # Parameters passed to go `isolated` client.
Takuto Ikuta879788c2020-01-10 08:00:26 +0000225 'go_cache_policies',
Junji Watanabeb03450b2020-09-25 05:09:27 +0000226 # Cache directory for `cas` client.
227 'cas_cache_dir',
228 # Parameters passed to `cas` client.
229 'cas_cache_policies',
Takuto Ikutaae391c52020-12-03 08:43:45 +0000230 # Parameters for kvs file used by `cas` client.
231 'cas_kvs',
Takuto Ikuta9a319502019-11-26 07:40:14 +0000232 # Environment variables to set.
233 'env',
234 # Environment variables to mutate with relative directories.
235 # Example: {"ENV_KEY": ['relative', 'paths', 'to', 'prepend']}
236 'env_prefix',
237 # Lowers the task process priority.
238 'lower_priority',
239 # subprocess42.Containment instance. Can be None.
240 'containment',
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000241 ])
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500242
243
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500244def _to_str(s):
245 """Downgrades a unicode instance to str. Pass str through as-is."""
246 if isinstance(s, str):
247 return s
248 # This is technically incorrect, especially on Windows. In theory
249 # sys.getfilesystemencoding() should be used to use the right 'ANSI code
250 # page' on Windows, but that causes other problems, as the character set
251 # is very limited.
252 return s.encode('utf-8')
253
254
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -0500255def _to_unicode(s):
256 """Upgrades a str instance to unicode. Pass unicode through as-is."""
Takuto Ikuta95459dd2019-10-29 12:39:47 +0000257 if isinstance(s, six.text_type) or s is None:
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -0500258 return s
259 return s.decode('utf-8')
260
261
maruel03e11842016-07-14 10:50:16 -0700262def make_temp_dir(prefix, root_dir):
263 """Returns a new unique temporary directory."""
Takuto Ikuta6e2ff962019-10-29 12:35:27 +0000264 return six.text_type(tempfile.mkdtemp(prefix=prefix, dir=root_dir))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +0000265
266
vadimsh9c54b2c2017-07-25 14:08:29 -0700267@contextlib.contextmanager
268def set_luci_context_account(account, tmp_dir):
269 """Sets LUCI_CONTEXT account to be used by the task.
270
271 If 'account' is None or '', does nothing at all. This happens when
272 run_isolated.py is called without '--switch-to-account' flag. In this case,
273 if run_isolated.py is running in some LUCI_CONTEXT environment, the task will
Takuto Ikuta33e2ff32019-09-30 12:44:03 +0000274 just inherit whatever account is already set. This may happen if users invoke
vadimsh9c54b2c2017-07-25 14:08:29 -0700275 run_isolated.py explicitly from their code.
276
277 If the requested account is not defined in the context, switches to
278 non-authenticated access. This happens for Swarming tasks that don't use
279 'task' service accounts.
280
281 If not using LUCI_CONTEXT-based auth, does nothing.
282 If already running as requested account, does nothing.
283 """
284 if not account:
285 # Not actually switching.
286 yield
287 return
288
289 local_auth = luci_context.read('local_auth')
290 if not local_auth:
291 # Not using LUCI_CONTEXT auth at all.
292 yield
293 return
294
295 # See LUCI_CONTEXT.md for the format of 'local_auth'.
296 if local_auth.get('default_account_id') == account:
297 # Already set, no need to switch.
298 yield
299 return
300
301 available = {a['id'] for a in local_auth.get('accounts') or []}
302 if account in available:
303 logging.info('Switching default LUCI_CONTEXT account to %r', account)
304 local_auth['default_account_id'] = account
305 else:
306 logging.warning(
307 'Requested LUCI_CONTEXT account %r is not available (have only %r), '
308 'disabling authentication', account, sorted(available))
309 local_auth.pop('default_account_id', None)
310
311 with luci_context.write(_tmpdir=tmp_dir, local_auth=local_auth):
312 yield
313
314
nodir90bc8dc2016-06-15 13:35:21 -0700315def process_command(command, out_dir, bot_file):
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000316 """Replaces parameters in a command line.
nodirbe642ff2016-06-09 15:51:51 -0700317
318 Raises:
319 ValueError if a parameter is requested in |command| but its value is not
320 provided.
321 """
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000322 return [replace_parameters(arg, out_dir, bot_file) for arg in command]
323
324
325def replace_parameters(arg, out_dir, bot_file):
326 """Replaces parameter tokens with appropriate values in a string.
327
328 Raises:
329 ValueError if a parameter is requested in |arg| but its value is not
330 provided.
331 """
332 arg = arg.replace(EXECUTABLE_SUFFIX_PARAMETER, cipd.EXECUTABLE_SUFFIX)
333 replace_slash = False
334 if ISOLATED_OUTDIR_PARAMETER in arg:
335 if not out_dir:
336 raise ValueError(
337 'output directory is requested in command or env var, but not '
338 'provided; please specify one')
339 arg = arg.replace(ISOLATED_OUTDIR_PARAMETER, out_dir)
340 replace_slash = True
341 if SWARMING_BOT_FILE_PARAMETER in arg:
342 if bot_file:
343 arg = arg.replace(SWARMING_BOT_FILE_PARAMETER, bot_file)
nodirbe642ff2016-06-09 15:51:51 -0700344 replace_slash = True
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000345 else:
346 logging.warning('SWARMING_BOT_FILE_PARAMETER found in command or env '
347 'var, but no bot_file specified. Leaving parameter '
348 'unchanged.')
349 if replace_slash:
350 # Replace slashes only if parameters are present
351 # because of arguments like '${ISOLATED_OUTDIR}/foo/bar'
352 arg = arg.replace('/', os.sep)
353 return arg
maruela9cfd6f2015-09-15 11:03:15 -0700354
355
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000356
357def get_command_env(tmp_dir, cipd_info, run_dir, env, env_prefixes, out_dir,
358 bot_file):
vadimsh232f5a82017-01-20 19:23:44 -0800359 """Returns full OS environment to run a command in.
360
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800361 Sets up TEMP, puts directory with cipd binary in front of PATH, exposes
362 CIPD_CACHE_DIR env var, and installs all env_prefixes.
vadimsh232f5a82017-01-20 19:23:44 -0800363
364 Args:
365 tmp_dir: temp directory.
366 cipd_info: CipdInfo object is cipd client is used, None if not.
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500367 run_dir: The root directory the isolated tree is mapped in.
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500368 env: environment variables to use
Robert Iannuccibf5f84c2017-11-22 12:56:50 -0800369 env_prefixes: {"ENV_KEY": ['cwd', 'relative', 'paths', 'to', 'prepend']}
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000370 out_dir: Isolated output directory. Required to be != None if any of the
371 env vars contain ISOLATED_OUTDIR_PARAMETER.
372 bot_file: Required to be != None if any of the env vars contain
373 SWARMING_BOT_FILE_PARAMETER.
vadimsh232f5a82017-01-20 19:23:44 -0800374 """
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500375 out = os.environ.copy()
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +0000376 for k, v in env.items():
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500377 if not v:
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500378 out.pop(k, None)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500379 else:
Roberto Carrillo71ade6d2018-10-08 22:30:24 +0000380 out[k] = replace_parameters(v, out_dir, bot_file)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500381
382 if cipd_info:
383 bin_dir = os.path.dirname(cipd_info.client.binary_path)
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500384 out['PATH'] = '%s%s%s' % (_to_str(bin_dir), os.pathsep, out['PATH'])
385 out['CIPD_CACHE_DIR'] = _to_str(cipd_info.cache_dir)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500386
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +0000387 for key, paths in env_prefixes.items():
Marc-Antoine Ruel9ec1e9f2017-12-20 16:36:54 -0500388 assert isinstance(paths, list), paths
389 paths = [os.path.normpath(os.path.join(run_dir, p)) for p in paths]
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500390 cur = out.get(key)
391 if cur:
392 paths.append(cur)
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -0500393 out[key] = _to_str(os.path.pathsep.join(paths))
vadimsh232f5a82017-01-20 19:23:44 -0800394
Marc-Antoine Ruelefb30b12018-07-25 18:34:36 +0000395 tmp_dir = _to_str(tmp_dir)
396 # pylint: disable=line-too-long
397 # * python respects $TMPDIR, $TEMP, and $TMP in this order, regardless of
398 # platform. So $TMPDIR must be set on all platforms.
399 # https://github.com/python/cpython/blob/2.7/Lib/tempfile.py#L155
400 out['TMPDIR'] = tmp_dir
401 if sys.platform == 'win32':
402 # * chromium's base utils uses GetTempPath().
403 # https://cs.chromium.org/chromium/src/base/files/file_util_win.cc?q=GetTempPath
404 # * Go uses GetTempPath().
405 # * GetTempDir() uses %TMP%, then %TEMP%, then other stuff. So %TMP% must be
406 # set.
407 # https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-gettemppathw
408 out['TMP'] = tmp_dir
409 # https://blogs.msdn.microsoft.com/oldnewthing/20150417-00/?p=44213
410 out['TEMP'] = tmp_dir
411 elif sys.platform == 'darwin':
412 # * Chromium uses an hack on macOS before calling into
413 # NSTemporaryDirectory().
414 # https://cs.chromium.org/chromium/src/base/files/file_util_mac.mm?q=GetTempDir
415 # https://developer.apple.com/documentation/foundation/1409211-nstemporarydirectory
416 out['MAC_CHROMIUM_TMPDIR'] = tmp_dir
417 else:
418 # TMPDIR is specified as the POSIX standard envvar for the temp directory.
419 # http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html
420 # * mktemp on linux respects $TMPDIR.
421 # * Chromium respects $TMPDIR on linux.
422 # https://cs.chromium.org/chromium/src/base/files/file_util_posix.cc?q=GetTempDir
423 # * Go uses $TMPDIR.
424 # https://go.googlesource.com/go/+/go1.10.3/src/os/file_unix.go#307
425 pass
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -0500426 return out
vadimsh232f5a82017-01-20 19:23:44 -0800427
428
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000429def run_command(
430 command, cwd, env, hard_timeout, grace_period, lower_priority, containment):
maruel6be7f9e2015-10-01 12:25:30 -0700431 """Runs the command.
432
433 Returns:
434 tuple(process exit code, bool if had a hard timeout)
435 """
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000436 logging.info(
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000437 'run_command(%s, %s, %s, %s, %s, %s)',
438 command, cwd, hard_timeout, grace_period, lower_priority, containment)
marueleb5fbee2015-09-17 13:01:36 -0700439
maruel6be7f9e2015-10-01 12:25:30 -0700440 exit_code = None
441 had_hard_timeout = False
maruela9cfd6f2015-09-15 11:03:15 -0700442 with tools.Profiler('RunTest'):
maruel6be7f9e2015-10-01 12:25:30 -0700443 proc = None
444 had_signal = []
maruela9cfd6f2015-09-15 11:03:15 -0700445 try:
maruel6be7f9e2015-10-01 12:25:30 -0700446 # TODO(maruel): This code is imperfect. It doesn't handle well signals
447 # during the download phase and there's short windows were things can go
448 # wrong.
449 def handler(signum, _frame):
450 if proc and not had_signal:
451 logging.info('Received signal %d', signum)
452 had_signal.append(True)
maruel556d9052015-10-05 11:12:44 -0700453 raise subprocess42.TimeoutExpired(command, None)
maruel6be7f9e2015-10-01 12:25:30 -0700454
Marc-Antoine Ruel30b80fe2019-02-08 13:51:31 +0000455 proc = subprocess42.Popen(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +0000456 command, cwd=cwd, env=env, detached=True, close_fds=True,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +0000457 lower_priority=lower_priority, containment=containment)
maruel6be7f9e2015-10-01 12:25:30 -0700458 with subprocess42.set_signal_handler(subprocess42.STOP_SIGNALS, handler):
459 try:
John Budorickc398f092019-06-10 22:49:44 +0000460 exit_code = proc.wait(hard_timeout or None)
maruel6be7f9e2015-10-01 12:25:30 -0700461 except subprocess42.TimeoutExpired:
462 if not had_signal:
463 logging.warning('Hard timeout')
464 had_hard_timeout = True
465 logging.warning('Sending SIGTERM')
466 proc.terminate()
467
Takuto Ikuta684f7912020-09-29 07:49:49 +0000468 kill_sent = False
maruel6be7f9e2015-10-01 12:25:30 -0700469 # Ignore signals in grace period. Forcibly give the grace period to the
470 # child process.
471 if exit_code is None:
472 ignore = lambda *_: None
473 with subprocess42.set_signal_handler(subprocess42.STOP_SIGNALS, ignore):
474 try:
475 exit_code = proc.wait(grace_period or None)
476 except subprocess42.TimeoutExpired:
477 # Now kill for real. The user can distinguish between the
478 # following states:
479 # - signal but process exited within grace period,
480 # hard_timed_out will be set but the process exit code will be
481 # script provided.
482 # - processed exited late, exit code will be -9 on posix.
483 logging.warning('Grace exhausted; sending SIGKILL')
484 proc.kill()
Takuto Ikuta684f7912020-09-29 07:49:49 +0000485 kill_sent = True
martiniss5c8043e2017-08-01 17:09:43 -0700486 logging.info('Waiting for process exit')
maruel6be7f9e2015-10-01 12:25:30 -0700487 exit_code = proc.wait()
Takuto Ikuta684f7912020-09-29 07:49:49 +0000488
489 # the process group / job object may be dangling so if we didn't kill
490 # it already, give it a poke now.
491 if not kill_sent:
492 proc.kill()
Takuto Ikutaeccf0862020-03-19 03:05:55 +0000493 except OSError as e:
maruela9cfd6f2015-09-15 11:03:15 -0700494 # This is not considered to be an internal error. The executable simply
495 # does not exit.
maruela72f46e2016-02-24 11:05:45 -0800496 sys.stderr.write(
tikuta2d678212019-09-23 23:12:08 +0000497 '<The executable does not exist, a dependent library is missing or '
498 'the command line is too long>\n'
499 '<Check for missing .so/.dll in the .isolate or GN file or length of '
500 'command line args>\n'
Takuto Ikutaeccf0862020-03-19 03:05:55 +0000501 '<Command: %s, Exception: %s>\n' % (command, e))
maruela72f46e2016-02-24 11:05:45 -0800502 if os.environ.get('SWARMING_TASK_ID'):
503 # Give an additional hint when running as a swarming task.
504 sys.stderr.write(
505 '<See the task\'s page for commands to help diagnose this issue '
506 'by reproducing the task locally>\n')
maruela9cfd6f2015-09-15 11:03:15 -0700507 exit_code = 1
508 logging.info(
509 'Command finished with exit code %d (%s)',
510 exit_code, hex(0xffffffff & exit_code))
maruel6be7f9e2015-10-01 12:25:30 -0700511 return exit_code, had_hard_timeout
maruela9cfd6f2015-09-15 11:03:15 -0700512
513
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000514def _run_go_cmd_and_wait(cmd):
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000515 """
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000516 Runs an external Go command, `isolated` or `cas`, and wait for its completion.
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000517
518 While this is a generic function to launch a subprocess, it has logic that
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000519 is specific to Go `isolated` and `cas` for waiting and logging.
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000520
521 Returns:
522 The subprocess object
523 """
Ye Kuang3c40e9f2020-07-28 13:15:25 +0000524 cmd_str = ' '.join(cmd)
Ye Kuangc1d800f2020-07-28 10:14:55 +0000525 try:
526 proc = subprocess42.Popen(cmd)
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000527
Ye Kuangc1d800f2020-07-28 10:14:55 +0000528 exceeded_max_timeout = True
529 check_period_sec = 30
530 max_checks = 100
531 # max timeout = max_checks * check_period_sec = 50 minutes
532 for i in range(max_checks):
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000533 # This is to prevent I/O timeout error during setup.
Ye Kuangc1d800f2020-07-28 10:14:55 +0000534 try:
535 retcode = proc.wait(check_period_sec)
536 if retcode != 0:
537 raise ValueError("retcode is not 0: %s (cmd=%s)" % (retcode, cmd_str))
538 exceeded_max_timeout = False
539 break
540 except subprocess42.TimeoutExpired:
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000541 print('still running (after %d seconds)' % ((i + 1) * check_period_sec))
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000542
Ye Kuangc1d800f2020-07-28 10:14:55 +0000543 if exceeded_max_timeout:
544 proc.terminate()
545 try:
546 proc.wait(check_period_sec)
547 except subprocess42.TimeoutExpired:
548 logging.exception(
549 "failed to terminate? timeout happened after %d seconds",
550 check_period_sec)
551 proc.kill()
552 proc.wait()
553 # Raise unconditionally, because |proc| was forcefully terminated.
554 raise ValueError("timedout after %d seconds (cmd=%s)" %
555 (check_period_sec * max_checks, cmd_str))
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000556
Ye Kuangc1d800f2020-07-28 10:14:55 +0000557 return proc
558 except Exception:
559 logging.exception('Failed to run Go cmd %s', cmd_str)
560 raise
Ye Kuangc0cf9ca2020-07-16 08:56:51 +0000561
562
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000563def _fetch_and_map_with_cas(cas_client, digest, instance, output_dir, cache_dir,
Takuto Ikutaae391c52020-12-03 08:43:45 +0000564 policies, kvs_file):
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000565 """
566 Fetches a CAS tree using cas client, create the tree and returns download
567 stats.
568 """
569
Takuto Ikuta34a86c52020-10-13 05:30:57 +0000570 # TODO(crbug.com/chrome-operations/49):
571 # remove this after isolate to RBE-CAS migration.
572 _CAS_EMPTY_DIR_DIGEST = (
573 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0')
574 if digest == _CAS_EMPTY_DIR_DIGEST:
575 return {
576 'duration': 0.0,
577 'items_cold': '',
578 'items_hot': '',
579 }
580
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000581 start = time.time()
582 result_json_handle, result_json_path = tempfile.mkstemp(
583 prefix=u'fetch-and-map-result-', suffix=u'.json')
584 os.close(result_json_handle)
585 try:
586 cmd = [
587 cas_client,
588 'download',
589 '-digest',
590 digest,
591 '-cas-instance',
592 instance,
593 # flags for cache.
594 '-cache-dir',
595 cache_dir,
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000596 '-cache-max-size',
597 str(policies.max_cache_size),
598 '-cache-min-free-space',
599 str(policies.min_free_space),
600 # flags for output.
601 '-dir',
602 output_dir,
603 '-dump-stats-json',
604 result_json_path,
605 ]
Takuto Ikutaae391c52020-12-03 08:43:45 +0000606
607 if kvs_file:
608 cmd.extend(['-kvs-file', kvs_file])
609
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000610 _run_go_cmd_and_wait(cmd)
611
612 with open(result_json_path) as json_file:
613 result_json = json.load(json_file)
614
615 return {
616 'duration': time.time() - start,
617 'items_cold': result_json['items_cold'],
618 'items_hot': result_json['items_hot'],
619 }
620 finally:
621 fs.remove(result_json_path)
622
623
624def _fetch_and_map_with_go_isolated(isolated_hash, storage, outdir,
625 go_cache_dir, policies, isolated_client):
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000626 """
627 Fetches an isolated tree using go client, create the tree and returns
Takuto Ikuta57219f42020-11-02 07:35:36 +0000628 stats.
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000629 """
630 start = time.time()
631 server_ref = storage.server_ref
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000632 result_json_handle, result_json_path = tempfile.mkstemp(
633 prefix=u'fetch-and-map-result-', suffix=u'.json')
634 os.close(result_json_handle)
635 try:
Ye Kuanga98764c2020-04-09 03:17:37 +0000636 cmd = [
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000637 isolated_client,
638 'download',
639 '-isolate-server',
640 server_ref.url,
641 '-namespace',
642 server_ref.namespace,
643 '-isolated',
644 isolated_hash,
645
646 # flags for cache
647 '-cache-dir',
Takuto Ikuta057c5342019-12-03 04:05:05 +0000648 go_cache_dir,
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000649 '-cache-max-items',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000650 str(policies.max_items),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000651 '-cache-max-size',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000652 str(policies.max_cache_size),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000653 '-cache-min-free-space',
Takuto Ikuta50bc0552019-12-03 03:26:46 +0000654 str(policies.min_free_space),
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000655
656 # flags for output
657 '-output-dir',
658 outdir,
659 '-fetch-and-map-result-json',
660 result_json_path,
Ye Kuanga98764c2020-04-09 03:17:37 +0000661 ]
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000662 _run_go_cmd_and_wait(cmd)
Takuto Ikuta3153e3b2020-02-18 06:11:47 +0000663
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000664 with open(result_json_path) as json_file:
665 result_json = json.load(json_file)
666
Takuto Ikuta57219f42020-11-02 07:35:36 +0000667 return {
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000668 'duration': time.time() - start,
669 'items_cold': result_json['items_cold'],
670 'items_hot': result_json['items_hot'],
Ye Kuang65a1de52020-10-16 08:31:16 +0000671 'initial_number_items': result_json['initial_number_items'],
672 'initial_size': result_json['initial_size'],
Takuto Ikutad03ffcc2019-12-02 01:04:23 +0000673 }
674 finally:
675 fs.remove(result_json_path)
676
677
678# TODO(crbug.com/932396): remove this function.
Takuto Ikuta16fac4b2019-12-09 04:57:18 +0000679def fetch_and_map(isolated_hash, storage, cache, outdir):
Takuto Ikuta57219f42020-11-02 07:35:36 +0000680 """Fetches an isolated tree, create the tree and returns stats."""
nodir6f801882016-04-29 14:41:50 -0700681 start = time.time()
Takuto Ikuta57219f42020-11-02 07:35:36 +0000682 isolateserver.fetch_isolated(
nodir6f801882016-04-29 14:41:50 -0700683 isolated_hash=isolated_hash,
684 storage=storage,
685 cache=cache,
maruel4409e302016-07-19 14:25:51 -0700686 outdir=outdir,
Takuto Ikuta16fac4b2019-12-09 04:57:18 +0000687 use_symlinks=False)
Takuto Ikuta2b9640e2019-06-19 00:53:23 +0000688 hot = (collections.Counter(cache.used) -
689 collections.Counter(cache.added)).elements()
Takuto Ikuta57219f42020-11-02 07:35:36 +0000690 return {
Takuto Ikuta630f99d2020-07-02 12:59:35 +0000691 'duration': time.time() - start,
692 'items_cold': base64.b64encode(large.pack(sorted(cache.added))).decode(),
693 'items_hot': base64.b64encode(large.pack(sorted(hot))).decode(),
nodir6f801882016-04-29 14:41:50 -0700694 }
695
696
aludwin0a8e17d2016-10-27 15:57:39 -0700697def link_outputs_to_outdir(run_dir, out_dir, outputs):
698 """Links any named outputs to out_dir so they can be uploaded.
699
700 Raises an error if the file already exists in that directory.
701 """
702 if not outputs:
703 return
704 isolateserver.create_directories(out_dir, outputs)
705 for o in outputs:
Sadaf Matinkhoo10743a62018-03-29 16:28:58 -0400706 copy_recursively(os.path.join(run_dir, o), os.path.join(out_dir, o))
707
708
709def copy_recursively(src, dst):
710 """Efficiently copies a file or directory from src_dir to dst_dir.
711
712 `item` may be a file, directory, or a symlink to a file or directory.
713 All symlinks are replaced with their targets, so the resulting
714 directory structure in dst_dir will never have any symlinks.
715
716 To increase speed, copy_recursively hardlinks individual files into the
717 (newly created) directory structure if possible, unlike Python's
718 shutil.copytree().
719 """
720 orig_src = src
721 try:
722 # Replace symlinks with their final target.
723 while fs.islink(src):
724 res = fs.readlink(src)
725 src = os.path.join(os.path.dirname(src), res)
726 # TODO(sadafm): Explicitly handle cyclic symlinks.
727
728 # Note that fs.isfile (which is a wrapper around os.path.isfile) throws
729 # an exception if src does not exist. A warning will be logged in that case.
730 if fs.isfile(src):
731 file_path.link_file(dst, src, file_path.HARDLINK_WITH_FALLBACK)
732 return
733
734 if not fs.exists(dst):
735 os.makedirs(dst)
736
737 for child in fs.listdir(src):
738 copy_recursively(os.path.join(src, child), os.path.join(dst, child))
739
740 except OSError as e:
741 if e.errno == errno.ENOENT:
742 logging.warning('Path %s does not exist or %s is a broken symlink',
743 src, orig_src)
744 else:
745 logging.info("Couldn't collect output file %s: %s", src, e)
aludwin0a8e17d2016-10-27 15:57:39 -0700746
747
Ye Kuangfb0bad62020-07-28 08:07:25 +0000748def _upload_with_py(storage, out_dir):
749
750 def process_stats(f_st):
751 st = sorted(i.size for i in f_st)
752 return base64.b64encode(large.pack(st)).decode()
753
754 try:
755 results, f_cold, f_hot = isolateserver.archive_files_to_storage(
756 storage, [out_dir], None, verify_push=True)
757
758 isolated = list(results.values())[0]
759 cold = process_stats(f_cold)
760 hot = process_stats(f_hot)
761 return isolated, cold, hot
762
763 except isolateserver.Aborted:
764 # This happens when a signal SIGTERM was received while uploading data.
765 # There is 2 causes:
766 # - The task was too slow and was about to be killed anyway due to
767 # exceeding the hard timeout.
768 # - The amount of data uploaded back is very large and took too much
769 # time to archive.
770 sys.stderr.write('Received SIGTERM while uploading')
771 # Re-raise, so it will be treated as an internal failure.
772 raise
773
774
775def _upload_with_go(storage, outdir, isolated_client):
776 """
777 Uploads results back using the Go `isolated` CLI.
778 """
779 server_ref = storage.server_ref
780 isolated_handle, isolated_path = tempfile.mkstemp(
781 prefix=u'isolated-hash-', suffix=u'.txt')
782 stats_json_handle, stats_json_path = tempfile.mkstemp(
783 prefix=u'dump-stats-', suffix=u'.json')
784 os.close(isolated_handle)
785 os.close(stats_json_handle)
786 try:
787 cmd = [
788 isolated_client,
789 'archive',
790 '-isolate-server',
791 server_ref.url,
792 '-namespace',
793 server_ref.namespace,
794 '-dirs',
795 # Format: <working directory>:<relative path to dir>
796 outdir + ':',
797
798 # output
799 '-dump-hash',
800 isolated_path,
801 '-dump-stats-json',
802 stats_json_path,
Ye Kuangbc4e8402020-07-29 09:54:30 +0000803 '-quiet',
Ye Kuangfb0bad62020-07-28 08:07:25 +0000804 ]
Ye Kuang0023dc52020-08-04 05:28:41 +0000805 # Will do exponential backoff, e.g. 10, 20, 40...
806 # This mitigates https://crbug.com/1094369, where there is a data race on
807 # the uploaded files.
808 backoff = 10
Takuto Ikutae0bfec72020-08-28 02:52:52 +0000809 started = time.time()
Ye Kuang0023dc52020-08-04 05:28:41 +0000810 while True:
811 try:
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000812 _run_go_cmd_and_wait(cmd)
Ye Kuang0023dc52020-08-04 05:28:41 +0000813 break
814 except Exception:
Takuto Ikutae0bfec72020-08-28 02:52:52 +0000815 if time.time() > started + 60 * 2:
816 # This is to not wait task having leaked process long time.
Ye Kuang0023dc52020-08-04 05:28:41 +0000817 raise
818
819 on_error.report('error before %d second backoff' % backoff)
820 logging.exception(
Junji Watanabe4b890ef2020-09-16 01:43:27 +0000821 '_run_go_cmd_and_wait() failed, will retry after %d seconds',
Ye Kuang0023dc52020-08-04 05:28:41 +0000822 backoff)
823 time.sleep(backoff)
824 backoff *= 2
Ye Kuangfb0bad62020-07-28 08:07:25 +0000825
826 with open(isolated_path) as isol_file:
827 isolated = isol_file.read()
828 with open(stats_json_path) as json_file:
829 stats_json = json.load(json_file)
830
831 return isolated, stats_json['items_cold'], stats_json['items_hot']
832 finally:
833 fs.remove(isolated_path)
834 fs.remove(stats_json_path)
835
836
Ye Kuangbc4e8402020-07-29 09:54:30 +0000837def upload_out_dir(storage, out_dir, go_isolated_client):
838 """Uploads the results in |out_dir| back, if there is any.
maruela9cfd6f2015-09-15 11:03:15 -0700839
840 Returns:
Ye Kuangbc4e8402020-07-29 09:54:30 +0000841 tuple(outputs_ref, stats)
maruel064c0a32016-04-05 11:47:15 -0700842 - outputs_ref: a dict referring to the results archived back to the isolated
843 server, if applicable.
nodir6f801882016-04-29 14:41:50 -0700844 - stats: uploading stats.
maruela9cfd6f2015-09-15 11:03:15 -0700845 """
maruela9cfd6f2015-09-15 11:03:15 -0700846 # Upload out_dir and generate a .isolated file out of this directory. It is
847 # only done if files were written in the directory.
848 outputs_ref = None
Ye Kuangfb0bad62020-07-28 08:07:25 +0000849 cold = ''
850 hot = ''
nodir6f801882016-04-29 14:41:50 -0700851 start = time.time()
852
maruel12e30012015-10-09 11:55:35 -0700853 if fs.isdir(out_dir) and fs.listdir(out_dir):
maruela9cfd6f2015-09-15 11:03:15 -0700854 with tools.Profiler('ArchiveOutput'):
Ye Kuangfb0bad62020-07-28 08:07:25 +0000855 isolated = None
Ye Kuang72e6fe82020-08-05 06:30:04 +0000856 if _USE_GO_ISOLATED_TO_UPLOAD and go_isolated_client is not None:
Ye Kuangfb0bad62020-07-28 08:07:25 +0000857 isolated, cold, hot = _upload_with_go(storage, out_dir,
858 go_isolated_client)
Ye Kuang72e6fe82020-08-05 06:30:04 +0000859 else:
860 isolated, cold, hot = _upload_with_py(storage, out_dir)
Ye Kuangfb0bad62020-07-28 08:07:25 +0000861 outputs_ref = {
862 'isolated': isolated,
863 'isolatedserver': storage.server_ref.url,
864 'namespace': storage.server_ref.namespace,
865 }
nodir6f801882016-04-29 14:41:50 -0700866
nodir6f801882016-04-29 14:41:50 -0700867 stats = {
Takuto Ikuta630f99d2020-07-02 12:59:35 +0000868 'duration': time.time() - start,
Ye Kuangfb0bad62020-07-28 08:07:25 +0000869 'items_cold': cold,
870 'items_hot': hot,
nodir6f801882016-04-29 14:41:50 -0700871 }
Ye Kuangbc4e8402020-07-29 09:54:30 +0000872 return outputs_ref, stats
maruela9cfd6f2015-09-15 11:03:15 -0700873
874
Junji Watanabe1adba7b2020-09-18 07:03:58 +0000875def upload_outdir_with_cas(cas_client, cas_instance, outdir):
876 """Uploads the results in |outdir|, if there is any.
877
878 Returns:
879 tuple(root_digest, stats)
880 - root_digest: a digest of the output directory.
881 - stats: uploading stats.
882 """
883 digest_file_handle, digest_path = tempfile.mkstemp(
884 prefix=u'cas-digest', suffix=u'.txt')
885 os.close(digest_file_handle)
886 stats_json_handle, stats_json_path = tempfile.mkstemp(
887 prefix=u'upload-stats', suffix=u'.json')
888 os.close(stats_json_handle)
889
890 try:
891 cmd = [
892 cas_client,
893 'archive',
894 '-cas-instance',
895 cas_instance,
896 '-paths',
897 # Format: <working directory>:<relative path to dir>
898 outdir + ':',
899 # output
900 '-dump-digest',
901 digest_path,
902 '-dump-stats-json',
903 stats_json_path,
904 ]
905
906 start = time.time()
907
908 _run_go_cmd_and_wait(cmd)
909
910 with open(digest_path) as digest_file:
911 digest = digest_file.read()
Junji Watanabec208b302020-09-25 09:18:27 +0000912 h, s = digest.split('/')
913 cas_output_root = {
914 'cas_instance': cas_instance,
915 'digest': {
916 'hash': h,
917 'size_bytes': int(s)
918 }
919 }
Junji Watanabe1adba7b2020-09-18 07:03:58 +0000920 with open(stats_json_path) as stats_file:
921 stats = json.load(stats_file)
922
923 stats['duration'] = time.time() - start
924
Junji Watanabec208b302020-09-25 09:18:27 +0000925 return cas_output_root, stats
Junji Watanabe1adba7b2020-09-18 07:03:58 +0000926 finally:
927 fs.remove(digest_path)
928 fs.remove(stats_json_path)
929
930
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500931def map_and_run(data, constant_run_path):
nodir55be77b2016-05-03 09:39:57 -0700932 """Runs a command with optional isolated input/output.
933
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500934 Arguments:
935 - data: TaskData instance.
936 - constant_run_path: TODO
nodir55be77b2016-05-03 09:39:57 -0700937
938 Returns metadata about the result.
939 """
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +0000940
941 if data.isolate_cache:
942 download_stats = {
943 #'duration': 0.,
944 'initial_number_items': len(data.isolate_cache),
945 'initial_size': data.isolate_cache.total_size,
946 #'items_cold': '<large.pack()>',
947 #'items_hot': '<large.pack()>',
948 }
949 else:
950 # TODO(tikuta): take stats from state.json in this case too.
951 download_stats = {}
952
maruela9cfd6f2015-09-15 11:03:15 -0700953 result = {
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000954 'duration': None,
955 'exit_code': None,
956 'had_hard_timeout': False,
957 'internal_failure': 'run_isolated did not complete properly',
958 'stats': {
959 #'cipd': {
960 # 'duration': 0.,
961 # 'get_client_duration': 0.,
962 #},
963 'isolated': {
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +0000964 'download': download_stats,
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000965 #'upload': {
966 # 'duration': 0.,
967 # 'items_cold': '<large.pack()>',
968 # 'items_hot': '<large.pack()>',
969 #},
970 },
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +0000971 },
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000972 #'cipd_pins': {
973 # 'packages': [
974 # {'package_name': ..., 'version': ..., 'path': ...},
975 # ...
976 # ],
977 # 'client_package': {'package_name': ..., 'version': ...},
978 #},
979 'outputs_ref': None,
Junji Watanabe54925c32020-09-08 00:56:18 +0000980 'cas_output_root': None,
Takuto Ikuta5ed62ad2019-09-26 09:16:00 +0000981 'version': 5,
maruela9cfd6f2015-09-15 11:03:15 -0700982 }
nodirbe642ff2016-06-09 15:51:51 -0700983
Takuto Ikutad46ea762020-10-07 05:43:22 +0000984 assert os.path.isabs(data.root_dir), ("data.root_dir is not abs path: %s" %
985 data.root_dir)
986 file_path.ensure_tree(data.root_dir, 0o700)
987
maruele2f2cb82016-07-13 14:41:03 -0700988 # See comment for these constants.
maruelcffa0542017-04-07 08:39:20 -0700989 # TODO(maruel): This is not obvious. Change this to become an error once we
990 # make the constant_run_path an exposed flag.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500991 if constant_run_path and data.root_dir:
992 run_dir = os.path.join(data.root_dir, ISOLATED_RUN_DIR)
maruel5c4eed82017-05-26 05:33:40 -0700993 if os.path.isdir(run_dir):
994 file_path.rmtree(run_dir)
Lei Leife202df2019-06-11 17:33:34 +0000995 os.mkdir(run_dir, 0o700)
maruelcffa0542017-04-07 08:39:20 -0700996 else:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -0500997 run_dir = make_temp_dir(ISOLATED_RUN_DIR, data.root_dir)
Junji Watanabe1adba7b2020-09-18 07:03:58 +0000998
999 # True if CAS is used for download/upload files.
1000 use_cas = bool(data.cas_digest)
1001
maruel03e11842016-07-14 10:50:16 -07001002 # storage should be normally set but don't crash if it is not. This can happen
1003 # as Swarming task can run without an isolate server.
Junji Watanabe1adba7b2020-09-18 07:03:58 +00001004 out_dir = None
1005 if data.storage or use_cas:
1006 out_dir = make_temp_dir(ISOLATED_OUT_DIR, data.root_dir)
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001007 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, data.root_dir)
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001008 isolated_client_dir = make_temp_dir(ISOLATED_CLIENT_DIR, data.root_dir)
nodir55be77b2016-05-03 09:39:57 -07001009 cwd = run_dir
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001010 if data.relative_cwd:
1011 cwd = os.path.normpath(os.path.join(cwd, data.relative_cwd))
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001012 command = data.command
Ye Kuangfb0bad62020-07-28 08:07:25 +00001013 go_isolated_client = None
1014 if data.use_go_isolated:
1015 go_isolated_client = os.path.join(isolated_client_dir,
1016 'isolated' + cipd.EXECUTABLE_SUFFIX)
Junji Watanabe1adba7b2020-09-18 07:03:58 +00001017
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001018 cas_client = None
1019 cas_client_dir = make_temp_dir(_CAS_CLIENT_DIR, data.root_dir)
Junji Watanabe1adba7b2020-09-18 07:03:58 +00001020 if use_cas:
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001021 cas_client = os.path.join(cas_client_dir, 'cas' + cipd.EXECUTABLE_SUFFIX)
1022
nodir55be77b2016-05-03 09:39:57 -07001023 try:
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001024 with data.install_packages_fn(run_dir, isolated_client_dir,
1025 cas_client_dir) as cipd_info:
vadimsh232f5a82017-01-20 19:23:44 -08001026 if cipd_info:
1027 result['stats']['cipd'] = cipd_info.stats
1028 result['cipd_pins'] = cipd_info.pins
nodir90bc8dc2016-06-15 13:35:21 -07001029
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001030 isolated_stats = result['stats'].setdefault('isolated', {})
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001031 if data.isolated_hash:
Takuto Ikutad03ffcc2019-12-02 01:04:23 +00001032 if data.use_go_isolated:
Takuto Ikuta57219f42020-11-02 07:35:36 +00001033 stats = _fetch_and_map_with_go_isolated(
Takuto Ikuta90397ca2020-01-08 10:07:55 +00001034 isolated_hash=data.isolated_hash,
1035 storage=data.storage,
Takuto Ikuta90397ca2020-01-08 10:07:55 +00001036 outdir=run_dir,
1037 go_cache_dir=data.go_cache_dir,
Takuto Ikuta879788c2020-01-10 08:00:26 +00001038 policies=data.go_cache_policies,
Ye Kuangfb0bad62020-07-28 08:07:25 +00001039 isolated_client=go_isolated_client)
Takuto Ikuta90397ca2020-01-08 10:07:55 +00001040 else:
Takuto Ikuta57219f42020-11-02 07:35:36 +00001041 stats = fetch_and_map(
Takuto Ikutad03ffcc2019-12-02 01:04:23 +00001042 isolated_hash=data.isolated_hash,
1043 storage=data.storage,
1044 cache=data.isolate_cache,
Takuto Ikuta16fac4b2019-12-09 04:57:18 +00001045 outdir=run_dir)
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +00001046 isolated_stats['download'].update(stats)
Takuto Ikutab58dbd12020-06-05 09:29:14 +00001047
Junji Watanabe54925c32020-09-08 00:56:18 +00001048 elif data.cas_digest:
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001049 stats = _fetch_and_map_with_cas(
1050 cas_client=cas_client,
1051 digest=data.cas_digest,
1052 instance=data.cas_instance,
1053 output_dir=run_dir,
Junji Watanabeb03450b2020-09-25 05:09:27 +00001054 cache_dir=data.cas_cache_dir,
Takuto Ikutaae391c52020-12-03 08:43:45 +00001055 policies=data.cas_cache_policies,
1056 kvs_file=data.cas_kvs)
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001057 isolated_stats['download'].update(stats)
Junji Watanabe54925c32020-09-08 00:56:18 +00001058
maruelabec63c2017-04-26 11:53:24 -07001059 if not command:
1060 # Handle this as a task failure, not an internal failure.
1061 sys.stderr.write(
1062 '<No command was specified!>\n'
1063 '<Please secify a command when triggering your Swarming task>\n')
1064 result['exit_code'] = 1
1065 return result
nodirbe642ff2016-06-09 15:51:51 -07001066
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001067 if not cwd.startswith(run_dir):
1068 # Handle this as a task failure, not an internal failure. This is a
1069 # 'last chance' way to gate against directory escape.
1070 sys.stderr.write('<Relative CWD is outside of run directory!>\n')
1071 result['exit_code'] = 1
1072 return result
1073
1074 if not os.path.isdir(cwd):
1075 # Accepts relative_cwd that does not exist.
Lei Leife202df2019-06-11 17:33:34 +00001076 os.makedirs(cwd, 0o700)
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001077
vadimsh232f5a82017-01-20 19:23:44 -08001078 # If we have an explicit list of files to return, make sure their
1079 # directories exist now.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001080 if data.storage and data.outputs:
1081 isolateserver.create_directories(run_dir, data.outputs)
aludwin0a8e17d2016-10-27 15:57:39 -07001082
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001083 with data.install_named_caches(run_dir):
nodird6160682017-02-02 13:03:35 -08001084 sys.stdout.flush()
1085 start = time.time()
1086 try:
vadimsh9c54b2c2017-07-25 14:08:29 -07001087 # Need to switch the default account before 'get_command_env' call,
1088 # so it can grab correct value of LUCI_CONTEXT env var.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001089 with set_luci_context_account(data.switch_to_account, tmp_dir):
1090 env = get_command_env(
Roberto Carrillo71ade6d2018-10-08 22:30:24 +00001091 tmp_dir, cipd_info, run_dir, data.env, data.env_prefix, out_dir,
1092 data.bot_file)
Brian Sheedy7a761172019-08-30 22:55:14 +00001093 command = tools.find_executable(command, env)
Robert Iannucci24ae76a2018-02-26 12:51:18 -08001094 command = process_command(command, out_dir, data.bot_file)
1095 file_path.ensure_command_has_abs_path(command, cwd)
1096
vadimsh9c54b2c2017-07-25 14:08:29 -07001097 result['exit_code'], result['had_hard_timeout'] = run_command(
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +00001098 command, cwd, env, data.hard_timeout, data.grace_period,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001099 data.lower_priority, data.containment)
nodird6160682017-02-02 13:03:35 -08001100 finally:
1101 result['duration'] = max(time.time() - start, 0)
Seth Koehler49139812017-12-19 13:59:33 -05001102
Ye Kuangbc4e8402020-07-29 09:54:30 +00001103 if out_dir:
1104 # Try to link files to the output directory, if specified.
1105 link_outputs_to_outdir(run_dir, out_dir, data.outputs)
1106 isolated_stats = result['stats'].setdefault('isolated', {})
Junji Watanabe1adba7b2020-09-18 07:03:58 +00001107 if use_cas:
1108 result['cas_output_root'], isolated_stats['upload'] = (
1109 upload_outdir_with_cas(cas_client, data.cas_instance, out_dir))
1110 else:
1111 # This could use |go_isolated_client|, so make sure it runs when the
1112 # CIPD package still exists.
1113 result['outputs_ref'], isolated_stats['upload'] = (
1114 upload_out_dir(data.storage, out_dir, go_isolated_client))
Seth Koehler49139812017-12-19 13:59:33 -05001115 # We successfully ran the command, set internal_failure back to
1116 # None (even if the command failed, it's not an internal error).
1117 result['internal_failure'] = None
maruela9cfd6f2015-09-15 11:03:15 -07001118 except Exception as e:
nodir90bc8dc2016-06-15 13:35:21 -07001119 # An internal error occurred. Report accordingly so the swarming task will
1120 # be retried automatically.
maruel12e30012015-10-09 11:55:35 -07001121 logging.exception('internal failure: %s', e)
maruela9cfd6f2015-09-15 11:03:15 -07001122 result['internal_failure'] = str(e)
1123 on_error.report(None)
aludwin0a8e17d2016-10-27 15:57:39 -07001124
1125 # Clean up
maruela9cfd6f2015-09-15 11:03:15 -07001126 finally:
1127 try:
Ye Kuangbc4e8402020-07-29 09:54:30 +00001128 success = True
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001129 if data.leak_temp_dir:
nodir32a1ec12016-10-26 18:34:07 -07001130 success = True
maruela9cfd6f2015-09-15 11:03:15 -07001131 logging.warning(
1132 'Deliberately leaking %s for later examination', run_dir)
marueleb5fbee2015-09-17 13:01:36 -07001133 else:
maruel84537cb2015-10-16 14:21:28 -07001134 # On Windows rmtree(run_dir) call above has a synchronization effect: it
1135 # finishes only when all task child processes terminate (since a running
1136 # process locks *.exe file). Examine out_dir only after that call
1137 # completes (since child processes may write to out_dir too and we need
1138 # to wait for them to finish).
Junji Watanabeb03450b2020-09-25 05:09:27 +00001139 dirs_to_remove = [run_dir, tmp_dir, isolated_client_dir, cas_client_dir]
Ye Kuangbc4e8402020-07-29 09:54:30 +00001140 if out_dir:
1141 dirs_to_remove.append(out_dir)
1142 for directory in dirs_to_remove:
Takuto Ikuta69c0d662019-11-27 01:18:08 +00001143 if not fs.isdir(directory):
1144 continue
Junji Watanabe9cdfff52021-01-08 07:20:35 +00001145 start = time.time()
maruel84537cb2015-10-16 14:21:28 -07001146 try:
Ye Kuangbc4e8402020-07-29 09:54:30 +00001147 success = success and file_path.rmtree(directory)
maruel84537cb2015-10-16 14:21:28 -07001148 except OSError as e:
Takuto Ikuta69c0d662019-11-27 01:18:08 +00001149 logging.error('rmtree(%r) failed: %s', directory, e)
maruel84537cb2015-10-16 14:21:28 -07001150 success = False
Junji Watanabe9cdfff52021-01-08 07:20:35 +00001151 finally:
1152 logging.info('Cleanup: rmtree(%r) took %d seconds', directory,
1153 time.time() - start)
maruel84537cb2015-10-16 14:21:28 -07001154 if not success:
Takuto Ikuta69c0d662019-11-27 01:18:08 +00001155 sys.stderr.write(
1156 OUTLIVING_ZOMBIE_MSG % (directory, data.grace_period))
Takuto Ikutad7d64e12020-07-31 06:18:45 +00001157 subprocess42.check_call(['tasklist.exe', '/V'], stdout=sys.stderr)
maruel84537cb2015-10-16 14:21:28 -07001158 if result['exit_code'] == 0:
1159 result['exit_code'] = 1
maruela9cfd6f2015-09-15 11:03:15 -07001160
maruela9cfd6f2015-09-15 11:03:15 -07001161 if not success and result['exit_code'] == 0:
1162 result['exit_code'] = 1
1163 except Exception as e:
1164 # Swallow any exception in the main finally clause.
nodir9130f072016-05-27 13:59:08 -07001165 if out_dir:
1166 logging.exception('Leaking out_dir %s: %s', out_dir, e)
maruela9cfd6f2015-09-15 11:03:15 -07001167 result['internal_failure'] = str(e)
Takuto Ikutaa9a907b2020-04-17 08:50:50 +00001168 on_error.report(None)
maruela9cfd6f2015-09-15 11:03:15 -07001169 return result
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -05001170
1171
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001172def run_tha_test(data, result_json):
nodir55be77b2016-05-03 09:39:57 -07001173 """Runs an executable and records execution metadata.
1174
nodir55be77b2016-05-03 09:39:57 -07001175 If isolated_hash is specified, downloads the dependencies in the cache,
1176 hardlinks them into a temporary directory and runs the command specified in
1177 the .isolated.
Marc-Antoine Ruel2283ad12014-02-09 11:14:57 -05001178
1179 A temporary directory is created to hold the output files. The content inside
1180 this directory will be uploaded back to |storage| packaged as a .isolated
1181 file.
1182
1183 Arguments:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001184 - data: TaskData instance.
1185 - result_json: File path to dump result metadata into. If set, the process
1186 exit code is always 0 unless an internal error occurred.
maruela9cfd6f2015-09-15 11:03:15 -07001187
1188 Returns:
1189 Process exit code that should be used.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001190 """
maruela76b9ee2015-12-15 06:18:08 -08001191 if result_json:
1192 # Write a json output file right away in case we get killed.
1193 result = {
Junji Watanabe54925c32020-09-08 00:56:18 +00001194 'exit_code': None,
1195 'had_hard_timeout': False,
1196 'internal_failure': 'Was terminated before completion',
1197 'outputs_ref': None,
1198 'cas_output_root': None,
1199 'version': 5,
maruela76b9ee2015-12-15 06:18:08 -08001200 }
1201 tools.write_json(result_json, result, dense=True)
1202
maruela9cfd6f2015-09-15 11:03:15 -07001203 # run_isolated exit code. Depends on if result_json is used or not.
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001204 result = map_and_run(data, True)
maruela9cfd6f2015-09-15 11:03:15 -07001205 logging.info('Result:\n%s', tools.format_json(result, dense=True))
bpastene3ae09522016-06-10 17:12:59 -07001206
maruela9cfd6f2015-09-15 11:03:15 -07001207 if result_json:
maruel05d5a882015-09-21 13:59:02 -07001208 # We've found tests to delete 'work' when quitting, causing an exception
1209 # here. Try to recreate the directory if necessary.
nodire5028a92016-04-29 14:38:21 -07001210 file_path.ensure_tree(os.path.dirname(result_json))
maruela9cfd6f2015-09-15 11:03:15 -07001211 tools.write_json(result_json, result, dense=True)
1212 # Only return 1 if there was an internal error.
1213 return int(bool(result['internal_failure']))
maruel@chromium.org781ccf62013-09-17 19:39:47 +00001214
maruela9cfd6f2015-09-15 11:03:15 -07001215 # Marshall into old-style inline output.
1216 if result['outputs_ref']:
Marc-Antoine Ruel793bff32019-04-18 17:50:48 +00001217 # pylint: disable=unsubscriptable-object
maruela9cfd6f2015-09-15 11:03:15 -07001218 data = {
Junji Watanabe38b28b02020-04-23 10:23:30 +00001219 'hash': result['outputs_ref']['isolated'],
1220 'namespace': result['outputs_ref']['namespace'],
1221 'storage': result['outputs_ref']['isolatedserver'],
maruela9cfd6f2015-09-15 11:03:15 -07001222 }
Marc-Antoine Ruelc44f5722015-01-08 16:10:01 -05001223 sys.stdout.flush()
Junji Watanabe38b28b02020-04-23 10:23:30 +00001224 print('[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
1225 tools.format_json(data, dense=True))
maruelb76604c2015-11-11 11:53:44 -08001226 sys.stdout.flush()
maruela9cfd6f2015-09-15 11:03:15 -07001227 return result['exit_code'] or int(bool(result['internal_failure']))
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001228
1229
iannuccib58d10d2017-03-18 02:00:25 -07001230# Yielded by 'install_client_and_packages'.
vadimsh232f5a82017-01-20 19:23:44 -08001231CipdInfo = collections.namedtuple('CipdInfo', [
1232 'client', # cipd.CipdClient object
1233 'cache_dir', # absolute path to bot-global cipd tag and instance cache
1234 'stats', # dict with stats to return to the server
1235 'pins', # dict with installed cipd pins to return to the server
1236])
1237
1238
1239@contextlib.contextmanager
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001240def noop_install_packages(_run_dir, _isolated_dir, _cas_dir):
iannuccib58d10d2017-03-18 02:00:25 -07001241 """Placeholder for 'install_client_and_packages' if cipd is disabled."""
vadimsh232f5a82017-01-20 19:23:44 -08001242 yield None
1243
1244
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001245def _install_packages(run_dir, cipd_cache_dir, client, packages):
iannuccib58d10d2017-03-18 02:00:25 -07001246 """Calls 'cipd ensure' for packages.
1247
1248 Args:
1249 run_dir (str): root of installation.
1250 cipd_cache_dir (str): the directory to use for the cipd package cache.
1251 client (CipdClient): the cipd client to use
1252 packages: packages to install, list [(path, package_name, version), ...].
iannuccib58d10d2017-03-18 02:00:25 -07001253
1254 Returns: list of pinned packages. Looks like [
1255 {
1256 'path': 'subdirectory',
1257 'package_name': 'resolved/package/name',
1258 'version': 'deadbeef...',
1259 },
1260 ...
1261 ]
1262 """
1263 package_pins = [None]*len(packages)
1264 def insert_pin(path, name, version, idx):
1265 package_pins[idx] = {
1266 'package_name': name,
1267 # swarming deals with 'root' as '.'
1268 'path': path or '.',
1269 'version': version,
1270 }
1271
1272 by_path = collections.defaultdict(list)
1273 for i, (path, name, version) in enumerate(packages):
1274 # cipd deals with 'root' as ''
1275 if path == '.':
1276 path = ''
1277 by_path[path].append((name, version, i))
1278
1279 pins = client.ensure(
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001280 run_dir,
1281 {
1282 subdir: [(name, vers) for name, vers, _ in pkgs
1283 ] for subdir, pkgs in by_path.items()
1284 },
1285 cache_dir=cipd_cache_dir,
iannuccib58d10d2017-03-18 02:00:25 -07001286 )
1287
Marc-Antoine Ruel04903a32019-10-09 21:09:25 +00001288 for subdir, pin_list in sorted(pins.items()):
iannuccib58d10d2017-03-18 02:00:25 -07001289 this_subdir = by_path[subdir]
1290 for i, (name, version) in enumerate(pin_list):
1291 insert_pin(subdir, name, version, this_subdir[i][2])
1292
Robert Iannucci461b30d2017-12-13 11:34:03 -08001293 assert None not in package_pins, (packages, pins, package_pins)
iannuccib58d10d2017-03-18 02:00:25 -07001294
1295 return package_pins
1296
1297
vadimsh232f5a82017-01-20 19:23:44 -08001298@contextlib.contextmanager
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001299def install_client_and_packages(run_dir, packages, service_url,
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001300 client_package_name, client_version, cache_dir,
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001301 isolated_dir, cas_dir):
vadimsh902948e2017-01-20 15:57:32 -08001302 """Bootstraps CIPD client and installs CIPD packages.
iannucci96fcccc2016-08-30 15:52:22 -07001303
vadimsh232f5a82017-01-20 19:23:44 -08001304 Yields CipdClient, stats, client info and pins (as single CipdInfo object).
1305
1306 Pins and the CIPD client info are in the form of:
iannucci96fcccc2016-08-30 15:52:22 -07001307 [
1308 {
1309 "path": path, "package_name": package_name, "version": version,
1310 },
1311 ...
1312 ]
vadimsh902948e2017-01-20 15:57:32 -08001313 (the CIPD client info is a single dictionary instead of a list)
iannucci96fcccc2016-08-30 15:52:22 -07001314
1315 such that they correspond 1:1 to all input package arguments from the command
1316 line. These dictionaries make their all the way back to swarming, where they
1317 become the arguments of CipdPackage.
nodirbe642ff2016-06-09 15:51:51 -07001318
vadimsh902948e2017-01-20 15:57:32 -08001319 If 'packages' list is empty, will bootstrap CIPD client, but won't install
1320 any packages.
1321
1322 The bootstrapped client (regardless whether 'packages' list is empty or not),
vadimsh232f5a82017-01-20 19:23:44 -08001323 will be made available to the task via $PATH.
vadimsh902948e2017-01-20 15:57:32 -08001324
nodirbe642ff2016-06-09 15:51:51 -07001325 Args:
nodir90bc8dc2016-06-15 13:35:21 -07001326 run_dir (str): root of installation.
vadimsh902948e2017-01-20 15:57:32 -08001327 packages: packages to install, list [(path, package_name, version), ...].
nodirbe642ff2016-06-09 15:51:51 -07001328 service_url (str): CIPD server url, e.g.
1329 "https://chrome-infra-packages.appspot.com."
nodir90bc8dc2016-06-15 13:35:21 -07001330 client_package_name (str): CIPD package name of CIPD client.
1331 client_version (str): Version of CIPD client.
nodirbe642ff2016-06-09 15:51:51 -07001332 cache_dir (str): where to keep cache of cipd clients, packages and tags.
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001333 isolated_dir (str): where to download isolated client.
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001334 cas_dir (str): where to download cas client.
nodirbe642ff2016-06-09 15:51:51 -07001335 """
1336 assert cache_dir
nodir90bc8dc2016-06-15 13:35:21 -07001337
nodirbe642ff2016-06-09 15:51:51 -07001338 start = time.time()
nodirbe642ff2016-06-09 15:51:51 -07001339
vadimsh902948e2017-01-20 15:57:32 -08001340 cache_dir = os.path.abspath(cache_dir)
vadimsh232f5a82017-01-20 19:23:44 -08001341 cipd_cache_dir = os.path.join(cache_dir, 'cache') # tag and instance caches
nodir90bc8dc2016-06-15 13:35:21 -07001342 run_dir = os.path.abspath(run_dir)
vadimsh902948e2017-01-20 15:57:32 -08001343 packages = packages or []
nodir90bc8dc2016-06-15 13:35:21 -07001344
nodirbe642ff2016-06-09 15:51:51 -07001345 get_client_start = time.time()
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001346 client_manager = cipd.get_client(cache_dir, service_url, client_package_name,
1347 client_version)
iannucci96fcccc2016-08-30 15:52:22 -07001348
nodirbe642ff2016-06-09 15:51:51 -07001349 with client_manager as client:
1350 get_client_duration = time.time() - get_client_start
nodir90bc8dc2016-06-15 13:35:21 -07001351
iannuccib58d10d2017-03-18 02:00:25 -07001352 package_pins = []
1353 if packages:
Takuto Ikuta2efc7792019-11-27 14:33:34 +00001354 package_pins = _install_packages(run_dir, cipd_cache_dir, client,
1355 packages)
iannuccib58d10d2017-03-18 02:00:25 -07001356
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001357 # Install isolated client to |isolated_dir|.
Takuto Ikuta02edca22019-11-29 10:04:51 +00001358 _install_packages(isolated_dir, cipd_cache_dir, client,
Takuto Ikuta9c4eb1d2020-10-05 03:40:14 +00001359 [('', ISOLATED_PACKAGE, _LUCI_GO_REVISION)])
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001360
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001361 # Install cas client to |cas_dir|.
1362 _install_packages(cas_dir, cipd_cache_dir, client,
Takuto Ikuta9c4eb1d2020-10-05 03:40:14 +00001363 [('', _CAS_PACKAGE, _LUCI_GO_REVISION)])
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001364
iannuccib58d10d2017-03-18 02:00:25 -07001365 file_path.make_tree_files_read_only(run_dir)
nodir90bc8dc2016-06-15 13:35:21 -07001366
vadimsh232f5a82017-01-20 19:23:44 -08001367 total_duration = time.time() - start
Junji Watanabe38b28b02020-04-23 10:23:30 +00001368 logging.info('Installing CIPD client and packages took %d seconds',
1369 total_duration)
nodir90bc8dc2016-06-15 13:35:21 -07001370
vadimsh232f5a82017-01-20 19:23:44 -08001371 yield CipdInfo(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001372 client=client,
1373 cache_dir=cipd_cache_dir,
1374 stats={
1375 'duration': total_duration,
1376 'get_client_duration': get_client_duration,
iannuccib58d10d2017-03-18 02:00:25 -07001377 },
Junji Watanabe38b28b02020-04-23 10:23:30 +00001378 pins={
1379 'client_package': {
1380 'package_name': client.package_name,
1381 'version': client.instance_id,
1382 },
1383 'packages': package_pins,
1384 })
nodirbe642ff2016-06-09 15:51:51 -07001385
1386
1387def create_option_parser():
Marc-Antoine Ruelf74cffe2015-07-15 15:21:34 -04001388 parser = logging_utils.OptionParserWithLogging(
nodir55be77b2016-05-03 09:39:57 -07001389 usage='%prog <options> [command to run or extra args]',
maruel@chromium.orgdedbf492013-09-12 20:42:11 +00001390 version=__version__,
1391 log_file=RUN_ISOLATED_LOG_FILE)
maruela9cfd6f2015-09-15 11:03:15 -07001392 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001393 '--clean',
1394 action='store_true',
maruel36a963d2016-04-08 17:15:49 -07001395 help='Cleans the cache, trimming it necessary and remove corrupted items '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001396 'and returns without executing anything; use with -v to know what '
1397 'was done')
maruel36a963d2016-04-08 17:15:49 -07001398 parser.add_option(
maruela9cfd6f2015-09-15 11:03:15 -07001399 '--json',
1400 help='dump output metadata to json file. When used, run_isolated returns '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001401 'non-zero only on internal failure')
maruel6be7f9e2015-10-01 12:25:30 -07001402 parser.add_option(
maruel5c9e47b2015-12-18 13:02:30 -08001403 '--hard-timeout', type='float', help='Enforce hard timeout in execution')
maruel6be7f9e2015-10-01 12:25:30 -07001404 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001405 '--grace-period',
1406 type='float',
maruel6be7f9e2015-10-01 12:25:30 -07001407 help='Grace period between SIGTERM and SIGKILL')
bpastene3ae09522016-06-10 17:12:59 -07001408 parser.add_option(
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001409 '--relative-cwd',
Takuto Ikuta18ca29a2020-12-04 07:34:20 +00001410 help='Ignore the isolated \'relative_cwd\' and use this one instead')
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001411 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001412 '--env',
1413 default=[],
1414 action='append',
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001415 help='Environment variables to set for the child process')
1416 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001417 '--env-prefix',
1418 default=[],
1419 action='append',
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001420 help='Specify a VAR=./path/fragment to put in the environment variable '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001421 'before executing the command. The path fragment must be relative '
1422 'to the isolated run directory, and must not contain a `..` token. '
1423 'The path will be made absolute and prepended to the indicated '
1424 '$VAR using the OS\'s path separator. Multiple items for the same '
1425 '$VAR will be prepended in order.')
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001426 parser.add_option(
bpastene3ae09522016-06-10 17:12:59 -07001427 '--bot-file',
1428 help='Path to a file describing the state of the host. The content is '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001429 'defined by on_before_task() in bot_config.')
aludwin7556e0c2016-10-26 08:46:10 -07001430 parser.add_option(
vadimsh9c54b2c2017-07-25 14:08:29 -07001431 '--switch-to-account',
1432 help='If given, switches LUCI_CONTEXT to given logical service account '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001433 '(e.g. "task" or "system") before launching the isolated process.')
vadimsh9c54b2c2017-07-25 14:08:29 -07001434 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001435 '--output',
1436 action='append',
aludwin0a8e17d2016-10-27 15:57:39 -07001437 help='Specifies an output to return. If no outputs are specified, all '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001438 'files located in $(ISOLATED_OUTDIR) will be returned; '
1439 'otherwise, outputs in both $(ISOLATED_OUTDIR) and those '
1440 'specified by --output option (there can be multiple) will be '
1441 'returned. Note that if a file in OUT_DIR has the same path '
1442 'as an --output option, the --output version will be returned.')
aludwin0a8e17d2016-10-27 15:57:39 -07001443 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001444 '-a',
1445 '--argsfile',
aludwin7556e0c2016-10-26 08:46:10 -07001446 # This is actually handled in parse_args; it's included here purely so it
1447 # can make it into the help text.
1448 help='Specify a file containing a JSON array of arguments to this '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001449 'script. If --argsfile is provided, no other argument may be '
1450 'provided on the command line.')
Takuto Ikutad4be2f12020-05-12 02:15:25 +00001451 parser.add_option(
1452 '--report-on-exception',
1453 action='store_true',
1454 help='Whether report exception during execution to isolate server. '
1455 'This flag should only be used in swarming bot.')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001456
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001457 group = optparse.OptionGroup(parser, 'Data source - Isolate server')
Junji Watanabe54925c32020-09-08 00:56:18 +00001458 # Deprecated. Isoate server is being migrated to RBE-CAS.
1459 # Remove --isolated and isolate server options after migration.
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001460 group.add_option(
Marc-Antoine Ruel185ded42015-01-28 20:49:18 -05001461 '-s', '--isolated',
nodir55be77b2016-05-03 09:39:57 -07001462 help='Hash of the .isolated to grab from the isolate server.')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001463 isolateserver.add_isolate_server_options(group)
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001464 parser.add_option_group(group)
1465
1466 group = optparse.OptionGroup(parser,
1467 'Data source - Content Addressed Storage')
Junji Watanabe54925c32020-09-08 00:56:18 +00001468 group.add_option(
1469 '--cas-instance', help='Full CAS instance name for input/output files.')
1470 group.add_option(
1471 '--cas-digest',
1472 help='Digest of the input root on RBE-CAS. The format is '
1473 '`{hash}/{size_bytes}`.')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001474 parser.add_option_group(group)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001475
Junji Watanabeb03450b2020-09-25 05:09:27 +00001476 # Cache options.
Marc-Antoine Ruela57d7db2014-10-15 20:31:19 -04001477 isolateserver.add_cache_options(parser)
Junji Watanabeb03450b2020-09-25 05:09:27 +00001478 add_cas_cache_options(parser)
nodirbe642ff2016-06-09 15:51:51 -07001479
1480 cipd.add_cipd_options(parser)
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001481
1482 group = optparse.OptionGroup(parser, 'Named caches')
1483 group.add_option(
1484 '--named-cache',
1485 dest='named_caches',
1486 action='append',
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001487 nargs=3,
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001488 default=[],
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001489 help='A named cache to request. Accepts 3 arguments: name, path, hint. '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001490 'name identifies the cache, must match regex [a-z0-9_]{1,4096}. '
1491 'path is a path relative to the run dir where the cache directory '
1492 'must be put to. '
1493 'This option can be specified more than once.')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001494 group.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001495 '--named-cache-root',
1496 default='named_caches',
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001497 help='Cache root directory. Default=%default')
1498 parser.add_option_group(group)
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001499
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001500 group = optparse.OptionGroup(parser, 'Process containment')
1501 parser.add_option(
1502 '--lower-priority', action='store_true',
1503 help='Lowers the child process priority')
1504 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001505 '--containment-type',
1506 choices=('NONE', 'AUTO', 'JOB_OBJECT'),
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001507 default='NONE',
1508 help='Type of container to use')
1509 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001510 '--limit-processes',
1511 type='int',
1512 default=0,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001513 help='Maximum number of active processes in the containment')
1514 parser.add_option(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001515 '--limit-total-committed-memory',
1516 type='int',
1517 default=0,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001518 help='Maximum sum of committed memory in the containment')
1519 parser.add_option_group(group)
1520
1521 group = optparse.OptionGroup(parser, 'Debugging')
1522 group.add_option(
Kenneth Russell61d42352014-09-15 11:41:16 -07001523 '--leak-temp-dir',
1524 action='store_true',
nodirbe642ff2016-06-09 15:51:51 -07001525 help='Deliberately leak isolate\'s temp dir for later examination. '
Junji Watanabe38b28b02020-04-23 10:23:30 +00001526 'Default: %default')
1527 group.add_option('--root-dir', help='Use a directory instead of a random one')
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001528 parser.add_option_group(group)
Kenneth Russell61d42352014-09-15 11:41:16 -07001529
Vadim Shtayurae34e13a2014-02-02 11:23:26 -08001530 auth.add_auth_options(parser)
nodirbe642ff2016-06-09 15:51:51 -07001531
Ye Kuang1d096cb2020-06-26 08:38:21 +00001532 parser.set_defaults(cache='cache')
nodirbe642ff2016-06-09 15:51:51 -07001533 return parser
1534
1535
Junji Watanabeb03450b2020-09-25 05:09:27 +00001536def add_cas_cache_options(parser):
1537 group = optparse.OptionGroup(parser, 'CAS cache management')
1538 group.add_option(
1539 '--cas-cache',
1540 metavar='DIR',
1541 default='cas-cache',
1542 help='Directory to keep a local cache of the files. Accelerates download '
1543 'by reusing already downloaded files. Default=%default')
Takuto Ikuta7ff4b242020-12-03 08:07:06 +00001544 group.add_option(
1545 '--kvs-file',
1546 default='',
1547 help='CAS cache using kvs for small files. Default=%default')
Junji Watanabeb03450b2020-09-25 05:09:27 +00001548 parser.add_option_group(group)
1549
1550
1551def process_cas_cache_options(options):
1552 if options.cas_cache:
1553 policies = local_caching.CachePolicies(
1554 max_cache_size=options.max_cache_size,
1555 min_free_space=options.min_free_space,
1556 # max_items isn't used for CAS cache for now.
1557 max_items=None,
1558 max_age_secs=MAX_AGE_SECS)
1559
1560 return local_caching.DiskContentAddressedCache(
1561 six.text_type(os.path.abspath(options.cas_cache)), policies, trim=False)
1562 return local_caching.MemoryContentAddressedCache()
1563
1564
Marc-Antoine Ruel49f9f8d2018-05-24 15:57:06 -04001565def process_named_cache_options(parser, options, time_fn=None):
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001566 """Validates named cache options and returns a CacheManager."""
1567 if options.named_caches and not options.named_cache_root:
1568 parser.error('--named-cache is specified, but --named-cache-root is empty')
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001569 for name, path, hint in options.named_caches:
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001570 if not CACHE_NAME_RE.match(name):
1571 parser.error(
1572 'cache name %r does not match %r' % (name, CACHE_NAME_RE.pattern))
1573 if not path:
1574 parser.error('cache path cannot be empty')
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001575 try:
Takuto Ikuta630f99d2020-07-02 12:59:35 +00001576 int(hint)
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001577 except ValueError:
1578 parser.error('cache hint must be a number')
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001579 if options.named_cache_root:
1580 # Make these configurable later if there is use case but for now it's fairly
1581 # safe values.
1582 # In practice, a fair chunk of bots are already recycled on a daily schedule
1583 # so this code doesn't have any effect to them, unless they are preloaded
1584 # with a really old cache.
1585 policies = local_caching.CachePolicies(
1586 # 1TiB.
1587 max_cache_size=1024*1024*1024*1024,
1588 min_free_space=options.min_free_space,
1589 max_items=50,
Marc-Antoine Ruel5d7606b2018-06-15 19:06:12 +00001590 max_age_secs=MAX_AGE_SECS)
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001591 root_dir = six.text_type(os.path.abspath(options.named_cache_root))
John Budorickc6186972020-02-26 00:58:14 +00001592 cache = local_caching.NamedCache(root_dir, policies, time_fn=time_fn)
1593 # Touch any named caches we're going to use to minimize thrashing
1594 # between tasks that request some (but not all) of the same named caches.
John Budorick0a4dab62020-03-02 22:23:35 +00001595 cache.touch(*[name for name, _, _ in options.named_caches])
John Budorickc6186972020-02-26 00:58:14 +00001596 return cache
Marc-Antoine Ruel8b11dbd2018-05-18 14:31:22 -04001597 return None
1598
1599
aludwin7556e0c2016-10-26 08:46:10 -07001600def parse_args(args):
1601 # Create a fake mini-parser just to get out the "-a" command. Note that
1602 # it's not documented here; instead, it's documented in create_option_parser
1603 # even though that parser will never actually get to parse it. This is
1604 # because --argsfile is exclusive with all other options and arguments.
1605 file_argparse = argparse.ArgumentParser(add_help=False)
1606 file_argparse.add_argument('-a', '--argsfile')
1607 (file_args, nonfile_args) = file_argparse.parse_known_args(args)
1608 if file_args.argsfile:
1609 if nonfile_args:
1610 file_argparse.error('Can\'t specify --argsfile with'
1611 'any other arguments (%s)' % nonfile_args)
1612 try:
1613 with open(file_args.argsfile, 'r') as f:
1614 args = json.load(f)
1615 except (IOError, OSError, ValueError) as e:
1616 # We don't need to error out here - "args" is now empty,
1617 # so the call below to parser.parse_args(args) will fail
1618 # and print the full help text.
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +00001619 print('Couldn\'t read arguments: %s' % e, file=sys.stderr)
aludwin7556e0c2016-10-26 08:46:10 -07001620
1621 # Even if we failed to read the args, just call the normal parser now since it
1622 # will print the correct help message.
nodirbe642ff2016-06-09 15:51:51 -07001623 parser = create_option_parser()
Marc-Antoine Ruel90c98162013-12-18 15:11:57 -05001624 options, args = parser.parse_args(args)
Ye Kuangfff1e502020-07-13 13:21:57 +00001625 if not isinstance(options.cipd_enabled, (bool, int)):
1626 options.cipd_enabled = distutils.util.strtobool(options.cipd_enabled)
aludwin7556e0c2016-10-26 08:46:10 -07001627 return (parser, options, args)
1628
1629
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001630def _calc_named_cache_hint(named_cache, named_caches):
1631 """Returns the expected size of the missing named caches."""
1632 present = named_cache.available
1633 size = 0
1634 for name, _, hint in named_caches:
1635 if name not in present:
Takuto Ikuta630f99d2020-07-02 12:59:35 +00001636 hint = int(hint)
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001637 if hint > 0:
1638 size += hint
1639 return size
1640
1641
Takuto Ikutaae391c52020-12-03 08:43:45 +00001642def _clean_cmd(parser, options, caches, root):
Takuto Ikuta7ff4b242020-12-03 08:07:06 +00001643 """Cleanup cache dirs/files."""
1644 if options.isolated:
1645 parser.error('Can\'t use --isolated with --clean.')
1646 if options.isolate_server:
1647 parser.error('Can\'t use --isolate-server with --clean.')
1648 if options.json:
1649 parser.error('Can\'t use --json with --clean.')
1650 if options.named_caches:
1651 parser.error('Can\t use --named-cache with --clean.')
1652 if options.cas_instance or options.cas_digest:
1653 parser.error('Can\t use --cas-instance, --cas-digest with --clean.')
1654
1655 logging.info("initial free space: %d", file_path.get_free_space(root))
1656
Takuto Ikutaae391c52020-12-03 08:43:45 +00001657 if options.kvs_file and fs.isfile(six.text_type(options.kvs_file)):
Takuto Ikuta7ff4b242020-12-03 08:07:06 +00001658 # Remove kvs file if its size exceeds fixed threshold.
Takuto Ikuta80bee362020-12-03 10:51:37 +00001659 st = fs.stat(six.text_type(options.kvs_file))
Takuto Ikuta7ff4b242020-12-03 08:07:06 +00001660 if st.st_size >= _CAS_KVS_CACHE_THRESHOLD:
1661 logging.info("remove kvs file with size: %d", st.st_size)
Takuto Ikuta80bee362020-12-03 10:51:37 +00001662 fs.remove(six.text_type(options.kvs_file))
Takuto Ikuta7ff4b242020-12-03 08:07:06 +00001663
1664 # Trim first, then clean.
1665 local_caching.trim_caches(
1666 caches,
1667 root,
1668 min_free_space=options.min_free_space,
1669 max_age_secs=MAX_AGE_SECS)
1670 logging.info("free space after trim: %d", file_path.get_free_space(root))
1671 for c in caches:
1672 c.cleanup()
1673 logging.info("free space after cleanup: %d", file_path.get_free_space(root))
1674
1675
aludwin7556e0c2016-10-26 08:46:10 -07001676def main(args):
Marc-Antoine Ruelee6ca622017-11-29 11:19:16 -05001677 # Warning: when --argsfile is used, the strings are unicode instances, when
1678 # parsed normally, the strings are str instances.
aludwin7556e0c2016-10-26 08:46:10 -07001679 (parser, options, args) = parse_args(args)
maruel36a963d2016-04-08 17:15:49 -07001680
Takuto Ikutad4be2f12020-05-12 02:15:25 +00001681 if options.report_on_exception and options.isolate_server:
1682 on_error.report_on_exception_exit(options.isolate_server)
1683
Marc-Antoine Ruel5028ba22017-08-25 17:37:51 -04001684 if not file_path.enable_symlink():
Marc-Antoine Ruel5a024272019-01-15 20:11:16 +00001685 logging.warning('Symlink support is not enabled')
Marc-Antoine Ruel5028ba22017-08-25 17:37:51 -04001686
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001687 named_cache = process_named_cache_options(parser, options)
Marc-Antoine Ruel0d8b0f62018-09-10 14:40:35 +00001688 # hint is 0 if there's no named cache.
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001689 hint = _calc_named_cache_hint(named_cache, options.named_caches)
1690 if hint:
1691 # Increase the --min-free-space value by the hint, and recreate the
1692 # NamedCache instance so it gets the updated CachePolicy.
1693 options.min_free_space += hint
1694 named_cache = process_named_cache_options(parser, options)
1695
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001696 # TODO(crbug.com/932396): Remove this.
Takuto Ikuta4a22c2c2020-06-05 02:02:23 +00001697 use_go_isolated = options.cipd_enabled
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001698
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001699 # TODO(maruel): CIPD caches should be defined at an higher level here too, so
1700 # they can be cleaned the same way.
Takuto Ikutaf1c58442020-10-20 09:03:27 +00001701
1702 isolate_cache = isolateserver.process_cache_options(options, trim=False)
1703 cas_cache = process_cas_cache_options(options)
Takuto Ikuta00cf8fc2020-01-14 01:36:00 +00001704
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001705 caches = []
1706 if isolate_cache:
1707 caches.append(isolate_cache)
Junji Watanabeb03450b2020-09-25 05:09:27 +00001708 if cas_cache:
1709 caches.append(cas_cache)
Marc-Antoine Ruel7139d912018-06-15 20:04:42 +00001710 if named_cache:
1711 caches.append(named_cache)
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001712 root = caches[0].cache_dir if caches else six.text_type(os.getcwd())
maruel36a963d2016-04-08 17:15:49 -07001713 if options.clean:
Takuto Ikutaae391c52020-12-03 08:43:45 +00001714 _clean_cmd(parser, options, caches, root)
maruel36a963d2016-04-08 17:15:49 -07001715 return 0
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001716
1717 # Trim must still be done for the following case:
1718 # - named-cache was used
1719 # - some entries, with a large hint, where missing
1720 # - --min-free-space was increased accordingly, thus trimming is needed
1721 # Otherwise, this will have no effect, as bot_main calls run_isolated with
1722 # --clean after each task.
Takuto Ikutaa010c532020-10-21 05:42:29 +00001723 local_caching.trim_caches(
1724 caches,
1725 root,
1726 # Add 1GB more buffer for Go CLI.
1727 min_free_space=options.min_free_space + _FREE_SPACE_BUFFER_FOR_GO,
1728 max_age_secs=MAX_AGE_SECS)
maruel36a963d2016-04-08 17:15:49 -07001729
Takuto Ikutaf1c58442020-10-20 09:03:27 +00001730 # Save state of isolate/cas cache not to overwrite state from go client.
1731 if use_go_isolated:
1732 isolate_cache.save()
1733 isolate_cache = None
1734 if cas_cache:
1735 cas_cache.save()
1736 cas_cache = None
1737
nodir55be77b2016-05-03 09:39:57 -07001738 if not options.isolated and not args:
1739 parser.error('--isolated or command to run is required.')
1740
Vadim Shtayura5d1efce2014-02-04 10:55:43 -08001741 auth.process_auth_options(parser, options)
nodir55be77b2016-05-03 09:39:57 -07001742
Takuto Ikutaae767b32020-05-11 01:22:19 +00001743 isolateserver.process_isolate_server_options(parser, options, False)
Junji Watanabeed9ce352020-09-25 12:32:07 +00001744 if ISOLATED_OUTDIR_PARAMETER in args and (not options.isolate_server and
1745 not options.cas_instance):
1746 parser.error('%s in args requires --isolate-server or --cas-instance' %
1747 ISOLATED_OUTDIR_PARAMETER)
1748
1749 if options.isolated and not options.isolate_server:
1750 parser.error('--isolated requires --isolate-server')
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001751
nodir90bc8dc2016-06-15 13:35:21 -07001752 if options.root_dir:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001753 options.root_dir = six.text_type(os.path.abspath(options.root_dir))
Takuto Ikutad46ea762020-10-07 05:43:22 +00001754 else:
1755 options.root_dir = six.text_type(tempfile.mkdtemp(prefix='root'))
maruel12e30012015-10-09 11:55:35 -07001756 if options.json:
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001757 options.json = six.text_type(os.path.abspath(options.json))
nodir55be77b2016-05-03 09:39:57 -07001758
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001759 if any('=' not in i for i in options.env):
1760 parser.error(
1761 '--env required key=value form. value can be skipped to delete '
1762 'the variable')
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -05001763 options.env = dict(i.split('=', 1) for i in options.env)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001764
1765 prefixes = {}
1766 cwd = os.path.realpath(os.getcwd())
1767 for item in options.env_prefix:
1768 if '=' not in item:
1769 parser.error(
1770 '--env-prefix %r is malformed, must be in the form `VAR=./path`'
1771 % item)
Marc-Antoine Ruel7a68f712017-12-01 18:45:18 -05001772 key, opath = item.split('=', 1)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001773 if os.path.isabs(opath):
1774 parser.error('--env-prefix %r path is bad, must be relative.' % opath)
1775 opath = os.path.normpath(opath)
1776 if not os.path.realpath(os.path.join(cwd, opath)).startswith(cwd):
1777 parser.error(
Junji Watanabe38b28b02020-04-23 10:23:30 +00001778 '--env-prefix %r path is bad, must be relative and not contain `..`.'
1779 % opath)
Marc-Antoine Ruel19dd8872017-11-28 18:33:39 -05001780 prefixes.setdefault(key, []).append(opath)
1781 options.env_prefix = prefixes
Robert Iannuccibf5f84c2017-11-22 12:56:50 -08001782
nodirbe642ff2016-06-09 15:51:51 -07001783 cipd.validate_cipd_options(parser, options)
1784
vadimsh232f5a82017-01-20 19:23:44 -08001785 install_packages_fn = noop_install_packages
Ye Kuang1d096cb2020-06-26 08:38:21 +00001786 tmp_cipd_cache_dir = None
vadimsh902948e2017-01-20 15:57:32 -08001787 if options.cipd_enabled:
Ye Kuang1d096cb2020-06-26 08:38:21 +00001788 cache_dir = options.cipd_cache
1789 if not cache_dir:
1790 tmp_cipd_cache_dir = six.text_type(tempfile.mkdtemp())
1791 cache_dir = tmp_cipd_cache_dir
Takuto Ikutab7ce0e32019-11-27 23:26:18 +00001792 install_packages_fn = (
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001793 lambda run_dir, isolated_dir, cas_dir: install_client_and_packages(
Ye Kuang1d096cb2020-06-26 08:38:21 +00001794 run_dir,
1795 cipd.parse_package_args(options.cipd_packages),
1796 options.cipd_server,
1797 options.cipd_client_package,
1798 options.cipd_client_version,
1799 cache_dir=cache_dir,
Junji Watanabe4b890ef2020-09-16 01:43:27 +00001800 isolated_dir=isolated_dir,
1801 cas_dir=cas_dir,
1802 ))
nodirbe642ff2016-06-09 15:51:51 -07001803
nodird6160682017-02-02 13:03:35 -08001804 @contextlib.contextmanager
nodir0ae98b32017-05-11 13:21:53 -07001805 def install_named_caches(run_dir):
nodird6160682017-02-02 13:03:35 -08001806 # WARNING: this function depends on "options" variable defined in the outer
1807 # function.
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001808 assert six.text_type(run_dir), repr(run_dir)
Marc-Antoine Ruel49f9f8d2018-05-24 15:57:06 -04001809 assert os.path.isabs(run_dir), run_dir
Takuto Ikuta6e2ff962019-10-29 12:35:27 +00001810 named_caches = [(os.path.join(run_dir, six.text_type(relpath)), name)
1811 for name, relpath, _ in options.named_caches]
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001812 for path, name in named_caches:
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001813 named_cache.install(path, name)
nodird6160682017-02-02 13:03:35 -08001814 try:
1815 yield
1816 finally:
dnje289d132017-07-07 11:16:44 -07001817 # Uninstall each named cache, returning it to the cache pool. If an
1818 # uninstall fails for a given cache, it will remain in the task's
1819 # temporary space, get cleaned up by the Swarming bot, and be lost.
1820 #
1821 # If the Swarming bot cannot clean up the cache, it will handle it like
1822 # any other bot file that could not be removed.
Marc-Antoine Ruelc7a704b2018-08-29 19:02:23 +00001823 for path, name in reversed(named_caches):
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001824 try:
Marc-Antoine Ruele9558372018-08-03 03:41:22 +00001825 # uninstall() doesn't trim but does call save() implicitly. Trimming
1826 # *must* be done manually via periodic 'run_isolated.py --clean'.
Marc-Antoine Ruele79ddbf2018-06-13 18:33:07 +00001827 named_cache.uninstall(path, name)
1828 except local_caching.NamedCacheError:
1829 logging.exception('Error while removing named cache %r at %r. '
1830 'The cache will be lost.', path, name)
nodirf33b8d62016-10-26 22:34:58 -07001831
Takuto Ikutaf3caa9b2020-11-02 05:38:26 +00001832 command = args
1833 if options.relative_cwd:
1834 a = os.path.normpath(os.path.abspath(options.relative_cwd))
1835 if not a.startswith(os.getcwd()):
1836 parser.error(
1837 '--relative-cwd must not try to escape the working directory')
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001838
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001839 containment_type = subprocess42.Containment.NONE
1840 if options.containment_type == 'AUTO':
1841 containment_type = subprocess42.Containment.AUTO
1842 if options.containment_type == 'JOB_OBJECT':
1843 containment_type = subprocess42.Containment.JOB_OBJECT
1844 containment = subprocess42.Containment(
1845 containment_type=containment_type,
1846 limit_processes=options.limit_processes,
1847 limit_total_committed_memory=options.limit_total_committed_memory)
1848
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001849 data = TaskData(
1850 command=command,
Marc-Antoine Ruel95068cf2017-12-07 21:35:05 -05001851 relative_cwd=options.relative_cwd,
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001852 isolated_hash=options.isolated,
1853 storage=None,
1854 isolate_cache=isolate_cache,
Junji Watanabe54925c32020-09-08 00:56:18 +00001855 cas_instance=options.cas_instance,
1856 cas_digest=options.cas_digest,
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001857 outputs=options.output,
1858 install_named_caches=install_named_caches,
1859 leak_temp_dir=options.leak_temp_dir,
1860 root_dir=_to_unicode(options.root_dir),
1861 hard_timeout=options.hard_timeout,
1862 grace_period=options.grace_period,
1863 bot_file=options.bot_file,
1864 switch_to_account=options.switch_to_account,
1865 install_packages_fn=install_packages_fn,
Takuto Ikuta5c59a842020-01-24 03:05:24 +00001866 use_go_isolated=use_go_isolated,
Takuto Ikuta10cae642020-01-08 08:12:07 +00001867 go_cache_dir=options.cache,
Takuto Ikuta879788c2020-01-10 08:00:26 +00001868 go_cache_policies=local_caching.CachePolicies(
1869 max_cache_size=options.max_cache_size,
1870 min_free_space=options.min_free_space,
1871 max_items=options.max_items,
1872 max_age_secs=None,
1873 ),
Junji Watanabeb03450b2020-09-25 05:09:27 +00001874 cas_cache_dir=options.cas_cache,
1875 cas_cache_policies=local_caching.CachePolicies(
1876 max_cache_size=options.max_cache_size,
1877 min_free_space=options.min_free_space,
1878 max_items=None,
1879 max_age_secs=None,
1880 ),
Takuto Ikutaae391c52020-12-03 08:43:45 +00001881 cas_kvs=options.kvs_file,
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001882 env=options.env,
Marc-Antoine Ruel03c6fd12019-04-30 12:12:55 +00001883 env_prefix=options.env_prefix,
Marc-Antoine Ruel1b65f4e2019-05-02 21:56:58 +00001884 lower_priority=bool(options.lower_priority),
1885 containment=containment)
nodirbe642ff2016-06-09 15:51:51 -07001886 try:
nodir90bc8dc2016-06-15 13:35:21 -07001887 if options.isolate_server:
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001888 server_ref = isolate_storage.ServerRef(
nodir90bc8dc2016-06-15 13:35:21 -07001889 options.isolate_server, options.namespace)
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001890 storage = isolateserver.get_storage(server_ref)
nodir90bc8dc2016-06-15 13:35:21 -07001891 with storage:
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001892 data = data._replace(storage=storage)
nodirf33b8d62016-10-26 22:34:58 -07001893 # Hashing schemes used by |storage| and |isolate_cache| MUST match.
Marc-Antoine Ruelb8513132018-11-20 19:48:53 +00001894 assert storage.server_ref.hash_algo == server_ref.hash_algo
Marc-Antoine Ruel7de52592017-12-07 10:41:12 -05001895 return run_tha_test(data, options.json)
1896 return run_tha_test(data, options.json)
Junji Watanabe38b28b02020-04-23 10:23:30 +00001897 except (cipd.Error, local_caching.NamedCacheError,
1898 local_caching.NoMoreSpace) as ex:
Marc-Antoine Ruelf899c482019-10-10 23:32:06 +00001899 print(ex.message, file=sys.stderr)
nodirbe642ff2016-06-09 15:51:51 -07001900 return 1
Ye Kuang1d096cb2020-06-26 08:38:21 +00001901 finally:
1902 if tmp_cipd_cache_dir is not None:
1903 try:
1904 file_path.rmtree(tmp_cipd_cache_dir)
1905 except OSError:
1906 logging.exception('Remove tmp_cipd_cache_dir=%s failed',
1907 tmp_cipd_cache_dir)
1908 # Best effort clean up. Failed to do so doesn't affect the outcome.
maruel@chromium.org9c72d4e2012-09-28 19:20:25 +00001909
1910
1911if __name__ == '__main__':
maruel8e4e40c2016-05-30 06:21:07 -07001912 subprocess42.inhibit_os_error_reporting()
csharp@chromium.orgbfb98742013-03-26 20:28:36 +00001913 # Ensure that we are always running with the correct encoding.
vadimsh@chromium.orga4326472013-08-24 02:05:41 +00001914 fix_encoding.fix_encoding()
Ye Kuang2dd17442020-04-22 08:45:52 +00001915 net.set_user_agent('run_isolated.py/' + __version__)
Marc-Antoine Ruel90c98162013-12-18 15:11:57 -05001916 sys.exit(main(sys.argv[1:]))