blob: f68b91af9c1bda5c0c5e83ef12bfc5d22a733d81 [file] [log] [blame]
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00001#!/usr/bin/env python
Marc-Antoine Ruel8add1242013-11-05 17:28:27 -05002# Copyright 2012 The Swarming Authors. All rights reserved.
Marc-Antoine Ruele98b1122013-11-05 20:27:57 -05003# Use of this source code is governed under the Apache License, Version 2.0 that
4# can be found in the LICENSE file.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00005
maruel@chromium.orge5322512013-08-19 20:17:57 +00006"""Front end tool to operate on .isolate files.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00007
maruel@chromium.orge5322512013-08-19 20:17:57 +00008This includes creating, merging or compiling them to generate a .isolated file.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00009
10See more information at
maruel@chromium.orge5322512013-08-19 20:17:57 +000011 https://code.google.com/p/swarming/wiki/IsolateDesign
12 https://code.google.com/p/swarming/wiki/IsolateUserGuide
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000013"""
maruel@chromium.orge5322512013-08-19 20:17:57 +000014# Run ./isolate.py --help for more detailed information.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000015
Vadim Shtayurafddb1432014-09-30 18:32:41 -070016__version__ = '0.4.1'
Marc-Antoine Ruelcfb60852014-07-02 15:22:00 -040017
Marc-Antoine Ruel9dfdcc22014-01-08 14:14:18 -050018import datetime
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000019import logging
20import optparse
21import os
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000022import re
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000023import subprocess
24import sys
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000025
Vadim Shtayurae34e13a2014-02-02 11:23:26 -080026import auth
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -050027import isolate_format
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -040028import isolated_format
maruel@chromium.orgfb78d432013-08-28 21:22:40 +000029import isolateserver
maruel@chromium.orgb8375c22012-10-05 18:10:01 +000030import run_isolated
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000031
maruel@chromium.orge5322512013-08-19 20:17:57 +000032from third_party import colorama
33from third_party.depot_tools import fix_encoding
34from third_party.depot_tools import subcommand
35
maruel@chromium.org561d4b22013-09-26 21:08:08 +000036from utils import file_path
vadimsh@chromium.orga4326472013-08-24 02:05:41 +000037from utils import tools
38
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000039
Vadim Shtayurafddb1432014-09-30 18:32:41 -070040# Supported version of *.isolated.gen.json files consumed by CMDbatcharchive.
41ISOLATED_GEN_JSON_VERSION = 1
42
43
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000044class ExecutionError(Exception):
45 """A generic error occurred."""
46 def __str__(self):
47 return self.args[0]
48
49
50### Path handling code.
51
52
maruel@chromium.org7b844a62013-09-17 13:04:59 +000053def recreate_tree(outdir, indir, infiles, action, as_hash):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000054 """Creates a new tree with only the input files in it.
55
56 Arguments:
57 outdir: Output directory to create the files in.
58 indir: Root directory the infiles are based in.
59 infiles: dict of files to map from |indir| to |outdir|.
maruel@chromium.orgba6489b2013-07-11 20:23:33 +000060 action: One of accepted action of run_isolated.link_file().
maruel@chromium.org7b844a62013-09-17 13:04:59 +000061 as_hash: Output filename is the hash instead of relfile.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000062 """
63 logging.info(
maruel@chromium.org7b844a62013-09-17 13:04:59 +000064 'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
65 (outdir, indir, len(infiles), action, as_hash))
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000066
maruel@chromium.org61a9b3b2012-12-12 17:18:52 +000067 assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000068 if not os.path.isdir(outdir):
maruel@chromium.org61a9b3b2012-12-12 17:18:52 +000069 logging.info('Creating %s' % outdir)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000070 os.makedirs(outdir)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000071
72 for relfile, metadata in infiles.iteritems():
73 infile = os.path.join(indir, relfile)
maruel@chromium.org7b844a62013-09-17 13:04:59 +000074 if as_hash:
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000075 # Do the hashtable specific checks.
maruel@chromium.orge5c17132012-11-21 18:18:46 +000076 if 'l' in metadata:
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000077 # Skip links when storing a hashtable.
78 continue
maruel@chromium.orge5c17132012-11-21 18:18:46 +000079 outfile = os.path.join(outdir, metadata['h'])
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000080 if os.path.isfile(outfile):
81 # Just do a quick check that the file size matches. No need to stat()
82 # again the input file, grab the value from the dict.
maruel@chromium.orge5c17132012-11-21 18:18:46 +000083 if not 's' in metadata:
Marc-Antoine Ruel1e7658c2014-08-28 19:46:39 -040084 raise isolated_format.MappingError(
maruel@chromium.org861a5e72012-10-09 14:49:42 +000085 'Misconfigured item %s: %s' % (relfile, metadata))
maruel@chromium.orge5c17132012-11-21 18:18:46 +000086 if metadata['s'] == os.stat(outfile).st_size:
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000087 continue
88 else:
maruel@chromium.orge5c17132012-11-21 18:18:46 +000089 logging.warn('Overwritting %s' % metadata['h'])
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000090 os.remove(outfile)
91 else:
92 outfile = os.path.join(outdir, relfile)
93 outsubdir = os.path.dirname(outfile)
94 if not os.path.isdir(outsubdir):
95 os.makedirs(outsubdir)
96
maruel@chromium.orge5c17132012-11-21 18:18:46 +000097 if 'l' in metadata:
98 pointed = metadata['l']
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +000099 logging.debug('Symlink: %s -> %s' % (outfile, pointed))
maruel@chromium.orgf43e68b2012-10-15 20:23:10 +0000100 # symlink doesn't exist on Windows.
101 os.symlink(pointed, outfile) # pylint: disable=E1101
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000102 else:
maruel@chromium.orgb8375c22012-10-05 18:10:01 +0000103 run_isolated.link_file(outfile, infile, action)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000104
105
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000106### Variable stuff.
107
108
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500109def _normalize_path_variable(cwd, relative_base_dir, key, value):
110 """Normalizes a path variable into a relative directory.
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500111 """
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500112 # Variables could contain / or \ on windows. Always normalize to
113 # os.path.sep.
114 x = os.path.join(cwd, value.strip().replace('/', os.path.sep))
115 normalized = file_path.get_native_path_case(os.path.normpath(x))
116 if not os.path.isdir(normalized):
117 raise ExecutionError('%s=%s is not a directory' % (key, normalized))
118
119 # All variables are relative to the .isolate file.
120 normalized = os.path.relpath(normalized, relative_base_dir)
121 logging.debug(
122 'Translated variable %s from %s to %s', key, value, normalized)
123 return normalized
124
125
126def normalize_path_variables(cwd, path_variables, relative_base_dir):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000127 """Processes path variables as a special case and returns a copy of the dict.
128
maruel@chromium.org61a9b3b2012-12-12 17:18:52 +0000129 For each 'path' variable: first normalizes it based on |cwd|, verifies it
130 exists then sets it as relative to relative_base_dir.
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500131 """
132 logging.info(
133 'normalize_path_variables(%s, %s, %s)', cwd, path_variables,
134 relative_base_dir)
Marc-Antoine Ruel9cc42c32013-12-11 09:35:55 -0500135 assert isinstance(cwd, unicode), cwd
136 assert isinstance(relative_base_dir, unicode), relative_base_dir
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500137 relative_base_dir = file_path.get_native_path_case(relative_base_dir)
138 return dict(
139 (k, _normalize_path_variable(cwd, relative_base_dir, k, v))
140 for k, v in path_variables.iteritems())
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000141
142
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500143### Internal state files.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000144
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500145
146def isolatedfile_to_state(filename):
147 """For a '.isolate' file, returns the path to the saved '.state' file."""
148 return filename + '.state'
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000149
150
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500151def chromium_save_isolated(isolated, data, path_variables, algo):
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000152 """Writes one or many .isolated files.
153
154 This slightly increases the cold cache cost but greatly reduce the warm cache
155 cost by splitting low-churn files off the master .isolated file. It also
156 reduces overall isolateserver memcache consumption.
157 """
158 slaves = []
159
160 def extract_into_included_isolated(prefix):
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000161 new_slave = {
162 'algo': data['algo'],
163 'files': {},
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000164 'version': data['version'],
165 }
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000166 for f in data['files'].keys():
167 if f.startswith(prefix):
168 new_slave['files'][f] = data['files'].pop(f)
169 if new_slave['files']:
170 slaves.append(new_slave)
171
172 # Split test/data/ in its own .isolated file.
173 extract_into_included_isolated(os.path.join('test', 'data', ''))
174
175 # Split everything out of PRODUCT_DIR in its own .isolated file.
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500176 if path_variables.get('PRODUCT_DIR'):
177 extract_into_included_isolated(path_variables['PRODUCT_DIR'])
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000178
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000179 files = []
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000180 for index, f in enumerate(slaves):
181 slavepath = isolated[:-len('.isolated')] + '.%d.isolated' % index
Marc-Antoine Ruelde011802013-11-12 15:19:47 -0500182 tools.write_json(slavepath, f, True)
maruel@chromium.org7b844a62013-09-17 13:04:59 +0000183 data.setdefault('includes', []).append(
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400184 isolated_format.hash_file(slavepath, algo))
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000185 files.append(os.path.basename(slavepath))
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000186
Marc-Antoine Ruel52436aa2014-08-28 21:57:57 -0400187 files.extend(isolated_format.save_isolated(isolated, data))
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000188 return files
189
190
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000191class Flattenable(object):
192 """Represents data that can be represented as a json file."""
193 MEMBERS = ()
194
195 def flatten(self):
196 """Returns a json-serializable version of itself.
197
198 Skips None entries.
199 """
200 items = ((member, getattr(self, member)) for member in self.MEMBERS)
201 return dict((member, value) for member, value in items if value is not None)
202
203 @classmethod
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000204 def load(cls, data, *args, **kwargs):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000205 """Loads a flattened version."""
206 data = data.copy()
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000207 out = cls(*args, **kwargs)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000208 for member in out.MEMBERS:
209 if member in data:
210 # Access to a protected member XXX of a client class
211 # pylint: disable=W0212
212 out._load_member(member, data.pop(member))
213 if data:
214 raise ValueError(
215 'Found unexpected entry %s while constructing an object %s' %
216 (data, cls.__name__), data, cls.__name__)
217 return out
218
219 def _load_member(self, member, value):
220 """Loads a member into self."""
221 setattr(self, member, value)
222
223 @classmethod
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000224 def load_file(cls, filename, *args, **kwargs):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000225 """Loads the data from a file or return an empty instance."""
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000226 try:
Marc-Antoine Ruelde011802013-11-12 15:19:47 -0500227 out = cls.load(tools.read_json(filename), *args, **kwargs)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000228 logging.debug('Loaded %s(%s)', cls.__name__, filename)
maruel@chromium.orge9403ab2013-09-20 18:03:49 +0000229 except (IOError, ValueError) as e:
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000230 # On failure, loads the default instance.
231 out = cls(*args, **kwargs)
maruel@chromium.orge9403ab2013-09-20 18:03:49 +0000232 logging.warn('Failed to load %s: %s', filename, e)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000233 return out
234
235
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000236class SavedState(Flattenable):
237 """Describes the content of a .state file.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000238
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000239 This file caches the items calculated by this script and is used to increase
240 the performance of the script. This file is not loaded by run_isolated.py.
241 This file can always be safely removed.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000242
243 It is important to note that the 'files' dict keys are using native OS path
244 separator instead of '/' used in .isolate file.
245 """
246 MEMBERS = (
Marc-Antoine Ruel05199462014-03-13 15:40:48 -0400247 # Value of sys.platform so that the file is rejected if loaded from a
248 # different OS. While this should never happen in practice, users are ...
249 # "creative".
250 'OS',
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000251 # Algorithm used to generate the hash. The only supported value is at the
252 # time of writting 'sha-1'.
253 'algo',
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400254 # List of included .isolated files. Used to support/remember 'slave'
255 # .isolated files. Relative path to isolated_basedir.
256 'child_isolated_files',
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000257 # Cache of the processed command. This value is saved because .isolated
258 # files are never loaded by isolate.py so it's the only way to load the
259 # command safely.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000260 'command',
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500261 # GYP variables that are used to generate conditions. The most frequent
262 # example is 'OS'.
263 'config_variables',
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500264 # GYP variables that will be replaced in 'command' and paths but will not be
265 # considered a relative directory.
266 'extra_variables',
maruel@chromium.org7b844a62013-09-17 13:04:59 +0000267 # Cache of the files found so the next run can skip hash calculation.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000268 'files',
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000269 # Path of the original .isolate file. Relative path to isolated_basedir.
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000270 'isolate_file',
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400271 # GYP variables used to generate the .isolated files paths based on path
272 # variables. Frequent examples are DEPTH and PRODUCT_DIR.
273 'path_variables',
Marc-Antoine Ruel33d442a2014-10-03 14:41:51 -0400274 # If the generated directory tree should be read-only. Defaults to 1.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000275 'read_only',
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000276 # Relative cwd to use to start the command.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000277 'relative_cwd',
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400278 # Root directory the files are mapped from.
279 'root_dir',
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400280 # Version of the saved state file format. Any breaking change must update
281 # the value.
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000282 'version',
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000283 )
284
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400285 # Bump this version whenever the saved state changes. It is also keyed on the
286 # .isolated file version so any change in the generator will invalidate .state
287 # files.
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400288 EXPECTED_VERSION = isolated_format.ISOLATED_FILE_VERSION + '.2'
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400289
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000290 def __init__(self, isolated_basedir):
291 """Creates an empty SavedState.
292
Marc-Antoine Ruel8472efa2014-03-18 14:32:50 -0400293 Arguments:
294 isolated_basedir: the directory where the .isolated and .isolated.state
295 files are saved.
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000296 """
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000297 super(SavedState, self).__init__()
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000298 assert os.path.isabs(isolated_basedir), isolated_basedir
299 assert os.path.isdir(isolated_basedir), isolated_basedir
300 self.isolated_basedir = isolated_basedir
301
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000302 # The default algorithm used.
Marc-Antoine Ruel05199462014-03-13 15:40:48 -0400303 self.OS = sys.platform
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400304 self.algo = isolated_format.SUPPORTED_ALGOS['sha-1']
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500305 self.child_isolated_files = []
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000306 self.command = []
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500307 self.config_variables = {}
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500308 self.extra_variables = {}
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000309 self.files = {}
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000310 self.isolate_file = None
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500311 self.path_variables = {}
Marc-Antoine Ruel33d442a2014-10-03 14:41:51 -0400312 # Defaults to 1 when compiling to .isolated.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000313 self.read_only = None
314 self.relative_cwd = None
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400315 self.root_dir = None
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400316 self.version = self.EXPECTED_VERSION
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000317
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400318 def update_config(self, config_variables):
319 """Updates the saved state with only config variables."""
320 self.config_variables.update(config_variables)
321
322 def update(self, isolate_file, path_variables, extra_variables):
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000323 """Updates the saved state with new data to keep GYP variables and internal
324 reference to the original .isolate file.
325 """
maruel@chromium.orge99c1512013-04-09 20:24:11 +0000326 assert os.path.isabs(isolate_file)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000327 # Convert back to a relative path. On Windows, if the isolate and
328 # isolated files are on different drives, isolate_file will stay an absolute
329 # path.
Marc-Antoine Ruel37989932013-11-19 16:28:08 -0500330 isolate_file = file_path.safe_relpath(isolate_file, self.isolated_basedir)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000331
332 # The same .isolate file should always be used to generate the .isolated and
333 # .isolated.state.
334 assert isolate_file == self.isolate_file or not self.isolate_file, (
335 isolate_file, self.isolate_file)
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500336 self.extra_variables.update(extra_variables)
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000337 self.isolate_file = isolate_file
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500338 self.path_variables.update(path_variables)
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000339
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400340 def update_isolated(self, command, infiles, read_only, relative_cwd):
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000341 """Updates the saved state with data necessary to generate a .isolated file.
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000342
maruel@chromium.org7b844a62013-09-17 13:04:59 +0000343 The new files in |infiles| are added to self.files dict but their hash is
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000344 not calculated here.
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000345 """
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000346 self.command = command
347 # Add new files.
348 for f in infiles:
349 self.files.setdefault(f, {})
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000350 # Prune extraneous files that are not a dependency anymore.
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400351 for f in set(self.files).difference(set(infiles)):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000352 del self.files[f]
353 if read_only is not None:
354 self.read_only = read_only
355 self.relative_cwd = relative_cwd
356
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000357 def to_isolated(self):
358 """Creates a .isolated dictionary out of the saved state.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000359
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000360 https://code.google.com/p/swarming/wiki/IsolatedDesign
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000361 """
362 def strip(data):
363 """Returns a 'files' entry with only the whitelisted keys."""
364 return dict((k, data[k]) for k in ('h', 'l', 'm', 's') if k in data)
365
366 out = {
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400367 'algo': isolated_format.SUPPORTED_ALGOS_REVERSE[self.algo],
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000368 'files': dict(
369 (filepath, strip(data)) for filepath, data in self.files.iteritems()),
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400370 # The version of the .state file is different than the one of the
371 # .isolated file.
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400372 'version': isolated_format.ISOLATED_FILE_VERSION,
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000373 }
374 if self.command:
375 out['command'] = self.command
Marc-Antoine Ruel33d442a2014-10-03 14:41:51 -0400376 out['read_only'] = self.read_only if self.read_only is not None else 1
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000377 if self.relative_cwd:
378 out['relative_cwd'] = self.relative_cwd
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000379 return out
380
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000381 @property
382 def isolate_filepath(self):
383 """Returns the absolute path of self.isolate_file."""
384 return os.path.normpath(
385 os.path.join(self.isolated_basedir, self.isolate_file))
386
387 # Arguments number differs from overridden method
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000388 @classmethod
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000389 def load(cls, data, isolated_basedir): # pylint: disable=W0221
390 """Special case loading to disallow different OS.
391
392 It is not possible to load a .isolated.state files from a different OS, this
393 file is saved in OS-specific format.
394 """
395 out = super(SavedState, cls).load(data, isolated_basedir)
Marc-Antoine Ruel05199462014-03-13 15:40:48 -0400396 if data.get('OS') != sys.platform:
Marc-Antoine Ruel1e7658c2014-08-28 19:46:39 -0400397 raise isolated_format.IsolatedError('Unexpected OS %s', data.get('OS'))
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000398
399 # Converts human readable form back into the proper class type.
Marc-Antoine Ruel05199462014-03-13 15:40:48 -0400400 algo = data.get('algo')
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400401 if not algo in isolated_format.SUPPORTED_ALGOS:
Marc-Antoine Ruel1e7658c2014-08-28 19:46:39 -0400402 raise isolated_format.IsolatedError('Unknown algo \'%s\'' % out.algo)
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400403 out.algo = isolated_format.SUPPORTED_ALGOS[algo]
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000404
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500405 # Refuse the load non-exact version, even minor difference. This is unlike
406 # isolateserver.load_isolated(). This is because .isolated.state could have
407 # changed significantly even in minor version difference.
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400408 if out.version != cls.EXPECTED_VERSION:
Marc-Antoine Ruel1e7658c2014-08-28 19:46:39 -0400409 raise isolated_format.IsolatedError(
maruel@chromium.org999a1fd2013-09-20 17:41:07 +0000410 'Unsupported version \'%s\'' % out.version)
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000411
Marc-Antoine Ruel16ebc2e2014-02-13 15:39:15 -0500412 # The .isolate file must be valid. If it is not present anymore, zap the
413 # value as if it was not noted, so .isolate_file can safely be overriden
414 # later.
415 if out.isolate_file and not os.path.isfile(out.isolate_filepath):
416 out.isolate_file = None
417 if out.isolate_file:
418 # It could be absolute on Windows if the drive containing the .isolate and
419 # the drive containing the .isolated files differ, .e.g .isolate is on
420 # C:\\ and .isolated is on D:\\ .
421 assert not os.path.isabs(out.isolate_file) or sys.platform == 'win32'
422 assert os.path.isfile(out.isolate_filepath), out.isolate_filepath
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000423 return out
424
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000425 def flatten(self):
426 """Makes sure 'algo' is in human readable form."""
427 out = super(SavedState, self).flatten()
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400428 out['algo'] = isolated_format.SUPPORTED_ALGOS_REVERSE[out['algo']]
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000429 return out
430
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000431 def __str__(self):
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500432 def dict_to_str(d):
433 return ''.join('\n %s=%s' % (k, d[k]) for k in sorted(d))
434
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000435 out = '%s(\n' % self.__class__.__name__
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000436 out += ' command: %s\n' % self.command
437 out += ' files: %d\n' % len(self.files)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000438 out += ' isolate_file: %s\n' % self.isolate_file
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000439 out += ' read_only: %s\n' % self.read_only
maruel@chromium.org9e9ceaa2013-04-05 15:42:42 +0000440 out += ' relative_cwd: %s\n' % self.relative_cwd
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000441 out += ' child_isolated_files: %s\n' % self.child_isolated_files
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500442 out += ' path_variables: %s\n' % dict_to_str(self.path_variables)
443 out += ' config_variables: %s\n' % dict_to_str(self.config_variables)
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500444 out += ' extra_variables: %s\n' % dict_to_str(self.extra_variables)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000445 return out
446
447
448class CompleteState(object):
449 """Contains all the state to run the task at hand."""
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000450 def __init__(self, isolated_filepath, saved_state):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000451 super(CompleteState, self).__init__()
maruel@chromium.org29029882013-08-30 12:15:40 +0000452 assert isolated_filepath is None or os.path.isabs(isolated_filepath)
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000453 self.isolated_filepath = isolated_filepath
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000454 # Contains the data to ease developer's use-case but that is not strictly
455 # necessary.
456 self.saved_state = saved_state
457
458 @classmethod
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000459 def load_files(cls, isolated_filepath):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000460 """Loads state from disk."""
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000461 assert os.path.isabs(isolated_filepath), isolated_filepath
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000462 isolated_basedir = os.path.dirname(isolated_filepath)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000463 return cls(
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000464 isolated_filepath,
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000465 SavedState.load_file(
466 isolatedfile_to_state(isolated_filepath), isolated_basedir))
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000467
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500468 def load_isolate(
469 self, cwd, isolate_file, path_variables, config_variables,
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500470 extra_variables, ignore_broken_items):
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000471 """Updates self.isolated and self.saved_state with information loaded from a
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000472 .isolate file.
473
474 Processes the loaded data, deduce root_dir, relative_cwd.
475 """
476 # Make sure to not depend on os.getcwd().
477 assert os.path.isabs(isolate_file), isolate_file
maruel@chromium.org561d4b22013-09-26 21:08:08 +0000478 isolate_file = file_path.get_native_path_case(isolate_file)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000479 logging.info(
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500480 'CompleteState.load_isolate(%s, %s, %s, %s, %s, %s)',
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500481 cwd, isolate_file, path_variables, config_variables, extra_variables,
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500482 ignore_broken_items)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000483
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400484 # Config variables are not affected by the paths and must be used to
485 # retrieve the paths, so update them first.
486 self.saved_state.update_config(config_variables)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000487
488 with open(isolate_file, 'r') as f:
489 # At that point, variables are not replaced yet in command and infiles.
490 # infiles may contain directory entries and is in posix style.
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400491 command, infiles, read_only, isolate_cmd_dir = (
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500492 isolate_format.load_isolate_for_config(
493 os.path.dirname(isolate_file), f.read(),
494 self.saved_state.config_variables))
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500495
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400496 # Processes the variables with the new found relative root. Note that 'cwd'
497 # is used when path variables are used.
498 path_variables = normalize_path_variables(
499 cwd, path_variables, isolate_cmd_dir)
500 # Update the rest of the saved state.
501 self.saved_state.update(isolate_file, path_variables, extra_variables)
502
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500503 total_variables = self.saved_state.path_variables.copy()
504 total_variables.update(self.saved_state.config_variables)
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500505 total_variables.update(self.saved_state.extra_variables)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500506 command = [
507 isolate_format.eval_variables(i, total_variables) for i in command
508 ]
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500509
510 total_variables = self.saved_state.path_variables.copy()
511 total_variables.update(self.saved_state.extra_variables)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500512 infiles = [
513 isolate_format.eval_variables(f, total_variables) for f in infiles
514 ]
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000515 # root_dir is automatically determined by the deepest root accessed with the
maruel@chromium.org75584e22013-06-20 01:40:24 +0000516 # form '../../foo/bar'. Note that path variables must be taken in account
517 # too, add them as if they were input files.
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400518 self.saved_state.root_dir = isolate_format.determine_root_dir(
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400519 isolate_cmd_dir, infiles + self.saved_state.path_variables.values())
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000520 # The relative directory is automatically determined by the relative path
521 # between root_dir and the directory containing the .isolate file,
522 # isolate_base_dir.
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400523 relative_cwd = os.path.relpath(isolate_cmd_dir, self.saved_state.root_dir)
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500524 # Now that we know where the root is, check that the path_variables point
benrg@chromium.org9ae72862013-02-11 05:05:51 +0000525 # inside it.
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500526 for k, v in self.saved_state.path_variables.iteritems():
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400527 dest = os.path.join(isolate_cmd_dir, relative_cwd, v)
528 if not file_path.path_starts_with(self.saved_state.root_dir, dest):
Marc-Antoine Ruel1e7658c2014-08-28 19:46:39 -0400529 raise isolated_format.MappingError(
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400530 'Path variable %s=%r points outside the inferred root directory '
531 '%s; %s'
532 % (k, v, self.saved_state.root_dir, dest))
533 # Normalize the files based to self.saved_state.root_dir. It is important to
534 # keep the trailing os.path.sep at that step.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000535 infiles = [
Marc-Antoine Ruel37989932013-11-19 16:28:08 -0500536 file_path.relpath(
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400537 file_path.normpath(os.path.join(isolate_cmd_dir, f)),
538 self.saved_state.root_dir)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000539 for f in infiles
540 ]
Marc-Antoine Ruel05199462014-03-13 15:40:48 -0400541 follow_symlinks = sys.platform != 'win32'
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000542 # Expand the directories by listing each file inside. Up to now, trailing
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400543 # os.path.sep must be kept.
Marc-Antoine Ruel92257792014-08-28 20:51:08 -0400544 infiles = isolated_format.expand_directories_and_symlinks(
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400545 self.saved_state.root_dir,
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000546 infiles,
csharp@chromium.org01856802012-11-12 17:48:13 +0000547 lambda x: re.match(r'.*\.(git|svn|pyc)$', x),
csharp@chromium.org84d2e2e2013-03-27 13:38:42 +0000548 follow_symlinks,
csharp@chromium.org01856802012-11-12 17:48:13 +0000549 ignore_broken_items)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000550
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000551 # Finally, update the new data to be able to generate the foo.isolated file,
552 # the file that is used by run_isolated.py.
Marc-Antoine Ruel6b1084e2014-09-30 15:14:58 -0400553 self.saved_state.update_isolated(command, infiles, read_only, relative_cwd)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000554 logging.debug(self)
555
Marc-Antoine Ruel92257792014-08-28 20:51:08 -0400556 def files_to_metadata(self, subdir):
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000557 """Updates self.saved_state.files with the files' mode and hash.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000558
maruel@chromium.org9268f042012-10-17 17:36:41 +0000559 If |subdir| is specified, filters to a subdirectory. The resulting .isolated
560 file is tainted.
561
Marc-Antoine Ruel92257792014-08-28 20:51:08 -0400562 See isolated_format.file_to_metadata() for more information.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000563 """
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000564 for infile in sorted(self.saved_state.files):
maruel@chromium.org9268f042012-10-17 17:36:41 +0000565 if subdir and not infile.startswith(subdir):
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000566 self.saved_state.files.pop(infile)
maruel@chromium.org9268f042012-10-17 17:36:41 +0000567 else:
568 filepath = os.path.join(self.root_dir, infile)
Marc-Antoine Ruel92257792014-08-28 20:51:08 -0400569 self.saved_state.files[infile] = isolated_format.file_to_metadata(
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000570 filepath,
571 self.saved_state.files[infile],
maruel@chromium.orgbaa108d2013-03-28 13:24:51 +0000572 self.saved_state.read_only,
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000573 self.saved_state.algo)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000574
575 def save_files(self):
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000576 """Saves self.saved_state and creates a .isolated file."""
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000577 logging.debug('Dumping to %s' % self.isolated_filepath)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000578 self.saved_state.child_isolated_files = chromium_save_isolated(
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000579 self.isolated_filepath,
580 self.saved_state.to_isolated(),
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500581 self.saved_state.path_variables,
maruel@chromium.org385d73d2013-09-19 18:33:21 +0000582 self.saved_state.algo)
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000583 total_bytes = sum(
584 i.get('s', 0) for i in self.saved_state.files.itervalues())
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000585 if total_bytes:
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000586 # TODO(maruel): Stats are missing the .isolated files.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000587 logging.debug('Total size: %d bytes' % total_bytes)
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000588 saved_state_file = isolatedfile_to_state(self.isolated_filepath)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000589 logging.debug('Dumping to %s' % saved_state_file)
Marc-Antoine Ruelde011802013-11-12 15:19:47 -0500590 tools.write_json(saved_state_file, self.saved_state.flatten(), True)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000591
592 @property
593 def root_dir(self):
Marc-Antoine Rueledf28952014-03-31 19:55:47 -0400594 return self.saved_state.root_dir
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000595
596 def __str__(self):
597 def indent(data, indent_length):
598 """Indents text."""
599 spacing = ' ' * indent_length
600 return ''.join(spacing + l for l in str(data).splitlines(True))
601
602 out = '%s(\n' % self.__class__.__name__
603 out += ' root_dir: %s\n' % self.root_dir
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000604 out += ' saved_state: %s)' % indent(self.saved_state, 2)
605 return out
606
607
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000608def load_complete_state(options, cwd, subdir, skip_update):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000609 """Loads a CompleteState.
610
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000611 This includes data from .isolate and .isolated.state files. Never reads the
612 .isolated file.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000613
614 Arguments:
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700615 options: Options instance generated with process_isolate_options. For either
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000616 options.isolate and options.isolated, if the value is set, it is an
617 absolute path.
618 cwd: base directory to be used when loading the .isolate file.
619 subdir: optional argument to only process file in the subdirectory, relative
620 to CompleteState.root_dir.
621 skip_update: Skip trying to load the .isolate file and processing the
622 dependencies. It is useful when not needed, like when tracing.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000623 """
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000624 assert not options.isolate or os.path.isabs(options.isolate)
625 assert not options.isolated or os.path.isabs(options.isolated)
maruel@chromium.org561d4b22013-09-26 21:08:08 +0000626 cwd = file_path.get_native_path_case(unicode(cwd))
maruel@chromium.org0cd0b182012-10-22 13:34:15 +0000627 if options.isolated:
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000628 # Load the previous state if it was present. Namely, "foo.isolated.state".
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000629 # Note: this call doesn't load the .isolate file.
maruel@chromium.org0cd0b182012-10-22 13:34:15 +0000630 complete_state = CompleteState.load_files(options.isolated)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000631 else:
632 # Constructs a dummy object that cannot be saved. Useful for temporary
Marc-Antoine Ruel8472efa2014-03-18 14:32:50 -0400633 # commands like 'run'. There is no directory containing a .isolated file so
634 # specify the current working directory as a valid directory.
635 complete_state = CompleteState(None, SavedState(os.getcwd()))
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000636
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000637 if not options.isolate:
638 if not complete_state.saved_state.isolate_file:
639 if not skip_update:
640 raise ExecutionError('A .isolate file is required.')
641 isolate = None
642 else:
643 isolate = complete_state.saved_state.isolate_filepath
644 else:
645 isolate = options.isolate
646 if complete_state.saved_state.isolate_file:
Marc-Antoine Ruel37989932013-11-19 16:28:08 -0500647 rel_isolate = file_path.safe_relpath(
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000648 options.isolate, complete_state.saved_state.isolated_basedir)
649 if rel_isolate != complete_state.saved_state.isolate_file:
Marc-Antoine Ruel8472efa2014-03-18 14:32:50 -0400650 # This happens if the .isolate file was moved for example. In this case,
651 # discard the saved state.
652 logging.warning(
653 '--isolated %s != %s as saved in %s. Discarding saved state',
654 rel_isolate,
655 complete_state.saved_state.isolate_file,
656 isolatedfile_to_state(options.isolated))
657 complete_state = CompleteState(
658 options.isolated,
659 SavedState(complete_state.saved_state.isolated_basedir))
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000660
661 if not skip_update:
662 # Then load the .isolate and expands directories.
663 complete_state.load_isolate(
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500664 cwd, isolate, options.path_variables, options.config_variables,
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -0500665 options.extra_variables, options.ignore_broken_items)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000666
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000667 # Regenerate complete_state.saved_state.files.
maruel@chromium.org9268f042012-10-17 17:36:41 +0000668 if subdir:
maruel@chromium.org306e0e72012-11-02 18:22:03 +0000669 subdir = unicode(subdir)
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500670 # This is tricky here. If it is a path, take it from the root_dir. If
671 # it is a variable, it must be keyed from the directory containing the
672 # .isolate file. So translate all variables first.
673 translated_path_variables = dict(
674 (k,
675 os.path.normpath(os.path.join(complete_state.saved_state.relative_cwd,
676 v)))
677 for k, v in complete_state.saved_state.path_variables.iteritems())
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500678 subdir = isolate_format.eval_variables(subdir, translated_path_variables)
maruel@chromium.org9268f042012-10-17 17:36:41 +0000679 subdir = subdir.replace('/', os.path.sep)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000680
681 if not skip_update:
Marc-Antoine Ruel92257792014-08-28 20:51:08 -0400682 complete_state.files_to_metadata(subdir)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000683 return complete_state
684
685
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500686def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
687 """Creates a isolated tree usable for test execution.
688
689 Returns the current working directory where the isolated command should be
690 started in.
691 """
Marc-Antoine Ruel361bfda2014-01-15 15:26:39 -0500692 # Forcibly copy when the tree has to be read only. Otherwise the inode is
693 # modified, and this cause real problems because the user's source tree
694 # becomes read only. On the other hand, the cost of doing file copy is huge.
695 if read_only not in (0, None):
696 action = run_isolated.COPY
697 else:
698 action = run_isolated.HARDLINK_WITH_FALLBACK
699
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500700 recreate_tree(
701 outdir=outdir,
702 indir=root_dir,
703 infiles=files,
Marc-Antoine Ruel361bfda2014-01-15 15:26:39 -0500704 action=action,
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500705 as_hash=False)
706 cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
707 if not os.path.isdir(cwd):
708 # It can happen when no files are mapped from the directory containing the
709 # .isolate file. But the directory must exist to be the current working
710 # directory.
711 os.makedirs(cwd)
712 run_isolated.change_tree_read_only(outdir, read_only)
713 return cwd
714
715
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500716def prepare_for_archival(options, cwd):
717 """Loads the isolated file and create 'infiles' for archival."""
718 complete_state = load_complete_state(
719 options, cwd, options.subdir, False)
720 # Make sure that complete_state isn't modified until save_files() is
721 # called, because any changes made to it here will propagate to the files
722 # created (which is probably not intended).
723 complete_state.save_files()
724
725 infiles = complete_state.saved_state.files
726 # Add all the .isolated files.
727 isolated_hash = []
728 isolated_files = [
729 options.isolated,
730 ] + complete_state.saved_state.child_isolated_files
731 for item in isolated_files:
732 item_path = os.path.join(
733 os.path.dirname(complete_state.isolated_filepath), item)
Marc-Antoine Ruel8bee66d2014-08-28 19:02:07 -0400734 # Do not use isolated_format.hash_file() here because the file is
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500735 # likely smallish (under 500kb) and its file size is needed.
736 with open(item_path, 'rb') as f:
737 content = f.read()
738 isolated_hash.append(
739 complete_state.saved_state.algo(content).hexdigest())
740 isolated_metadata = {
741 'h': isolated_hash[-1],
742 's': len(content),
743 'priority': '0'
744 }
745 infiles[item_path] = isolated_metadata
746 return complete_state, infiles, isolated_hash
747
748
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700749def isolate_and_archive(options, cwd, isolate_server, namespace):
750 """Isolates and uploads isolated tree.
maruel@chromium.org29029882013-08-30 12:15:40 +0000751
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700752 Returns hash of the *.isolated file on success, None on failure.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000753 """
vadimsh@chromium.orga4326472013-08-24 02:05:41 +0000754 with tools.Profiler('GenerateHashtable'):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000755 success = False
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700756 isolated_hash = None
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000757 try:
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500758 complete_state, infiles, isolated_hash = prepare_for_archival(
759 options, cwd)
maruel@chromium.orgd3a17762012-12-13 14:17:15 +0000760 logging.info('Creating content addressed object store with %d item',
761 len(infiles))
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000762
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500763 isolateserver.upload_tree(
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700764 base_url=isolate_server,
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500765 indir=complete_state.root_dir,
766 infiles=infiles,
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700767 namespace=namespace)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000768 success = True
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000769 print('%s %s' % (isolated_hash[0], os.path.basename(options.isolated)))
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700770 except Exception:
771 logging.exception(
772 'Exception when archiving %s', os.path.basename(options.isolated))
773 success = False
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000774 finally:
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000775 # If the command failed, delete the .isolated file if it exists. This is
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000776 # important so no stale swarm job is executed.
maruel@chromium.org0cd0b182012-10-22 13:34:15 +0000777 if not success and os.path.isfile(options.isolated):
778 os.remove(options.isolated)
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700779 return isolated_hash[0] if success else None
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000780
781
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700782def parse_archive_command_line(args, cwd):
783 """Given list of arguments for 'archive' command returns parsed options.
784
785 Used by CMDbatcharchive to parse options passed via JSON. See also CMDarchive.
786 """
787 parser = optparse.OptionParser()
788 add_isolate_options(parser)
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500789 add_subdir_option(parser)
maruel@chromium.org2f952d82013-09-13 01:53:17 +0000790 options, args = parser.parse_args(args)
791 if args:
792 parser.error('Unsupported argument: %s' % args)
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700793 process_isolate_options(parser, options, cwd)
794 return options
795
796
797### Commands.
798
799
800def CMDarchive(parser, args):
801 """Creates a .isolated file and uploads the tree to an isolate server.
802
803 All the files listed in the .isolated file are put in the isolate server
804 cache via isolateserver.py.
805 """
806 add_isolate_options(parser)
807 add_subdir_option(parser)
808 isolateserver.add_isolate_server_options(parser, False)
809 auth.add_auth_options(parser)
810 options, args = parser.parse_args(args)
811 process_isolate_options(parser, options)
812 auth.process_auth_options(parser, options)
813 isolateserver.process_isolate_server_options(parser, options)
814 if args:
815 parser.error('Unsupported argument: %s' % args)
816 if not file_path.is_url(options.isolate_server):
817 parser.error('Not a valid server URL: %s' % options.isolate_server)
818 auth.ensure_logged_in(options.isolate_server)
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700819 isolated_hash = isolate_and_archive(
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700820 options, os.getcwd(), options.isolate_server, options.namespace)
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700821 return int(not isolated_hash)
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700822
823
824@subcommand.usage('-- GEN_JSON_1 GEN_JSON_2 ...')
825def CMDbatcharchive(parser, args):
826 """Archives multiple isolated trees at once.
827
828 Using single command instead of multiple sequential invocations allows to cut
829 redundant work when isolated trees share common files (e.g. file hashes are
830 checked only once, their presence on the server is checked only once, and
831 so on).
832
833 Takes a list of paths to *.isolated.gen.json files that describe what trees to
834 isolate. Format of files is:
835 {
836 'version': 1,
837 'dir': <absolute path to a directory all other paths are relative to>,
838 'arg': [list of command line arguments for single 'archive' command]
839 }
840 """
841 isolateserver.add_isolate_server_options(parser, False)
842 auth.add_auth_options(parser)
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700843 parser.add_option(
844 '--dump-json',
845 metavar='FILE',
846 help='Write isolated hashes of archived trees to this file as JSON')
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700847 options, args = parser.parse_args(args)
848 auth.process_auth_options(parser, options)
849 isolateserver.process_isolate_server_options(parser, options)
850 if not file_path.is_url(options.isolate_server):
851 parser.error('Not a valid server URL: %s' % options.isolate_server)
852 auth.ensure_logged_in(options.isolate_server)
853
854 # Validate all incoming options, prepare what needs to be archived as a list
855 # of tuples (working directory, archival options).
856 work_units = []
857 for gen_json_path in args:
858 # Validate JSON format of a *.isolated.gen.json file.
859 data = tools.read_json(gen_json_path)
860 if data.get('version') != ISOLATED_GEN_JSON_VERSION:
861 parser.error('Invalid version in %s' % gen_json_path)
862 cwd = data.get('dir')
863 if not isinstance(cwd, unicode) or not os.path.isdir(cwd):
864 parser.error('Invalid dir in %s' % gen_json_path)
865 args = data.get('args')
866 if (not isinstance(args, list) or
867 not all(isinstance(x, unicode) for x in args)):
868 parser.error('Invalid args in %s' % gen_json_path)
869 # Convert command line (embedded in JSON) to Options object.
870 work_units.append((cwd, parse_archive_command_line(args, cwd)))
871
872 # Perform the archival.
873 # TODO(vadimsh): Start optimizing this by removing redundant work.
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700874 isolated_hashes = {}
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700875 for cwd, opts in work_units:
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700876 target_name = os.path.splitext(os.path.basename(opts.isolated))[0]
877 isolated_hashes[target_name] = isolate_and_archive(
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700878 opts, cwd, options.isolate_server, options.namespace)
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700879
Vadim Shtayuraf4e9ccb2014-10-01 21:24:53 -0700880 if options.dump_json:
881 tools.write_json(options.dump_json, isolated_hashes, False)
882
883 # 'isolate_and_archive' returns None on failure. At least one None -> failure.
884 return int(not all(isolated_hashes.itervalues()))
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700885
886
887def CMDcheck(parser, args):
888 """Checks that all the inputs are present and generates .isolated."""
889 add_isolate_options(parser)
890 add_subdir_option(parser)
891 options, args = parser.parse_args(args)
892 if args:
893 parser.error('Unsupported argument: %s' % args)
894 process_isolate_options(parser, options)
maruel@chromium.org2f952d82013-09-13 01:53:17 +0000895
896 complete_state = load_complete_state(
897 options, os.getcwd(), options.subdir, False)
898
899 # Nothing is done specifically. Just store the result and state.
900 complete_state.save_files()
901 return 0
902
903
maruel@chromium.orge5322512013-08-19 20:17:57 +0000904def CMDremap(parser, args):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000905 """Creates a directory with all the dependencies mapped into it.
906
907 Useful to test manually why a test is failing. The target executable is not
908 run.
909 """
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700910 add_isolate_options(parser)
Marc-Antoine Ruele236b5c2014-09-08 18:40:40 -0400911 add_outdir_options(parser)
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500912 add_skip_refresh_option(parser)
maruel@chromium.org9268f042012-10-17 17:36:41 +0000913 options, args = parser.parse_args(args)
914 if args:
915 parser.error('Unsupported argument: %s' % args)
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500916 cwd = os.getcwd()
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700917 process_isolate_options(parser, options, cwd, require_isolated=False)
Marc-Antoine Ruele236b5c2014-09-08 18:40:40 -0400918 process_outdir_options(parser, options, cwd)
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500919 complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000920
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500921 if not os.path.isdir(options.outdir):
922 os.makedirs(options.outdir)
923 print('Remapping into %s' % options.outdir)
924 if os.listdir(options.outdir):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000925 raise ExecutionError('Can\'t remap in a non-empty directory')
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000926
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500927 create_isolate_tree(
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500928 options.outdir, complete_state.root_dir, complete_state.saved_state.files,
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500929 complete_state.saved_state.relative_cwd,
930 complete_state.saved_state.read_only)
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000931 if complete_state.isolated_filepath:
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000932 complete_state.save_files()
933 return 0
934
935
maruel@chromium.org29029882013-08-30 12:15:40 +0000936@subcommand.usage('-- [extra arguments]')
maruel@chromium.orge5322512013-08-19 20:17:57 +0000937def CMDrun(parser, args):
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000938 """Runs the test executable in an isolated (temporary) directory.
939
940 All the dependencies are mapped into the temporary directory and the
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500941 directory is cleaned up after the target exits.
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000942
maruel@chromium.org29029882013-08-30 12:15:40 +0000943 Argument processing stops at -- and these arguments are appended to the
944 command line of the target to run. For example, use:
945 isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000946 """
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700947 add_isolate_options(parser)
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500948 add_skip_refresh_option(parser)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000949 options, args = parser.parse_args(args)
Vadim Shtayurafddb1432014-09-30 18:32:41 -0700950 process_isolate_options(parser, options, require_isolated=False)
maruel@chromium.org8abec8b2013-04-16 19:34:20 +0000951 complete_state = load_complete_state(
952 options, os.getcwd(), None, options.skip_refresh)
maruel@chromium.orgf75b0cb2012-12-11 21:39:00 +0000953 cmd = complete_state.saved_state.command + args
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000954 if not cmd:
maruel@chromium.org29029882013-08-30 12:15:40 +0000955 raise ExecutionError('No command to run.')
vadimsh@chromium.orga4326472013-08-24 02:05:41 +0000956 cmd = tools.fix_python_path(cmd)
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500957
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500958 outdir = run_isolated.make_temp_dir(
959 'isolate-%s' % datetime.date.today(),
960 os.path.dirname(complete_state.root_dir))
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000961 try:
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500962 # TODO(maruel): Use run_isolated.run_tha_test().
Marc-Antoine Ruel1e9ad0c2014-01-14 15:20:33 -0500963 cwd = create_isolate_tree(
964 outdir, complete_state.root_dir, complete_state.saved_state.files,
965 complete_state.saved_state.relative_cwd,
966 complete_state.saved_state.read_only)
John Abd-El-Malek3f998682014-09-17 17:48:09 -0700967 file_path.ensure_command_has_abs_path(cmd, cwd)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000968 logging.info('Running %s, cwd=%s' % (cmd, cwd))
Marc-Antoine Ruel926dccd2014-09-17 13:40:24 -0400969 try:
970 result = subprocess.call(cmd, cwd=cwd)
971 except OSError:
972 sys.stderr.write(
973 'Failed to executed the command; executable is missing, maybe you\n'
974 'forgot to map it in the .isolate file?\n %s\n in %s\n' %
975 (' '.join(cmd), cwd))
976 result = 1
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000977 finally:
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -0500978 run_isolated.rmtree(outdir)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000979
maruel@chromium.org4b57f692012-10-05 20:33:09 +0000980 if complete_state.isolated_filepath:
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +0000981 complete_state.save_files()
982 return result
983
984
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500985def _process_variable_arg(option, opt, _value, parser):
986 """Called by OptionParser to process a --<foo>-variable argument."""
maruel@chromium.org712454d2013-04-04 17:52:34 +0000987 if not parser.rargs:
988 raise optparse.OptionValueError(
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500989 'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
maruel@chromium.org712454d2013-04-04 17:52:34 +0000990 k = parser.rargs.pop(0)
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500991 variables = getattr(parser.values, option.dest)
maruel@chromium.org712454d2013-04-04 17:52:34 +0000992 if '=' in k:
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500993 k, v = k.split('=', 1)
maruel@chromium.org712454d2013-04-04 17:52:34 +0000994 else:
995 if not parser.rargs:
996 raise optparse.OptionValueError(
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -0500997 'Please use %s FOO=BAR or %s FOO BAR' % (opt, opt))
maruel@chromium.org712454d2013-04-04 17:52:34 +0000998 v = parser.rargs.pop(0)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500999 if not re.match('^' + isolate_format.VALID_VARIABLE + '$', k):
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001000 raise optparse.OptionValueError(
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -05001001 'Variable \'%s\' doesn\'t respect format \'%s\'' %
1002 (k, isolate_format.VALID_VARIABLE))
Marc-Antoine Ruel9cc42c32013-12-11 09:35:55 -05001003 variables.append((k, v.decode('utf-8')))
maruel@chromium.org712454d2013-04-04 17:52:34 +00001004
1005
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001006def add_variable_option(parser):
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001007 """Adds --isolated and --<foo>-variable to an OptionParser."""
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001008 parser.add_option(
1009 '-s', '--isolated',
1010 metavar='FILE',
1011 help='.isolated file to generate or read')
1012 # Keep for compatibility. TODO(maruel): Remove once not used anymore.
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001013 parser.add_option(
1014 '-r', '--result',
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001015 dest='isolated',
1016 help=optparse.SUPPRESS_HELP)
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001017 is_win = sys.platform in ('win32', 'cygwin')
1018 # There is really 3 kind of variables:
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001019 # - path variables, like DEPTH or PRODUCT_DIR that should be
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001020 # replaced opportunistically when tracing tests.
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001021 # - extraneous things like EXECUTABE_SUFFIX.
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001022 # - configuration variables that are to be used in deducing the matrix to
1023 # reduce.
1024 # - unrelated variables that are used as command flags for example.
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001025 parser.add_option(
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001026 '--config-variable',
maruel@chromium.org712454d2013-04-04 17:52:34 +00001027 action='callback',
1028 callback=_process_variable_arg,
Marc-Antoine Ruel05199462014-03-13 15:40:48 -04001029 default=[],
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001030 dest='config_variables',
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001031 metavar='FOO BAR',
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001032 help='Config variables are used to determine which conditions should be '
1033 'matched when loading a .isolate file, default: %default. '
1034 'All 3 kinds of variables are persistent accross calls, they are '
1035 'saved inside <.isolated>.state')
1036 parser.add_option(
1037 '--path-variable',
1038 action='callback',
1039 callback=_process_variable_arg,
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001040 default=[],
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001041 dest='path_variables',
1042 metavar='FOO BAR',
1043 help='Path variables are used to replace file paths when loading a '
1044 '.isolate file, default: %default')
1045 parser.add_option(
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001046 '--extra-variable',
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001047 action='callback',
1048 callback=_process_variable_arg,
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001049 default=[('EXECUTABLE_SUFFIX', '.exe' if is_win else '')],
1050 dest='extra_variables',
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001051 metavar='FOO BAR',
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001052 help='Extraneous variables are replaced on the \'command\' entry and on '
1053 'paths in the .isolate file but are not considered relative paths.')
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001054
1055
Vadim Shtayurafddb1432014-09-30 18:32:41 -07001056def add_isolate_options(parser):
1057 """Adds --isolate, --isolated, --out and --<foo>-variable options."""
1058 group = optparse.OptionGroup(parser, 'Common options')
1059 group.add_option(
1060 '-i', '--isolate',
1061 metavar='FILE',
1062 help='.isolate file to load the dependency data from')
1063 add_variable_option(group)
1064 group.add_option(
1065 '--ignore_broken_items', action='store_true',
1066 default=bool(os.environ.get('ISOLATE_IGNORE_BROKEN_ITEMS')),
1067 help='Indicates that invalid entries in the isolated file to be '
1068 'only be logged and not stop processing. Defaults to True if '
1069 'env var ISOLATE_IGNORE_BROKEN_ITEMS is set')
1070 parser.add_option_group(group)
1071
1072
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -05001073def add_subdir_option(parser):
1074 parser.add_option(
1075 '--subdir',
1076 help='Filters to a subdirectory. Its behavior changes depending if it '
1077 'is a relative path as a string or as a path variable. Path '
1078 'variables are always keyed from the directory containing the '
1079 '.isolate file. Anything else is keyed on the root directory.')
1080
1081
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -05001082def add_skip_refresh_option(parser):
1083 parser.add_option(
1084 '--skip-refresh', action='store_true',
1085 help='Skip reading .isolate file and do not refresh the hash of '
1086 'dependencies')
1087
Marc-Antoine Ruelf9538ee2014-01-30 10:43:54 -05001088
Marc-Antoine Ruele236b5c2014-09-08 18:40:40 -04001089def add_outdir_options(parser):
1090 """Adds --outdir, which is orthogonal to --isolate-server.
1091
1092 Note: On upload, separate commands are used between 'archive' and 'hashtable'.
1093 On 'download', the same command can download from either an isolate server or
1094 a file system.
1095 """
1096 parser.add_option(
1097 '-o', '--outdir', metavar='DIR',
1098 help='Directory used to recreate the tree.')
1099
1100
1101def process_outdir_options(parser, options, cwd):
1102 if not options.outdir:
1103 parser.error('--outdir is required.')
1104 if file_path.is_url(options.outdir):
1105 parser.error('Can\'t use an URL for --outdir.')
1106 options.outdir = unicode(options.outdir).replace('/', os.path.sep)
1107 # outdir doesn't need native path case since tracing is never done from there.
1108 options.outdir = os.path.abspath(
1109 os.path.normpath(os.path.join(cwd, options.outdir)))
1110 # In theory, we'd create the directory outdir right away. Defer doing it in
1111 # case there's errors in the command line.
1112
1113
Vadim Shtayurafddb1432014-09-30 18:32:41 -07001114def process_isolate_options(parser, options, cwd=None, require_isolated=True):
1115 """Handles options added with 'add_isolate_options'.
1116
1117 Mutates |options| in place, by normalizing path to isolate file, values of
1118 variables, etc.
1119 """
1120 cwd = file_path.get_native_path_case(unicode(cwd or os.getcwd()))
1121
1122 # Parse --isolated option.
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001123 if options.isolated:
maruel@chromium.org61a9b3b2012-12-12 17:18:52 +00001124 options.isolated = os.path.normpath(
1125 os.path.join(cwd, options.isolated.replace('/', os.path.sep)))
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001126 if require_isolated and not options.isolated:
maruel@chromium.org75c05b42013-07-25 15:51:48 +00001127 parser.error('--isolated is required.')
maruel@chromium.org0cd0b182012-10-22 13:34:15 +00001128 if options.isolated and not options.isolated.endswith('.isolated'):
1129 parser.error('--isolated value must end with \'.isolated\'')
maruel@chromium.org715e3fb2013-03-19 15:44:06 +00001130
Vadim Shtayurafddb1432014-09-30 18:32:41 -07001131 # Processes all the --<foo>-variable flags.
benrg@chromium.org609b7982013-02-07 16:44:46 +00001132 def try_make_int(s):
maruel@chromium.orge83215b2013-02-21 14:16:59 +00001133 """Converts a value to int if possible, converts to unicode otherwise."""
benrg@chromium.org609b7982013-02-07 16:44:46 +00001134 try:
1135 return int(s)
1136 except ValueError:
maruel@chromium.orge83215b2013-02-21 14:16:59 +00001137 return s.decode('utf-8')
Marc-Antoine Ruel1c1edd62013-12-06 09:13:13 -05001138 options.config_variables = dict(
1139 (k, try_make_int(v)) for k, v in options.config_variables)
1140 options.path_variables = dict(options.path_variables)
Marc-Antoine Ruelf3589b12013-12-06 14:26:16 -05001141 options.extra_variables = dict(options.extra_variables)
maruel@chromium.orgb253fb82012-10-16 21:44:48 +00001142
Vadim Shtayurafddb1432014-09-30 18:32:41 -07001143 # Normalize the path in --isolate.
1144 if options.isolate:
1145 # TODO(maruel): Work with non-ASCII.
1146 # The path must be in native path case for tracing purposes.
1147 options.isolate = unicode(options.isolate).replace('/', os.path.sep)
1148 options.isolate = os.path.normpath(os.path.join(cwd, options.isolate))
1149 options.isolate = file_path.get_native_path_case(options.isolate)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00001150
1151
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00001152def main(argv):
maruel@chromium.orge5322512013-08-19 20:17:57 +00001153 dispatcher = subcommand.CommandDispatcher(__name__)
Vadim Shtayurafddb1432014-09-30 18:32:41 -07001154 parser = tools.OptionParserWithLogging(
1155 version=__version__, verbose=int(os.environ.get('ISOLATE_DEBUG', 0)))
1156 return dispatcher.execute(parser, argv)
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00001157
1158
1159if __name__ == '__main__':
maruel@chromium.orge5322512013-08-19 20:17:57 +00001160 fix_encoding.fix_encoding()
vadimsh@chromium.orga4326472013-08-24 02:05:41 +00001161 tools.disable_buffering()
maruel@chromium.orge5322512013-08-19 20:17:57 +00001162 colorama.init()
maruel@chromium.org8fb47fe2012-10-03 20:13:15 +00001163 sys.exit(main(sys.argv[1:]))