blob: 7a026a8d42fc3e4215bd94be90f3436810525f4c [file] [log] [blame]
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -05001# Copyright 2014 The Swarming Authors. All rights reserved.
2# Use of this source code is governed under the Apache License, Version 2.0 that
3# can be found in the LICENSE file.
4
5"""Contains logic to parse .isolate files.
6
7This module doesn't touch the file system. It's the job of the client code to do
8I/O on behalf of this module.
9
10See more information at
11 https://code.google.com/p/swarming/wiki/IsolateDesign
12 https://code.google.com/p/swarming/wiki/IsolateUserGuide
13"""
14
15import ast
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -050016import itertools
17import logging
18import os
19import re
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -040020import sys
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -050021
22import isolateserver
23
24from utils import short_expression_finder
25
26# Files that should be 0-length when mapped.
27KEY_TOUCHED = 'isolate_dependency_touched'
28# Files that should be tracked by the build tool.
29KEY_TRACKED = 'isolate_dependency_tracked'
30# Files that should not be tracked by the build tool.
31KEY_UNTRACKED = 'isolate_dependency_untracked'
32
33# Valid variable name.
34VALID_VARIABLE = '[A-Za-z_][A-Za-z_0-9]*'
35
36
37def determine_root_dir(relative_root, infiles):
38 """For a list of infiles, determines the deepest root directory that is
39 referenced indirectly.
40
41 All arguments must be using os.path.sep.
42 """
43 # The trick used to determine the root directory is to look at "how far" back
44 # up it is looking up.
45 deepest_root = relative_root
46 for i in infiles:
47 x = relative_root
48 while i.startswith('..' + os.path.sep):
49 i = i[3:]
50 assert not i.startswith(os.path.sep)
51 x = os.path.dirname(x)
52 if deepest_root.startswith(x):
53 deepest_root = x
54 logging.debug(
55 'determine_root_dir(%s, %d files) -> %s' % (
56 relative_root, len(infiles), deepest_root))
57 return deepest_root
58
59
60def replace_variable(part, variables):
61 m = re.match(r'<\((' + VALID_VARIABLE + ')\)', part)
62 if m:
63 if m.group(1) not in variables:
64 raise isolateserver.ConfigError(
65 'Variable "%s" was not found in %s.\nDid you forget to specify '
66 '--path-variable?' % (m.group(1), variables))
67 return variables[m.group(1)]
68 return part
69
70
71def eval_variables(item, variables):
72 """Replaces the .isolate variables in a string item.
73
74 Note that the .isolate format is a subset of the .gyp dialect.
75 """
76 return ''.join(
77 replace_variable(p, variables)
78 for p in re.split(r'(<\(' + VALID_VARIABLE + '\))', item))
79
80
81def split_touched(files):
82 """Splits files that are touched vs files that are read."""
83 tracked = []
84 touched = []
85 for f in files:
86 if f.size:
87 tracked.append(f)
88 else:
89 touched.append(f)
90 return tracked, touched
91
92
93def pretty_print(variables, stdout):
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -040094 """Outputs a .isolate file from the decoded variables.
95
96 The .isolate format is GYP compatible.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -050097
98 Similar to pprint.print() but with NIH syndrome.
99 """
100 # Order the dictionary keys by these keys in priority.
101 ORDER = (
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400102 'variables', 'condition', 'command', 'read_only',
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500103 KEY_TRACKED, KEY_UNTRACKED)
104
105 def sorting_key(x):
106 """Gives priority to 'most important' keys before the others."""
107 if x in ORDER:
108 return str(ORDER.index(x))
109 return x
110
111 def loop_list(indent, items):
112 for item in items:
113 if isinstance(item, basestring):
114 stdout.write('%s\'%s\',\n' % (indent, item))
115 elif isinstance(item, dict):
116 stdout.write('%s{\n' % indent)
117 loop_dict(indent + ' ', item)
118 stdout.write('%s},\n' % indent)
119 elif isinstance(item, list):
120 # A list inside a list will write the first item embedded.
121 stdout.write('%s[' % indent)
122 for index, i in enumerate(item):
123 if isinstance(i, basestring):
124 stdout.write(
125 '\'%s\', ' % i.replace('\\', '\\\\').replace('\'', '\\\''))
126 elif isinstance(i, dict):
127 stdout.write('{\n')
128 loop_dict(indent + ' ', i)
129 if index != len(item) - 1:
130 x = ', '
131 else:
132 x = ''
133 stdout.write('%s}%s' % (indent, x))
134 else:
135 assert False
136 stdout.write('],\n')
137 else:
138 assert False
139
140 def loop_dict(indent, items):
141 for key in sorted(items, key=sorting_key):
142 item = items[key]
143 stdout.write("%s'%s': " % (indent, key))
144 if isinstance(item, dict):
145 stdout.write('{\n')
146 loop_dict(indent + ' ', item)
147 stdout.write(indent + '},\n')
148 elif isinstance(item, list):
149 stdout.write('[\n')
150 loop_list(indent + ' ', item)
151 stdout.write(indent + '],\n')
152 elif isinstance(item, basestring):
153 stdout.write(
154 '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\''))
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500155 elif isinstance(item, (int, bool)) or item is None:
Marc-Antoine Ruelfdc9a552014-03-28 13:52:11 -0400156 stdout.write('%s,\n' % item)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500157 else:
158 assert False, item
159
160 stdout.write('{\n')
161 loop_dict(' ', variables)
162 stdout.write('}\n')
163
164
165def print_all(comment, data, stream):
166 """Prints a complete .isolate file and its top-level file comment into a
167 stream.
168 """
169 if comment:
170 stream.write(comment)
171 pretty_print(data, stream)
172
173
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500174def extract_comment(content):
175 """Extracts file level comment."""
176 out = []
177 for line in content.splitlines(True):
178 if line.startswith('#'):
179 out.append(line)
180 else:
181 break
182 return ''.join(out)
183
184
185def eval_content(content):
186 """Evaluates a python file and return the value defined in it.
187
188 Used in practice for .isolate files.
189 """
190 globs = {'__builtins__': None}
191 locs = {}
192 try:
193 value = eval(content, globs, locs)
194 except TypeError as e:
195 e.args = list(e.args) + [content]
196 raise
197 assert locs == {}, locs
198 assert globs == {'__builtins__': None}, globs
199 return value
200
201
202def match_configs(expr, config_variables, all_configs):
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500203 """Returns the list of values from |values| that match the condition |expr|.
204
205 Arguments:
206 expr: string that is evaluatable with eval(). It is a GYP condition.
207 config_variables: list of the name of the variables.
208 all_configs: list of the list of possible values.
209
210 If a variable is not referenced at all, it is marked as unbounded (free) with
211 a value set to None.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500212 """
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500213 # It is more than just eval'ing the variable, it needs to be double checked to
214 # see if the variable is referenced at all. If not, the variable is free
215 # (unbounded).
216 # TODO(maruel): Use the intelligent way by inspecting expr instead of doing
217 # trial and error to figure out which variable is bound.
218 combinations = []
219 for bound_variables in itertools.product(
220 (True, False), repeat=len(config_variables)):
221 # Add the combination of variables bound.
222 combinations.append(
223 (
224 [c for c, b in zip(config_variables, bound_variables) if b],
225 set(
226 tuple(v if b else None for v, b in zip(line, bound_variables))
227 for line in all_configs)
228 ))
229
230 out = []
231 for variables, configs in combinations:
232 # Strip variables and see if expr can still be evaluated.
233 for values in configs:
234 globs = {'__builtins__': None}
235 globs.update(zip(variables, (v for v in values if v is not None)))
236 try:
237 assertion = eval(expr, globs, {})
238 except NameError:
239 continue
240 if not isinstance(assertion, bool):
241 raise isolateserver.ConfigError('Invalid condition')
242 if assertion:
243 out.append(values)
244 return out
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500245
246
247def verify_variables(variables):
248 """Verifies the |variables| dictionary is in the expected format."""
249 VALID_VARIABLES = [
250 KEY_TOUCHED,
251 KEY_TRACKED,
252 KEY_UNTRACKED,
253 'command',
254 'read_only',
255 ]
256 assert isinstance(variables, dict), variables
257 assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys()
258 for name, value in variables.iteritems():
259 if name == 'read_only':
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500260 assert value in (0, 1, 2, None), value
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500261 else:
262 assert isinstance(value, list), value
263 assert all(isinstance(i, basestring) for i in value), value
264
265
266def verify_ast(expr, variables_and_values):
267 """Verifies that |expr| is of the form
268 expr ::= expr ( "or" | "and" ) expr
269 | identifier "==" ( string | int )
270 Also collects the variable identifiers and string/int values in the dict
271 |variables_and_values|, in the form {'var': set([val1, val2, ...]), ...}.
272 """
273 assert isinstance(expr, (ast.BoolOp, ast.Compare))
274 if isinstance(expr, ast.BoolOp):
275 assert isinstance(expr.op, (ast.And, ast.Or))
276 for subexpr in expr.values:
277 verify_ast(subexpr, variables_and_values)
278 else:
279 assert isinstance(expr.left.ctx, ast.Load)
280 assert len(expr.ops) == 1
281 assert isinstance(expr.ops[0], ast.Eq)
282 var_values = variables_and_values.setdefault(expr.left.id, set())
283 rhs = expr.comparators[0]
284 assert isinstance(rhs, (ast.Str, ast.Num))
285 var_values.add(rhs.n if isinstance(rhs, ast.Num) else rhs.s)
286
287
288def verify_condition(condition, variables_and_values):
289 """Verifies the |condition| dictionary is in the expected format.
290 See verify_ast() for the meaning of |variables_and_values|.
291 """
292 VALID_INSIDE_CONDITION = ['variables']
293 assert isinstance(condition, list), condition
294 assert len(condition) == 2, condition
295 expr, then = condition
296
297 test_ast = compile(expr, '<condition>', 'eval', ast.PyCF_ONLY_AST)
298 verify_ast(test_ast.body, variables_and_values)
299
300 assert isinstance(then, dict), then
301 assert set(VALID_INSIDE_CONDITION).issuperset(set(then)), then.keys()
302 if not 'variables' in then:
303 raise isolateserver.ConfigError('Missing \'variables\' in condition %s' %
304 condition)
305 verify_variables(then['variables'])
306
307
308def verify_root(value, variables_and_values):
309 """Verifies that |value| is the parsed form of a valid .isolate file.
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400310
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500311 See verify_ast() for the meaning of |variables_and_values|.
312 """
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400313 VALID_ROOTS = ['includes', 'conditions', 'variables']
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500314 assert isinstance(value, dict), value
315 assert set(VALID_ROOTS).issuperset(set(value)), value.keys()
316
317 includes = value.get('includes', [])
318 assert isinstance(includes, list), includes
319 for include in includes:
320 assert isinstance(include, basestring), include
321
322 conditions = value.get('conditions', [])
323 assert isinstance(conditions, list), conditions
324 for condition in conditions:
325 verify_condition(condition, variables_and_values)
326
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400327 variables = value.get('variables', {})
328 verify_variables(variables)
329
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500330
331def remove_weak_dependencies(values, key, item, item_configs):
332 """Removes any configs from this key if the item is already under a
333 strong key.
334 """
335 if key == KEY_TOUCHED:
336 item_configs = set(item_configs)
337 for stronger_key in (KEY_TRACKED, KEY_UNTRACKED):
338 try:
339 item_configs -= values[stronger_key][item]
340 except KeyError:
341 pass
342
343 return item_configs
344
345
346def remove_repeated_dependencies(folders, key, item, item_configs):
347 """Removes any configs from this key if the item is in a folder that is
348 already included."""
349
350 if key in (KEY_UNTRACKED, KEY_TRACKED, KEY_TOUCHED):
351 item_configs = set(item_configs)
352 for (folder, configs) in folders.iteritems():
353 if folder != item and item.startswith(folder):
354 item_configs -= configs
355
356 return item_configs
357
358
359def get_folders(values_dict):
360 """Returns a dict of all the folders in the given value_dict."""
361 return dict(
362 (item, configs) for (item, configs) in values_dict.iteritems()
363 if item.endswith('/')
364 )
365
366
367def invert_map(variables):
368 """Converts {config: {deptype: list(depvals)}} to
369 {deptype: {depval: set(configs)}}.
370 """
371 KEYS = (
372 KEY_TOUCHED,
373 KEY_TRACKED,
374 KEY_UNTRACKED,
375 'command',
376 'read_only',
377 )
378 out = dict((key, {}) for key in KEYS)
379 for config, values in variables.iteritems():
380 for key in KEYS:
381 if key == 'command':
382 items = [tuple(values[key])] if key in values else []
383 elif key == 'read_only':
384 items = [values[key]] if key in values else []
385 else:
386 assert key in (KEY_TOUCHED, KEY_TRACKED, KEY_UNTRACKED)
387 items = values.get(key, [])
388 for item in items:
389 out[key].setdefault(item, set()).add(config)
390 return out
391
392
393def reduce_inputs(values):
394 """Reduces the output of invert_map() to the strictest minimum list.
395
396 Looks at each individual file and directory, maps where they are used and
397 reconstructs the inverse dictionary.
398
399 Returns the minimized dictionary.
400 """
401 KEYS = (
402 KEY_TOUCHED,
403 KEY_TRACKED,
404 KEY_UNTRACKED,
405 'command',
406 'read_only',
407 )
408
409 # Folders can only live in KEY_UNTRACKED.
410 folders = get_folders(values.get(KEY_UNTRACKED, {}))
411
412 out = dict((key, {}) for key in KEYS)
413 for key in KEYS:
414 for item, item_configs in values.get(key, {}).iteritems():
415 item_configs = remove_weak_dependencies(values, key, item, item_configs)
416 item_configs = remove_repeated_dependencies(
417 folders, key, item, item_configs)
418 if item_configs:
419 out[key][item] = item_configs
420 return out
421
422
423def convert_map_to_isolate_dict(values, config_variables):
424 """Regenerates back a .isolate configuration dict from files and dirs
425 mappings generated from reduce_inputs().
426 """
427 # Gather a list of configurations for set inversion later.
428 all_mentioned_configs = set()
429 for configs_by_item in values.itervalues():
430 for configs in configs_by_item.itervalues():
431 all_mentioned_configs.update(configs)
432
433 # Invert the mapping to make it dict first.
434 conditions = {}
435 for key in values:
436 for item, configs in values[key].iteritems():
437 then = conditions.setdefault(frozenset(configs), {})
438 variables = then.setdefault('variables', {})
439
Marc-Antoine Ruelbb20b6d2014-01-10 18:47:47 -0500440 if key == 'read_only':
441 if not isinstance(item, int):
442 raise isolateserver.ConfigError(
443 'Unexpected entry type %r for key %s' % (item, key))
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500444 variables[key] = item
Marc-Antoine Ruelbb20b6d2014-01-10 18:47:47 -0500445 elif key == 'command':
446 if not isinstance(item, tuple):
447 raise isolateserver.ConfigError(
448 'Unexpected entry type %r for key %s' % (item, key))
449 if key in variables:
450 raise isolateserver.ConfigError('Unexpected duplicate key %s' % key)
451 if not item:
452 raise isolateserver.ConfigError(
453 'Expected non empty entry in %s' % key)
454 variables[key] = list(item)
455 elif key in (KEY_TOUCHED, KEY_TRACKED, KEY_UNTRACKED):
456 if not isinstance(item, basestring):
457 raise isolateserver.ConfigError('Unexpected entry type %r' % item)
458 if not item:
459 raise isolateserver.ConfigError(
460 'Expected non empty entry in %s' % key)
461 # The list of items (files or dirs). Append the new item and keep
462 # the list sorted.
463 l = variables.setdefault(key, [])
464 l.append(item)
465 l.sort()
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500466 else:
Marc-Antoine Ruelbb20b6d2014-01-10 18:47:47 -0500467 raise isolateserver.ConfigError('Unexpected key %s' % key)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500468
469 if all_mentioned_configs:
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500470 # Change [(1, 2), (3, 4)] to [set(1, 3), set(2, 4)]
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500471 config_values = map(set, zip(*all_mentioned_configs))
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500472 for i in config_values:
473 i.discard(None)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500474 sef = short_expression_finder.ShortExpressionFinder(
475 zip(config_variables, config_values))
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500476 conditions = sorted([sef.get_expr(c), v] for c, v in conditions.iteritems())
477 else:
478 conditions = []
Marc-Antoine Rueld27c6632014-03-13 15:29:36 -0400479 out = {'conditions': conditions}
480 for c in conditions:
481 if c[0] == '':
482 # Extract the global.
483 out.update(c[1])
484 conditions.remove(c)
485 break
486 return out
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500487
488
489class ConfigSettings(object):
490 """Represents the dependency variables for a single build configuration.
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400491
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500492 The structure is immutable.
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400493
494 .touch, .tracked and .untracked are the list of dependencies. The items in
495 these lists use '/' as a path separator.
496 .command and .isolate_dir describe how to run the command. .isolate_dir uses
497 the OS' native path separator. It must be an absolute path, it's the path
498 where to start the command from.
499 .read_only describe how to map the files.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500500 """
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400501 def __init__(self, values, isolate_dir):
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500502 verify_variables(values)
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400503 if isolate_dir is None:
504 # It must be an empty object if isolate_dir is None.
505 assert values == {}, values
506 else:
507 # Otherwise, the path must be absolute.
508 assert os.path.isabs(isolate_dir), isolate_dir
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500509 self.touched = sorted(values.get(KEY_TOUCHED, []))
510 self.tracked = sorted(values.get(KEY_TRACKED, []))
511 self.untracked = sorted(values.get(KEY_UNTRACKED, []))
512 self.command = values.get('command', [])[:]
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400513 self.isolate_dir = isolate_dir
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500514 self.read_only = values.get('read_only')
515
516 def union(self, rhs):
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400517 """Merges two config settings together into a new instance.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500518
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400519 self has priority over rhs for .command. Use the same
520 .isolate_dir as the one having a .command. Preferring self over rhs.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500521 """
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500522 var = {
523 KEY_TOUCHED: sorted(self.touched + rhs.touched),
524 KEY_TRACKED: sorted(self.tracked + rhs.tracked),
525 KEY_UNTRACKED: sorted(self.untracked + rhs.untracked),
526 'command': self.command or rhs.command,
527 'read_only': rhs.read_only if self.read_only is None else self.read_only,
528 }
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400529 isolate_dir = self.isolate_dir if self.command else rhs.isolate_dir
530 return ConfigSettings(var, isolate_dir)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500531
532 def flatten(self):
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400533 """Converts the object into a dict."""
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500534 out = {}
535 if self.command:
536 out['command'] = self.command
537 if self.touched:
538 out[KEY_TOUCHED] = self.touched
539 if self.tracked:
540 out[KEY_TRACKED] = self.tracked
541 if self.untracked:
542 out[KEY_UNTRACKED] = self.untracked
543 if self.read_only is not None:
544 out['read_only'] = self.read_only
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400545 # TODO(maruel): Probably better to not output it if command is None?
546 if self.isolate_dir is not None:
547 out['isolate_dir'] = self.isolate_dir
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500548 return out
549
550
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500551def _safe_index(l, k):
552 try:
553 return l.index(k)
554 except ValueError:
555 return None
556
557
558def _get_map_keys(dest_keys, in_keys):
559 """Returns a tuple of the indexes of each item in in_keys found in dest_keys.
560
561 For example, if in_keys is ('A', 'C') and dest_keys is ('A', 'B', 'C'), the
562 return value will be (0, None, 1).
563 """
564 return tuple(_safe_index(in_keys, k) for k in dest_keys)
565
566
567def _map_keys(mapping, items):
568 """Returns a tuple with items placed at mapping index.
569
570 For example, if mapping is (1, None, 0) and items is ('a', 'b'), it will
571 return ('b', None, 'c').
572 """
573 return tuple(items[i] if i != None else None for i in mapping)
574
575
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500576class Configs(object):
577 """Represents a processed .isolate file.
578
579 Stores the file in a processed way, split by configuration.
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500580
581 At this point, we don't know all the possibilities. So mount a partial view
582 that we have.
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400583
584 This class doesn't hold isolate_dir, since it is dependent on the final
585 configuration selected. It is implicitly dependent on which .isolate defines
586 the 'command' that will take effect.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500587 """
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500588 def __init__(self, file_comment, config_variables):
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500589 self.file_comment = file_comment
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500590 # Contains the names of the config variables seen while processing
591 # .isolate file(s). The order is important since the same order is used for
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500592 # keys in self._by_config.
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500593 assert isinstance(config_variables, tuple)
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400594 assert all(isinstance(c, basestring) for c in config_variables), (
595 config_variables)
Marc-Antoine Ruelf0d07872014-03-27 16:59:03 -0400596 config_variables = tuple(config_variables)
597 assert tuple(sorted(config_variables)) == config_variables, config_variables
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500598 self._config_variables = config_variables
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500599 # The keys of _by_config are tuples of values for each of the items in
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500600 # self._config_variables. A None item in the list of the key means the value
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500601 # is unbounded.
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500602 self._by_config = {}
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500603
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500604 @property
605 def config_variables(self):
606 return self._config_variables
607
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500608 def get_config(self, config):
609 """Returns all configs that matches this config as a single ConfigSettings.
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400610
611 Returns an empty ConfigSettings if none apply.
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500612 """
Marc-Antoine Ruelf0d07872014-03-27 16:59:03 -0400613 # TODO(maruel): Fix ordering based on the bounded values. The keys are not
614 # necessarily sorted in the way that makes sense, they are alphabetically
615 # sorted. It is important because the left-most takes predescence.
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400616 out = ConfigSettings({}, None)
Marc-Antoine Ruelf0d07872014-03-27 16:59:03 -0400617 for k, v in sorted(self._by_config.iteritems()):
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500618 if all(i == j or j is None for i, j in zip(config, k)):
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400619 out = out.union(v)
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500620 return out
621
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400622 def set_config(self, key, value):
623 """Sets the ConfigSettings for this key.
624
625 The key is a tuple of bounded or unbounded variables. The global variable
626 is the key where all values are unbounded, e.g.:
627 (None,) * len(self._config_variables)
628 """
629 assert key not in self._by_config, (key, self._by_config.keys())
630 assert isinstance(key, tuple)
631 assert len(key) == len(self._config_variables), (
632 key, self._config_variables)
633 assert isinstance(value, ConfigSettings)
634 self._by_config[key] = value
635
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500636 def union(self, rhs):
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400637 """Returns a new Configs instance, the union of variables from self and rhs.
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500638
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400639 Uses self.file_comment if available, otherwise rhs.file_comment.
Marc-Antoine Ruelf0d07872014-03-27 16:59:03 -0400640 It keeps config_variables sorted in the output.
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400641 """
642 # Merge the keys of config_variables for each Configs instances. All the new
643 # variables will become unbounded. This requires realigning the keys.
644 config_variables = tuple(sorted(
645 set(self.config_variables) | set(rhs.config_variables)))
646 out = Configs(self.file_comment or rhs.file_comment, config_variables)
647 mapping_lhs = _get_map_keys(out.config_variables, self.config_variables)
648 mapping_rhs = _get_map_keys(out.config_variables, rhs.config_variables)
649 lhs_config = dict(
650 (_map_keys(mapping_lhs, k), v) for k, v in self._by_config.iteritems())
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500651 # pylint: disable=W0212
Marc-Antoine Ruel8170f492014-03-13 15:26:56 -0400652 rhs_config = dict(
653 (_map_keys(mapping_rhs, k), v) for k, v in rhs._by_config.iteritems())
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500654
655 for key in set(lhs_config) | set(rhs_config):
Marc-Antoine Ruelbd1b2842014-03-28 13:56:43 -0400656 l = lhs_config.get(key)
657 r = rhs_config.get(key)
658 out.set_config(key, l.union(r) if (l and r) else (l or r))
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500659 return out
660
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500661 def flatten(self):
662 """Returns a flat dictionary representation of the configuration.
663 """
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500664 return dict((k, v.flatten()) for k, v in self._by_config.iteritems())
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500665
666 def make_isolate_file(self):
667 """Returns a dictionary suitable for writing to a .isolate file.
668 """
669 dependencies_by_config = self.flatten()
670 configs_by_dependency = reduce_inputs(invert_map(dependencies_by_config))
671 return convert_map_to_isolate_dict(configs_by_dependency,
672 self.config_variables)
673
674
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500675def load_isolate_as_config(isolate_dir, value, file_comment):
676 """Parses one .isolate file and returns a Configs() instance.
677
678 Arguments:
679 isolate_dir: only used to load relative includes so it doesn't depend on
680 cwd.
681 value: is the loaded dictionary that was defined in the gyp file.
682 file_comment: comments found at the top of the file so it can be preserved.
683
684 The expected format is strict, anything diverting from the format below will
685 throw an assert:
686 {
687 'includes': [
688 'foo.isolate',
689 ],
690 'conditions': [
691 ['OS=="vms" and foo=42', {
692 'variables': {
693 'command': [
694 ...
695 ],
696 'isolate_dependency_tracked': [
697 ...
698 ],
699 'isolate_dependency_untracked': [
700 ...
701 ],
Marc-Antoine Ruel7124e392014-01-09 11:49:21 -0500702 'read_only': 0,
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500703 },
704 }],
705 ...
706 ],
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400707 'variables': {
708 ...
709 },
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500710 }
711 """
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400712 assert os.path.isabs(isolate_dir), isolate_dir
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400713 if any(len(cond) == 3 for cond in value.get('conditions', [])):
714 raise isolateserver.ConfigError('Using \'else\' is not supported anymore.')
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500715 variables_and_values = {}
716 verify_root(value, variables_and_values)
717 if variables_and_values:
718 config_variables, config_values = zip(
719 *sorted(variables_and_values.iteritems()))
720 all_configs = list(itertools.product(*config_values))
721 else:
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500722 config_variables = ()
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500723 all_configs = []
724
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500725 isolate = Configs(file_comment, config_variables)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500726
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400727 # Add global variables. The global variables are on the empty tuple key.
728 isolate.set_config(
729 (None,) * len(config_variables),
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400730 ConfigSettings(value.get('variables', {}), isolate_dir))
Marc-Antoine Rueld7c032b2014-03-13 15:32:16 -0400731
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500732 # Add configuration-specific variables.
733 for expr, then in value.get('conditions', []):
734 configs = match_configs(expr, config_variables, all_configs)
Marc-Antoine Ruel67d3c0a2014-01-10 09:12:39 -0500735 new = Configs(None, config_variables)
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500736 for config in configs:
Marc-Antoine Ruelb53d0c12014-03-28 13:46:27 -0400737 new.set_config(config, ConfigSettings(then['variables'], isolate_dir))
Marc-Antoine Ruel9ac1b912014-01-10 09:08:42 -0500738 isolate = isolate.union(new)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500739
740 # Load the includes. Process them in reverse so the last one take precedence.
741 for include in reversed(value.get('includes', [])):
742 if os.path.isabs(include):
743 raise isolateserver.ConfigError(
744 'Failed to load configuration; absolute include path \'%s\'' %
745 include)
746 included_isolate = os.path.normpath(os.path.join(isolate_dir, include))
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400747 if sys.platform == 'win32':
748 if included_isolate[0].lower() != isolate_dir[0].lower():
749 raise isolateserver.ConfigError(
750 'Can\'t reference a .isolate file from another drive')
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500751 with open(included_isolate, 'r') as f:
752 included_isolate = load_isolate_as_config(
753 os.path.dirname(included_isolate),
754 eval_content(f.read()),
755 None)
Marc-Antoine Ruelbd1b2842014-03-28 13:56:43 -0400756 isolate = isolate.union(included_isolate)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500757
758 return isolate
759
760
761def load_isolate_for_config(isolate_dir, content, config_variables):
762 """Loads the .isolate file and returns the information unprocessed but
763 filtered for the specific OS.
764
Marc-Antoine Ruelfdc9a552014-03-28 13:52:11 -0400765 Returns:
766 tuple of command, dependencies, touched, read_only flag, isolate_dir.
767 The dependencies are fixed to use os.path.sep.
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500768 """
769 # Load the .isolate file, process its conditions, retrieve the command and
770 # dependencies.
771 isolate = load_isolate_as_config(isolate_dir, eval_content(content), None)
772 try:
773 config_name = tuple(
774 config_variables[var] for var in isolate.config_variables)
775 except KeyError:
776 raise isolateserver.ConfigError(
777 'These configuration variables were missing from the command line: %s' %
778 ', '.join(
779 sorted(set(isolate.config_variables) - set(config_variables))))
Marc-Antoine Ruel3ae9e6e2014-01-13 15:42:16 -0500780
781 # A configuration is to be created with all the combinations of free
782 # variables.
783 config = isolate.get_config(config_name)
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500784 # Merge tracked and untracked variables, isolate.py doesn't care about the
785 # trackability of the variables, only the build tool does.
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400786 dependencies = sorted(
Marc-Antoine Ruela5a36222014-01-09 10:35:45 -0500787 f.replace('/', os.path.sep) for f in config.tracked + config.untracked
Marc-Antoine Ruel226ab802014-03-29 16:22:36 -0400788 )
789 touched = sorted(f.replace('/', os.path.sep) for f in config.touched)
Marc-Antoine Ruelfdc9a552014-03-28 13:52:11 -0400790 return (
791 config.command, dependencies, touched, config.read_only,
792 config.isolate_dir)