blob: 4d0b50730d401b16ce11a2c83a3d00f9b558834a [file] [log] [blame]
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02001# Copyright 2017 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import ast
Edward Lesmes6f64a052018-03-20 17:35:49 -04006import cStringIO
Paweł Hajdan, Jr7cf96a42017-05-26 20:28:35 +02007import collections
Edward Lesmes6f64a052018-03-20 17:35:49 -04008import tokenize
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02009
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +020010from third_party import schema
11
12
Edward Lesmes6f64a052018-03-20 17:35:49 -040013class _NodeDict(collections.MutableMapping):
14 """Dict-like type that also stores information on AST nodes and tokens."""
15 def __init__(self, data, tokens=None):
16 self.data = collections.OrderedDict(data)
17 self.tokens = tokens
18
19 def __str__(self):
20 return str({k: v[0] for k, v in self.data.iteritems()})
21
22 def __getitem__(self, key):
23 return self.data[key][0]
24
25 def __setitem__(self, key, value):
26 self.data[key] = (value, None)
27
28 def __delitem__(self, key):
29 del self.data[key]
30
31 def __iter__(self):
32 return iter(self.data)
33
34 def __len__(self):
35 return len(self.data)
36
Edward Lesmes3d993812018-04-02 12:52:49 -040037 def MoveTokens(self, origin, delta):
38 if self.tokens:
39 new_tokens = {}
40 for pos, token in self.tokens.iteritems():
41 if pos[0] >= origin:
42 pos = (pos[0] + delta, pos[1])
43 token = token[:2] + (pos,) + token[3:]
44 new_tokens[pos] = token
45
46 for value, node in self.data.values():
47 if node.lineno >= origin:
48 node.lineno += delta
49 if isinstance(value, _NodeDict):
50 value.MoveTokens(origin, delta)
51
Edward Lesmes6f64a052018-03-20 17:35:49 -040052 def GetNode(self, key):
53 return self.data[key][1]
54
Edward Lesmes6c24d372018-03-28 12:52:29 -040055 def SetNode(self, key, value, node):
Edward Lesmes6f64a052018-03-20 17:35:49 -040056 self.data[key] = (value, node)
57
58
59def _NodeDictSchema(dict_schema):
60 """Validate dict_schema after converting _NodeDict to a regular dict."""
61 def validate(d):
62 schema.Schema(dict_schema).validate(dict(d))
63 return True
64 return validate
65
66
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +020067# See https://github.com/keleshev/schema for docs how to configure schema.
Edward Lesmes6f64a052018-03-20 17:35:49 -040068_GCLIENT_DEPS_SCHEMA = _NodeDictSchema({
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +020069 schema.Optional(basestring): schema.Or(
70 None,
71 basestring,
Edward Lesmes6f64a052018-03-20 17:35:49 -040072 _NodeDictSchema({
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +020073 # Repo and revision to check out under the path
74 # (same as if no dict was used).
Michael Moss012013e2018-03-30 17:03:19 -070075 'url': schema.Or(None, basestring),
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +020076
77 # Optional condition string. The dep will only be processed
78 # if the condition evaluates to True.
79 schema.Optional('condition'): basestring,
John Budorick0f7b2002018-01-19 15:46:17 -080080
81 schema.Optional('dep_type', default='git'): basestring,
Edward Lesmes6f64a052018-03-20 17:35:49 -040082 }),
John Budorick0f7b2002018-01-19 15:46:17 -080083 # CIPD package.
Edward Lesmes6f64a052018-03-20 17:35:49 -040084 _NodeDictSchema({
John Budorick0f7b2002018-01-19 15:46:17 -080085 'packages': [
Edward Lesmes6f64a052018-03-20 17:35:49 -040086 _NodeDictSchema({
John Budorick0f7b2002018-01-19 15:46:17 -080087 'package': basestring,
88
89 'version': basestring,
Edward Lesmes6f64a052018-03-20 17:35:49 -040090 })
John Budorick0f7b2002018-01-19 15:46:17 -080091 ],
92
93 schema.Optional('condition'): basestring,
94
95 schema.Optional('dep_type', default='cipd'): basestring,
Edward Lesmes6f64a052018-03-20 17:35:49 -040096 }),
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +020097 ),
Edward Lesmes6f64a052018-03-20 17:35:49 -040098})
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +020099
Edward Lesmes6f64a052018-03-20 17:35:49 -0400100_GCLIENT_HOOKS_SCHEMA = [_NodeDictSchema({
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200101 # Hook action: list of command-line arguments to invoke.
102 'action': [basestring],
103
104 # Name of the hook. Doesn't affect operation.
105 schema.Optional('name'): basestring,
106
107 # Hook pattern (regex). Originally intended to limit some hooks to run
108 # only when files matching the pattern have changed. In practice, with git,
109 # gclient runs all the hooks regardless of this field.
110 schema.Optional('pattern'): basestring,
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200111
112 # Working directory where to execute the hook.
113 schema.Optional('cwd'): basestring,
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200114
115 # Optional condition string. The hook will only be run
116 # if the condition evaluates to True.
117 schema.Optional('condition'): basestring,
Edward Lesmes6f64a052018-03-20 17:35:49 -0400118})]
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200119
Edward Lesmes6f64a052018-03-20 17:35:49 -0400120_GCLIENT_SCHEMA = schema.Schema(_NodeDictSchema({
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200121 # List of host names from which dependencies are allowed (whitelist).
122 # NOTE: when not present, all hosts are allowed.
123 # NOTE: scoped to current DEPS file, not recursive.
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200124 schema.Optional('allowed_hosts'): [schema.Optional(basestring)],
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200125
126 # Mapping from paths to repo and revision to check out under that path.
127 # Applying this mapping to the on-disk checkout is the main purpose
128 # of gclient, and also why the config file is called DEPS.
129 #
130 # The following functions are allowed:
131 #
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200132 # Var(): allows variable substitution (either from 'vars' dict below,
133 # or command-line override)
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +0200134 schema.Optional('deps'): _GCLIENT_DEPS_SCHEMA,
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200135
136 # Similar to 'deps' (see above) - also keyed by OS (e.g. 'linux').
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200137 # Also see 'target_os'.
Edward Lesmes6f64a052018-03-20 17:35:49 -0400138 schema.Optional('deps_os'): _NodeDictSchema({
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +0200139 schema.Optional(basestring): _GCLIENT_DEPS_SCHEMA,
Edward Lesmes6f64a052018-03-20 17:35:49 -0400140 }),
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200141
Michael Moss848c86e2018-05-03 16:05:50 -0700142 # Dependency to get gclient_gn_args* settings from. This allows these values
143 # to be set in a recursedeps file, rather than requiring that they exist in
144 # the top-level solution.
145 schema.Optional('gclient_gn_args_from'): basestring,
146
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200147 # Path to GN args file to write selected variables.
148 schema.Optional('gclient_gn_args_file'): basestring,
149
150 # Subset of variables to write to the GN args file (see above).
151 schema.Optional('gclient_gn_args'): [schema.Optional(basestring)],
152
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200153 # Hooks executed after gclient sync (unless suppressed), or explicitly
154 # on gclient hooks. See _GCLIENT_HOOKS_SCHEMA for details.
155 # Also see 'pre_deps_hooks'.
156 schema.Optional('hooks'): _GCLIENT_HOOKS_SCHEMA,
157
Scott Grahamc4826742017-05-11 16:59:23 -0700158 # Similar to 'hooks', also keyed by OS.
Edward Lesmes6f64a052018-03-20 17:35:49 -0400159 schema.Optional('hooks_os'): _NodeDictSchema({
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200160 schema.Optional(basestring): _GCLIENT_HOOKS_SCHEMA
Edward Lesmes6f64a052018-03-20 17:35:49 -0400161 }),
Scott Grahamc4826742017-05-11 16:59:23 -0700162
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200163 # Rules which #includes are allowed in the directory.
164 # Also see 'skip_child_includes' and 'specific_include_rules'.
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200165 schema.Optional('include_rules'): [schema.Optional(basestring)],
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200166
167 # Hooks executed before processing DEPS. See 'hooks' for more details.
168 schema.Optional('pre_deps_hooks'): _GCLIENT_HOOKS_SCHEMA,
169
Paweł Hajdan, Jr6f796792017-06-02 08:40:06 +0200170 # Recursion limit for nested DEPS.
171 schema.Optional('recursion'): int,
172
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200173 # Whitelists deps for which recursion should be enabled.
174 schema.Optional('recursedeps'): [
Paweł Hajdan, Jr05fec032017-05-30 23:04:23 +0200175 schema.Optional(schema.Or(
176 basestring,
177 (basestring, basestring),
178 [basestring, basestring]
179 )),
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200180 ],
181
182 # Blacklists directories for checking 'include_rules'.
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200183 schema.Optional('skip_child_includes'): [schema.Optional(basestring)],
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200184
185 # Mapping from paths to include rules specific for that path.
186 # See 'include_rules' for more details.
Edward Lesmes6f64a052018-03-20 17:35:49 -0400187 schema.Optional('specific_include_rules'): _NodeDictSchema({
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200188 schema.Optional(basestring): [basestring]
Edward Lesmes6f64a052018-03-20 17:35:49 -0400189 }),
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200190
191 # List of additional OS names to consider when selecting dependencies
192 # from deps_os.
193 schema.Optional('target_os'): [schema.Optional(basestring)],
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200194
195 # For recursed-upon sub-dependencies, check out their own dependencies
196 # relative to the paren't path, rather than relative to the .gclient file.
197 schema.Optional('use_relative_paths'): bool,
198
199 # Variables that can be referenced using Var() - see 'deps'.
Edward Lesmes6f64a052018-03-20 17:35:49 -0400200 schema.Optional('vars'): _NodeDictSchema({
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200201 schema.Optional(basestring): schema.Or(basestring, bool),
Edward Lesmes6f64a052018-03-20 17:35:49 -0400202 }),
203}))
Paweł Hajdan, Jrbeec0062017-05-10 21:51:05 +0200204
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200205
Edward Lesmes6c24d372018-03-28 12:52:29 -0400206def _gclient_eval(node_or_string, vars_dict=None, expand_vars=False,
207 filename='<unknown>'):
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200208 """Safely evaluates a single expression. Returns the result."""
209 _allowed_names = {'None': None, 'True': True, 'False': False}
210 if isinstance(node_or_string, basestring):
211 node_or_string = ast.parse(node_or_string, filename=filename, mode='eval')
212 if isinstance(node_or_string, ast.Expression):
213 node_or_string = node_or_string.body
214 def _convert(node):
215 if isinstance(node, ast.Str):
Edward Lesmes6c24d372018-03-28 12:52:29 -0400216 if not expand_vars:
217 return node.s
218 try:
219 return node.s.format(**vars_dict)
220 except KeyError as e:
221 raise ValueError(
222 '%s was used as a variable, but was not declared in the vars dict '
223 '(file %r, line %s)' % (
224 e.message, filename, getattr(node, 'lineno', '<unknown>')))
Paweł Hajdan, Jr6f796792017-06-02 08:40:06 +0200225 elif isinstance(node, ast.Num):
226 return node.n
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200227 elif isinstance(node, ast.Tuple):
228 return tuple(map(_convert, node.elts))
229 elif isinstance(node, ast.List):
230 return list(map(_convert, node.elts))
231 elif isinstance(node, ast.Dict):
Edward Lesmes6f64a052018-03-20 17:35:49 -0400232 return _NodeDict((_convert(k), (_convert(v), v))
Paweł Hajdan, Jr7cf96a42017-05-26 20:28:35 +0200233 for k, v in zip(node.keys, node.values))
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200234 elif isinstance(node, ast.Name):
235 if node.id not in _allowed_names:
236 raise ValueError(
237 'invalid name %r (file %r, line %s)' % (
238 node.id, filename, getattr(node, 'lineno', '<unknown>')))
239 return _allowed_names[node.id]
240 elif isinstance(node, ast.Call):
Edward Lesmes9f531292018-03-20 21:27:15 -0400241 if not isinstance(node.func, ast.Name) or node.func.id != 'Var':
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200242 raise ValueError(
Edward Lesmes9f531292018-03-20 21:27:15 -0400243 'Var is the only allowed function (file %r, line %s)' % (
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200244 filename, getattr(node, 'lineno', '<unknown>')))
Edward Lesmes9f531292018-03-20 21:27:15 -0400245 if node.keywords or node.starargs or node.kwargs or len(node.args) != 1:
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200246 raise ValueError(
Edward Lesmes9f531292018-03-20 21:27:15 -0400247 'Var takes exactly one argument (file %r, line %s)' % (
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200248 filename, getattr(node, 'lineno', '<unknown>')))
Edward Lesmes9f531292018-03-20 21:27:15 -0400249 arg = _convert(node.args[0])
250 if not isinstance(arg, basestring):
251 raise ValueError(
252 'Var\'s argument must be a variable name (file %r, line %s)' % (
253 filename, getattr(node, 'lineno', '<unknown>')))
Edward Lesmes6c24d372018-03-28 12:52:29 -0400254 if not expand_vars:
255 return '{%s}' % arg
256 if vars_dict is None:
257 raise ValueError(
258 'vars must be declared before Var can be used (file %r, line %s)'
259 % (filename, getattr(node, 'lineno', '<unknown>')))
260 if arg not in vars_dict:
261 raise ValueError(
262 '%s was used as a variable, but was not declared in the vars dict '
263 '(file %r, line %s)' % (
264 arg, filename, getattr(node, 'lineno', '<unknown>')))
265 return vars_dict[arg]
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200266 elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
267 return _convert(node.left) + _convert(node.right)
Paweł Hajdan, Jrb7e53332017-05-23 16:57:37 +0200268 elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mod):
269 return _convert(node.left) % _convert(node.right)
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200270 else:
271 raise ValueError(
Paweł Hajdan, Jr1ba610b2017-05-24 20:14:44 +0200272 'unexpected AST node: %s %s (file %r, line %s)' % (
273 node, ast.dump(node), filename,
274 getattr(node, 'lineno', '<unknown>')))
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200275 return _convert(node_or_string)
276
277
Edward Lesmes6c24d372018-03-28 12:52:29 -0400278def Exec(content, expand_vars=True, filename='<unknown>', vars_override=None):
279 """Safely execs a set of assignments."""
280 def _validate_statement(node, local_scope):
281 if not isinstance(node, ast.Assign):
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200282 raise ValueError(
Paweł Hajdan, Jr1ba610b2017-05-24 20:14:44 +0200283 'unexpected AST node: %s %s (file %r, line %s)' % (
284 node, ast.dump(node), filename,
285 getattr(node, 'lineno', '<unknown>')))
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200286
Edward Lesmes6c24d372018-03-28 12:52:29 -0400287 if len(node.targets) != 1:
288 raise ValueError(
289 'invalid assignment: use exactly one target (file %r, line %s)' % (
290 filename, getattr(node, 'lineno', '<unknown>')))
291
292 target = node.targets[0]
293 if not isinstance(target, ast.Name):
294 raise ValueError(
295 'invalid assignment: target should be a name (file %r, line %s)' % (
296 filename, getattr(node, 'lineno', '<unknown>')))
297 if target.id in local_scope:
298 raise ValueError(
299 'invalid assignment: overrides var %r (file %r, line %s)' % (
300 target.id, filename, getattr(node, 'lineno', '<unknown>')))
301
302 node_or_string = ast.parse(content, filename=filename, mode='exec')
303 if isinstance(node_or_string, ast.Expression):
304 node_or_string = node_or_string.body
305
306 if not isinstance(node_or_string, ast.Module):
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200307 raise ValueError(
Paweł Hajdan, Jr1ba610b2017-05-24 20:14:44 +0200308 'unexpected AST node: %s %s (file %r, line %s)' % (
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200309 node_or_string,
Paweł Hajdan, Jr1ba610b2017-05-24 20:14:44 +0200310 ast.dump(node_or_string),
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200311 filename,
312 getattr(node_or_string, 'lineno', '<unknown>')))
313
Edward Lesmes6c24d372018-03-28 12:52:29 -0400314 statements = {}
315 for statement in node_or_string.body:
316 _validate_statement(statement, statements)
317 statements[statement.targets[0].id] = statement.value
318
319 tokens = {
320 token[2]: list(token)
321 for token in tokenize.generate_tokens(
322 cStringIO.StringIO(content).readline)
323 }
324 local_scope = _NodeDict({}, tokens)
325
326 # Process vars first, so we can expand variables in the rest of the DEPS file.
327 vars_dict = {}
328 if 'vars' in statements:
329 vars_statement = statements['vars']
330 value = _gclient_eval(vars_statement, None, False, filename)
331 local_scope.SetNode('vars', value, vars_statement)
332 # Update the parsed vars with the overrides, but only if they are already
333 # present (overrides do not introduce new variables).
334 vars_dict.update(value)
335 if vars_override:
336 vars_dict.update({
337 k: v
338 for k, v in vars_override.iteritems()
339 if k in vars_dict})
340
341 for name, node in statements.iteritems():
342 value = _gclient_eval(node, vars_dict, expand_vars, filename)
343 local_scope.SetNode(name, value, node)
344
John Budorick0f7b2002018-01-19 15:46:17 -0800345 return _GCLIENT_SCHEMA.validate(local_scope)
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200346
347
Edward Lesmes6c24d372018-03-28 12:52:29 -0400348def Parse(content, expand_vars, validate_syntax, filename, vars_override=None):
349 """Parses DEPS strings.
350
351 Executes the Python-like string stored in content, resulting in a Python
352 dictionary specifyied by the schema above. Supports syntax validation and
353 variable expansion.
354
355 Args:
356 content: str. DEPS file stored as a string.
357 expand_vars: bool. Whether variables should be expanded to their values.
358 validate_syntax: bool. Whether syntax should be validated using the schema
359 defined above.
360 filename: str. The name of the DEPS file, or a string describing the source
361 of the content, e.g. '<string>', '<unknown>'.
362 vars_override: dict, optional. A dictionary with overrides for the variables
363 defined by the DEPS file.
364
365 Returns:
366 A Python dict with the parsed contents of the DEPS file, as specified by the
367 schema above.
368 """
369 # TODO(ehmaldonado): Make validate_syntax = True the only case
370 if validate_syntax:
371 return Exec(content, expand_vars, filename, vars_override)
372
373 local_scope = {}
374 global_scope = {'Var': lambda var_name: '{%s}' % var_name}
375
376 # If we use 'exec' directly, it complains that 'Parse' contains a nested
377 # function with free variables.
378 # This is because on versions of Python < 2.7.9, "exec(a, b, c)" not the same
379 # as "exec a in b, c" (See https://bugs.python.org/issue21591).
380 eval(compile(content, filename, 'exec'), global_scope, local_scope)
381
382 if 'vars' not in local_scope or not expand_vars:
383 return local_scope
384
385 vars_dict = {}
386 vars_dict.update(local_scope['vars'])
387 if vars_override:
388 vars_dict.update({
389 k: v
390 for k, v in vars_override.iteritems()
391 if k in vars_dict
392 })
393
394 def _DeepFormat(node):
395 if isinstance(node, basestring):
396 return node.format(**vars_dict)
397 elif isinstance(node, dict):
398 return {
399 k.format(**vars_dict): _DeepFormat(v)
400 for k, v in node.iteritems()
401 }
402 elif isinstance(node, list):
403 return [_DeepFormat(elem) for elem in node]
404 elif isinstance(node, tuple):
405 return tuple(_DeepFormat(elem) for elem in node)
406 else:
407 return node
408
409 return _DeepFormat(local_scope)
410
411
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200412def EvaluateCondition(condition, variables, referenced_variables=None):
413 """Safely evaluates a boolean condition. Returns the result."""
414 if not referenced_variables:
415 referenced_variables = set()
416 _allowed_names = {'None': None, 'True': True, 'False': False}
417 main_node = ast.parse(condition, mode='eval')
418 if isinstance(main_node, ast.Expression):
419 main_node = main_node.body
420 def _convert(node):
421 if isinstance(node, ast.Str):
422 return node.s
423 elif isinstance(node, ast.Name):
424 if node.id in referenced_variables:
425 raise ValueError(
426 'invalid cyclic reference to %r (inside %r)' % (
427 node.id, condition))
428 elif node.id in _allowed_names:
429 return _allowed_names[node.id]
430 elif node.id in variables:
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200431 value = variables[node.id]
432
433 # Allow using "native" types, without wrapping everything in strings.
434 # Note that schema constraints still apply to variables.
435 if not isinstance(value, basestring):
436 return value
437
438 # Recursively evaluate the variable reference.
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200439 return EvaluateCondition(
440 variables[node.id],
441 variables,
442 referenced_variables.union([node.id]))
443 else:
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200444 # Implicitly convert unrecognized names to strings.
445 # If we want to change this, we'll need to explicitly distinguish
446 # between arguments for GN to be passed verbatim, and ones to
447 # be evaluated.
448 return node.id
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200449 elif isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or):
450 if len(node.values) != 2:
451 raise ValueError(
452 'invalid "or": exactly 2 operands required (inside %r)' % (
453 condition))
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200454 left = _convert(node.values[0])
455 right = _convert(node.values[1])
456 if not isinstance(left, bool):
457 raise ValueError(
458 'invalid "or" operand %r (inside %r)' % (left, condition))
459 if not isinstance(right, bool):
460 raise ValueError(
461 'invalid "or" operand %r (inside %r)' % (right, condition))
462 return left or right
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200463 elif isinstance(node, ast.BoolOp) and isinstance(node.op, ast.And):
464 if len(node.values) != 2:
465 raise ValueError(
466 'invalid "and": exactly 2 operands required (inside %r)' % (
467 condition))
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200468 left = _convert(node.values[0])
469 right = _convert(node.values[1])
470 if not isinstance(left, bool):
471 raise ValueError(
472 'invalid "and" operand %r (inside %r)' % (left, condition))
473 if not isinstance(right, bool):
474 raise ValueError(
475 'invalid "and" operand %r (inside %r)' % (right, condition))
476 return left and right
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200477 elif isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +0200478 value = _convert(node.operand)
479 if not isinstance(value, bool):
480 raise ValueError(
481 'invalid "not" operand %r (inside %r)' % (value, condition))
482 return not value
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200483 elif isinstance(node, ast.Compare):
484 if len(node.ops) != 1:
485 raise ValueError(
486 'invalid compare: exactly 1 operator required (inside %r)' % (
487 condition))
488 if len(node.comparators) != 1:
489 raise ValueError(
490 'invalid compare: exactly 1 comparator required (inside %r)' % (
491 condition))
492
493 left = _convert(node.left)
494 right = _convert(node.comparators[0])
495
496 if isinstance(node.ops[0], ast.Eq):
497 return left == right
Dirk Pranke77b76872017-10-05 18:29:27 -0700498 if isinstance(node.ops[0], ast.NotEq):
499 return left != right
Paweł Hajdan, Jr76c6ea22017-06-02 21:46:57 +0200500
501 raise ValueError(
502 'unexpected operator: %s %s (inside %r)' % (
503 node.ops[0], ast.dump(node), condition))
504 else:
505 raise ValueError(
506 'unexpected AST node: %s %s (inside %r)' % (
507 node, ast.dump(node), condition))
508 return _convert(main_node)
Edward Lesmes6f64a052018-03-20 17:35:49 -0400509
510
511def RenderDEPSFile(gclient_dict):
512 contents = sorted(gclient_dict.tokens.values(), key=lambda token: token[2])
513 return tokenize.untokenize(contents)
514
515
516def _UpdateAstString(tokens, node, value):
517 position = node.lineno, node.col_offset
Edward Lesmes62af4e42018-03-30 18:15:44 -0400518 quote_char = tokens[position][1][0]
519 tokens[position][1] = quote_char + value + quote_char
Edward Lesmes6f64a052018-03-20 17:35:49 -0400520 node.s = value
521
522
Edward Lesmes3d993812018-04-02 12:52:49 -0400523def _ShiftLinesInTokens(tokens, delta, start):
524 new_tokens = {}
525 for token in tokens.values():
526 if token[2][0] >= start:
527 token[2] = token[2][0] + delta, token[2][1]
528 token[3] = token[3][0] + delta, token[3][1]
529 new_tokens[token[2]] = token
530 return new_tokens
531
532
533def AddVar(gclient_dict, var_name, value):
534 if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
535 raise ValueError(
536 "Can't use SetVar for the given gclient dict. It contains no "
537 "formatting information.")
538
539 if 'vars' not in gclient_dict:
540 raise KeyError("vars dict is not defined.")
541
542 if var_name in gclient_dict['vars']:
543 raise ValueError(
544 "%s has already been declared in the vars dict. Consider using SetVar "
545 "instead." % var_name)
546
547 if not gclient_dict['vars']:
548 raise ValueError('vars dict is empty. This is not yet supported.')
549
Edward Lesmes8d626572018-04-05 17:53:10 -0400550 # We will attempt to add the var right after 'vars = {'.
551 node = gclient_dict.GetNode('vars')
Edward Lesmes3d993812018-04-02 12:52:49 -0400552 if node is None:
553 raise ValueError(
554 "The vars dict has no formatting information." % var_name)
Edward Lesmes8d626572018-04-05 17:53:10 -0400555 line = node.lineno + 1
556
557 # We will try to match the new var's indentation to the next variable.
558 col = node.keys[0].col_offset
Edward Lesmes3d993812018-04-02 12:52:49 -0400559
560 # We use a minimal Python dictionary, so that ast can parse it.
561 var_content = '{\n%s"%s": "%s",\n}' % (' ' * col, var_name, value)
562 var_ast = ast.parse(var_content).body[0].value
563
564 # Set the ast nodes for the key and value.
565 vars_node = gclient_dict.GetNode('vars')
566
567 var_name_node = var_ast.keys[0]
568 var_name_node.lineno += line - 2
569 vars_node.keys.insert(0, var_name_node)
570
571 value_node = var_ast.values[0]
572 value_node.lineno += line - 2
573 vars_node.values.insert(0, value_node)
574
575 # Update the tokens.
576 var_tokens = list(tokenize.generate_tokens(
577 cStringIO.StringIO(var_content).readline))
578 var_tokens = {
579 token[2]: list(token)
580 # Ignore the tokens corresponding to braces and new lines.
581 for token in var_tokens[2:-2]
582 }
583
584 gclient_dict.tokens = _ShiftLinesInTokens(gclient_dict.tokens, 1, line)
585 gclient_dict.tokens.update(_ShiftLinesInTokens(var_tokens, line - 2, 0))
586
587
Edward Lesmes6f64a052018-03-20 17:35:49 -0400588def SetVar(gclient_dict, var_name, value):
589 if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
590 raise ValueError(
591 "Can't use SetVar for the given gclient dict. It contains no "
592 "formatting information.")
593 tokens = gclient_dict.tokens
594
Edward Lesmes3d993812018-04-02 12:52:49 -0400595 if 'vars' not in gclient_dict:
596 raise KeyError("vars dict is not defined.")
597
598 if var_name not in gclient_dict['vars']:
Edward Lesmes6f64a052018-03-20 17:35:49 -0400599 raise ValueError(
Edward Lesmes3d993812018-04-02 12:52:49 -0400600 "%s has not been declared in the vars dict. Consider using AddVar "
601 "instead." % var_name)
Edward Lesmes6f64a052018-03-20 17:35:49 -0400602
603 node = gclient_dict['vars'].GetNode(var_name)
604 if node is None:
605 raise ValueError(
606 "The vars entry for %s has no formatting information." % var_name)
607
608 _UpdateAstString(tokens, node, value)
Edward Lesmes6c24d372018-03-28 12:52:29 -0400609 gclient_dict['vars'].SetNode(var_name, value, node)
Edward Lesmes6f64a052018-03-20 17:35:49 -0400610
611
612def SetCIPD(gclient_dict, dep_name, package_name, new_version):
613 if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
614 raise ValueError(
615 "Can't use SetCIPD for the given gclient dict. It contains no "
616 "formatting information.")
617 tokens = gclient_dict.tokens
618
619 if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
Edward Lesmes3d993812018-04-02 12:52:49 -0400620 raise KeyError(
Edward Lesmes6f64a052018-03-20 17:35:49 -0400621 "Could not find any dependency called %s." % dep_name)
622
623 # Find the package with the given name
624 packages = [
625 package
626 for package in gclient_dict['deps'][dep_name]['packages']
627 if package['package'] == package_name
628 ]
629 if len(packages) != 1:
630 raise ValueError(
631 "There must be exactly one package with the given name (%s), "
632 "%s were found." % (package_name, len(packages)))
633
634 # TODO(ehmaldonado): Support Var in package's version.
635 node = packages[0].GetNode('version')
636 if node is None:
637 raise ValueError(
638 "The deps entry for %s:%s has no formatting information." %
639 (dep_name, package_name))
640
641 new_version = 'version:' + new_version
642 _UpdateAstString(tokens, node, new_version)
Edward Lesmes6c24d372018-03-28 12:52:29 -0400643 packages[0].SetNode('version', new_version, node)
Edward Lesmes6f64a052018-03-20 17:35:49 -0400644
645
Edward Lesmes9f531292018-03-20 21:27:15 -0400646def SetRevision(gclient_dict, dep_name, new_revision):
Edward Lesmes62af4e42018-03-30 18:15:44 -0400647 def _GetVarName(node):
648 if isinstance(node, ast.Call):
649 return node.args[0].s
650 elif node.s.endswith('}'):
651 last_brace = node.s.rfind('{')
652 return node.s[last_brace+1:-1]
653 return None
654
655 def _UpdateRevision(dep_dict, dep_key, new_revision):
656 dep_node = dep_dict.GetNode(dep_key)
657 if dep_node is None:
658 raise ValueError(
659 "The deps entry for %s has no formatting information." % dep_name)
660
661 node = dep_node
662 if isinstance(node, ast.BinOp):
663 node = node.right
664
665 if not isinstance(node, ast.Call) and not isinstance(node, ast.Str):
666 raise ValueError(
667 "Unsupported dependency revision format. Please file a bug.")
668
669 var_name = _GetVarName(node)
670 if var_name is not None:
671 SetVar(gclient_dict, var_name, new_revision)
672 else:
673 if '@' in node.s:
Edward Lesmes1118a212018-04-05 18:37:07 -0400674 # '@' is part of the last string, which we want to modify. Discard
675 # whatever was after the '@' and put the new revision in its place.
Edward Lesmes62af4e42018-03-30 18:15:44 -0400676 new_revision = node.s.split('@')[0] + '@' + new_revision
Edward Lesmes1118a212018-04-05 18:37:07 -0400677 elif '@' not in dep_dict[dep_key]:
678 # '@' is not part of the URL at all. This mean the dependency is
679 # unpinned and we should pin it.
680 new_revision = node.s + '@' + new_revision
Edward Lesmes62af4e42018-03-30 18:15:44 -0400681 _UpdateAstString(tokens, node, new_revision)
682 dep_dict.SetNode(dep_key, new_revision, node)
683
Edward Lesmes6f64a052018-03-20 17:35:49 -0400684 if not isinstance(gclient_dict, _NodeDict) or gclient_dict.tokens is None:
685 raise ValueError(
686 "Can't use SetRevision for the given gclient dict. It contains no "
687 "formatting information.")
688 tokens = gclient_dict.tokens
689
690 if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
Edward Lesmes3d993812018-04-02 12:52:49 -0400691 raise KeyError(
Edward Lesmes6f64a052018-03-20 17:35:49 -0400692 "Could not find any dependency called %s." % dep_name)
693
Edward Lesmes6f64a052018-03-20 17:35:49 -0400694 if isinstance(gclient_dict['deps'][dep_name], _NodeDict):
Edward Lesmes62af4e42018-03-30 18:15:44 -0400695 _UpdateRevision(gclient_dict['deps'][dep_name], 'url', new_revision)
Edward Lesmes6f64a052018-03-20 17:35:49 -0400696 else:
Edward Lesmes62af4e42018-03-30 18:15:44 -0400697 _UpdateRevision(gclient_dict['deps'], dep_name, new_revision)
Edward Lesmes411041f2018-04-05 20:12:55 -0400698
699
700def GetVar(gclient_dict, var_name):
701 if 'vars' not in gclient_dict or var_name not in gclient_dict['vars']:
702 raise KeyError(
703 "Could not find any variable called %s." % var_name)
704
705 return gclient_dict['vars'][var_name]
706
707
708def GetCIPD(gclient_dict, dep_name, package_name):
709 if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
710 raise KeyError(
711 "Could not find any dependency called %s." % dep_name)
712
713 # Find the package with the given name
714 packages = [
715 package
716 for package in gclient_dict['deps'][dep_name]['packages']
717 if package['package'] == package_name
718 ]
719 if len(packages) != 1:
720 raise ValueError(
721 "There must be exactly one package with the given name (%s), "
722 "%s were found." % (package_name, len(packages)))
723
724 return packages[0]['version'][len('version:'):]
725
726
727def GetRevision(gclient_dict, dep_name):
728 if 'deps' not in gclient_dict or dep_name not in gclient_dict['deps']:
729 raise KeyError(
730 "Could not find any dependency called %s." % dep_name)
731
732 dep = gclient_dict['deps'][dep_name]
733 if dep is None:
734 return None
735 elif isinstance(dep, basestring):
736 _, _, revision = dep.partition('@')
737 return revision or None
738 elif isinstance(dep, collections.Mapping) and 'url' in dep:
739 _, _, revision = dep['url'].partition('@')
740 return revision or None
741 else:
742 raise ValueError(
743 '%s is not a valid git dependency.' % dep_name)