blob: 92c477f1d64f5d97e33d7ce2dabcec7eb0d7d849 [file] [log] [blame]
iannucci@chromium.org405b87e2015-11-12 18:08:34 +00001#!/usr/bin/env python
thakis@chromium.org4f474b62012-01-18 01:31:29 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgba551772010-02-03 18:21:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00005
agabled437d762016-10-17 09:35:11 -07006"""Meta checkout dependency manager for Git."""
maruel@chromium.org39c0b222013-08-17 16:57:01 +00007# Files
8# .gclient : Current client configuration, written by 'config' command.
9# Format is a Python script defining 'solutions', a list whose
10# entries each are maps binding the strings "name" and "url"
11# to strings specifying the name and location of the client
12# module, as well as "custom_deps" to a map similar to the
13# deps section of the DEPS file below, as well as
14# "custom_hooks" to a list similar to the hooks sections of
15# the DEPS file below.
16# .gclient_entries : A cache constructed by 'update' command. Format is a
17# Python script defining 'entries', a list of the names
18# of all modules in the client
19# <module>/DEPS : Python script defining var 'deps' as a map from each
20# requisite submodule name to a URL where it can be found (via
21# one SCM)
22#
23# Hooks
24# .gclient and DEPS files may optionally contain a list named "hooks" to
25# allow custom actions to be performed based on files that have changed in the
26# working copy as a result of a "sync"/"update" or "revert" operation. This
27# can be prevented by using --nohooks (hooks run by default). Hooks can also
28# be forced to run with the "runhooks" operation. If "sync" is run with
29# --force, all known but not suppressed hooks will run regardless of the state
30# of the working copy.
31#
32# Each item in a "hooks" list is a dict, containing these two keys:
33# "pattern" The associated value is a string containing a regular
34# expression. When a file whose pathname matches the expression
35# is checked out, updated, or reverted, the hook's "action" will
36# run.
37# "action" A list describing a command to run along with its arguments, if
38# any. An action command will run at most one time per gclient
39# invocation, regardless of how many files matched the pattern.
40# The action is executed in the same directory as the .gclient
41# file. If the first item in the list is the string "python",
42# the current Python interpreter (sys.executable) will be used
43# to run the command. If the list contains string
44# "$matching_files" it will be removed from the list and the list
45# will be extended by the list of matching files.
46# "name" An optional string specifying the group to which a hook belongs
47# for overriding and organizing.
48#
49# Example:
50# hooks = [
51# { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
52# "action": ["python", "image_indexer.py", "--all"]},
53# { "pattern": ".",
54# "name": "gyp",
55# "action": ["python", "src/build/gyp_chromium"]},
56# ]
57#
borenet@google.com2d1ee9e2013-10-15 08:13:16 +000058# Pre-DEPS Hooks
59# DEPS files may optionally contain a list named "pre_deps_hooks". These are
60# the same as normal hooks, except that they run before the DEPS are
61# processed. Pre-DEPS run with "sync" and "revert" unless the --noprehooks
62# flag is used.
rdsmith@chromium.orgd9591f02014-02-05 19:28:20 +000063#
maruel@chromium.org39c0b222013-08-17 16:57:01 +000064# Specifying a target OS
65# An optional key named "target_os" may be added to a gclient file to specify
66# one or more additional operating systems that should be considered when
Scott Grahamc4826742017-05-11 16:59:23 -070067# processing the deps_os/hooks_os dict of a DEPS file.
maruel@chromium.org39c0b222013-08-17 16:57:01 +000068#
69# Example:
70# target_os = [ "android" ]
71#
72# If the "target_os_only" key is also present and true, then *only* the
73# operating systems listed in "target_os" will be used.
74#
75# Example:
76# target_os = [ "ios" ]
77# target_os_only = True
Tom Andersonc31ae0b2018-02-06 14:48:56 -080078#
79# Specifying a target CPU
80# To specify a target CPU, the variables target_cpu and target_cpu_only
81# are available and are analagous to target_os and target_os_only.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000082
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +000083from __future__ import print_function
84
maruel@chromium.org39c0b222013-08-17 16:57:01 +000085__version__ = '0.7'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000086
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +020087import collections
maruel@chromium.org9e5317a2010-08-13 20:35:11 +000088import copy
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +000089import json
maruel@chromium.org754960e2009-09-21 12:31:05 +000090import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000091import optparse
92import os
bradnelson@google.com4949dab2012-04-19 16:41:07 +000093import platform
maruel@chromium.org621939b2010-08-10 20:12:00 +000094import posixpath
msb@chromium.org2e38de72009-09-28 17:04:47 +000095import pprint
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000096import re
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000097import sys
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +000098import time
bradnelson@google.com4949dab2012-04-19 16:41:07 +000099import urlparse
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000100
Tom Andersonc31ae0b2018-02-06 14:48:56 -0800101import detect_host_arch
maruel@chromium.org35625c72011-03-23 17:34:02 +0000102import fix_encoding
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200103import gclient_eval
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000104import gclient_scm
105import gclient_utils
szager@chromium.org848fd492014-04-09 19:06:44 +0000106import git_cache
nasser@codeaurora.org1f7a3d12010-02-04 15:11:50 +0000107from third_party.repo.progress import Progress
maruel@chromium.org39c0b222013-08-17 16:57:01 +0000108import subcommand
maruel@chromium.org31cb48a2011-04-04 18:01:36 +0000109import subprocess2
iannucci@chromium.org596cd5c2016-04-04 21:34:39 +0000110import setup_color
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000111
112
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200113class GNException(Exception):
114 pass
115
116
117def ToGNString(value, allow_dicts = True):
118 """Returns a stringified GN equivalent of the Python value.
119
120 allow_dicts indicates if this function will allow converting dictionaries
121 to GN scopes. This is only possible at the top level, you can't nest a
122 GN scope in a list, so this should be set to False for recursive calls."""
123 if isinstance(value, basestring):
124 if value.find('\n') >= 0:
125 raise GNException("Trying to print a string with a newline in it.")
126 return '"' + \
127 value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \
128 '"'
129
130 if isinstance(value, unicode):
131 return ToGNString(value.encode('utf-8'))
132
133 if isinstance(value, bool):
134 if value:
135 return "true"
136 return "false"
137
138 # NOTE: some type handling removed compared to chromium/src copy.
139
140 raise GNException("Unsupported type when printing to GN.")
141
142
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200143class Hook(object):
144 """Descriptor of command ran before/after sync or on demand."""
145
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200146 def __init__(self, action, pattern=None, name=None, cwd=None, condition=None,
Daniel Chenga0c5f082017-10-19 13:35:19 -0700147 variables=None, verbose=False):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200148 """Constructor.
149
150 Arguments:
151 action (list of basestring): argv of the command to run
152 pattern (basestring regex): noop with git; deprecated
153 name (basestring): optional name; no effect on operation
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200154 cwd (basestring): working directory to use
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200155 condition (basestring): condition when to run the hook
156 variables (dict): variables for evaluating the condition
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200157 """
158 self._action = gclient_utils.freeze(action)
159 self._pattern = pattern
160 self._name = name
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200161 self._cwd = cwd
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200162 self._condition = condition
163 self._variables = variables
Daniel Chenga0c5f082017-10-19 13:35:19 -0700164 self._verbose = verbose
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200165
166 @staticmethod
Michael Moss42d02c22018-02-05 10:32:24 -0800167 def from_dict(d, variables=None, verbose=False, conditions=None):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200168 """Creates a Hook instance from a dict like in the DEPS file."""
Michael Moss42d02c22018-02-05 10:32:24 -0800169 # Merge any local and inherited conditions.
170 if conditions and d.get('condition'):
171 condition = '(%s) and (%s)' % (conditions, d['condition'])
172 else:
173 condition = conditions or d.get('condition')
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200174 return Hook(
175 d['action'],
176 d.get('pattern'),
177 d.get('name'),
178 d.get('cwd'),
Michael Moss42d02c22018-02-05 10:32:24 -0800179 condition,
Daniel Chenga0c5f082017-10-19 13:35:19 -0700180 variables=variables,
181 # Always print the header if not printing to a TTY.
182 verbose=verbose or not setup_color.IS_TTY)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200183
184 @property
185 def action(self):
186 return self._action
187
188 @property
189 def pattern(self):
190 return self._pattern
191
192 @property
193 def name(self):
194 return self._name
195
Paweł Hajdan, Jrecf53fe2017-09-29 18:28:49 +0200196 @property
197 def condition(self):
198 return self._condition
199
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200200 def matches(self, file_list):
201 """Returns true if the pattern matches any of files in the list."""
202 if not self._pattern:
203 return True
204 pattern = re.compile(self._pattern)
205 return bool([f for f in file_list if pattern.search(f)])
206
207 def run(self, root):
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200208 """Executes the hook's command (provided the condition is met)."""
209 if (self._condition and
210 not gclient_eval.EvaluateCondition(self._condition, self._variables)):
211 return
212
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200213 cmd = [arg.format(**self._variables) for arg in self._action]
214
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200215 if cmd[0] == 'python':
216 # If the hook specified "python" as the first item, the action is a
217 # Python script. Run it by starting a new copy of the same
218 # interpreter.
219 cmd[0] = sys.executable
Nodir Turakulov0ffcc872017-11-09 16:44:58 -0800220 elif cmd[0] == 'vpython' and _detect_host_os() == 'win':
221 cmd[0] += '.bat'
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200222
223 cwd = root
224 if self._cwd:
225 cwd = os.path.join(cwd, self._cwd)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200226 try:
227 start_time = time.time()
228 gclient_utils.CheckCallAndFilterAndHeader(
Daniel Chenga0c5f082017-10-19 13:35:19 -0700229 cmd, cwd=cwd, always=self._verbose)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200230 except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
231 # Use a discrete exit status code of 2 to indicate that a hook action
232 # failed. Users of this script may wish to treat hook action failures
233 # differently from VC failures.
234 print('Error: %s' % str(e), file=sys.stderr)
235 sys.exit(2)
236 finally:
237 elapsed_time = time.time() - start_time
238 if elapsed_time > 10:
239 print("Hook '%s' took %.2f secs" % (
240 gclient_utils.CommandToStr(cmd), elapsed_time))
241
242
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200243class DependencySettings(object):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000244 """Immutable configuration settings."""
245 def __init__(
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200246 self, parent, raw_url, url, managed, custom_deps, custom_vars,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200247 custom_hooks, deps_file, should_process, relative,
248 condition, condition_value):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000249 # These are not mutable:
250 self._parent = parent
mmoss@chromium.org8f93f792014-08-26 23:24:09 +0000251 self._deps_file = deps_file
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200252 self._raw_url = raw_url
maruel@chromium.org064186c2011-09-27 23:53:33 +0000253 self._url = url
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200254 # The condition as string (or None). Useful to keep e.g. for flatten.
255 self._condition = condition
256 # Boolean value of the condition. If there's no condition, just True.
257 self._condition_value = condition_value
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000258 # 'managed' determines whether or not this dependency is synced/updated by
259 # gclient after gclient checks it out initially. The difference between
260 # 'managed' and 'should_process' is that the user specifies 'managed' via
smutae7ea312016-07-18 11:59:41 -0700261 # the --unmanaged command-line flag or a .gclient config, where
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000262 # 'should_process' is dynamically set by gclient if it goes over its
263 # recursion limit and controls gclient's behavior so it does not misbehave.
264 self._managed = managed
265 self._should_process = should_process
agabledce6ddc2016-09-08 10:02:16 -0700266 # If this is a recursed-upon sub-dependency, and the parent has
267 # use_relative_paths set, then this dependency should check out its own
268 # dependencies relative to that parent's path for this, rather than
269 # relative to the .gclient file.
270 self._relative = relative
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +0000271 # This is a mutable value which has the list of 'target_os' OSes listed in
272 # the current deps file.
273 self.local_target_os = None
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000274
275 # These are only set in .gclient and not in DEPS files.
276 self._custom_vars = custom_vars or {}
277 self._custom_deps = custom_deps or {}
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000278 self._custom_hooks = custom_hooks or []
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000279
maruel@chromium.org064186c2011-09-27 23:53:33 +0000280 # Post process the url to remove trailing slashes.
281 if isinstance(self._url, basestring):
282 # urls are sometime incorrectly written as proto://host/path/@rev. Replace
283 # it to proto://host/path@rev.
maruel@chromium.org064186c2011-09-27 23:53:33 +0000284 self._url = self._url.replace('/@', '@')
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200285 elif not isinstance(self._url, (None.__class__)):
maruel@chromium.org064186c2011-09-27 23:53:33 +0000286 raise gclient_utils.Error(
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200287 ('dependency url must be either string or None, '
288 'instead of %s') % self._url.__class__.__name__)
mmoss@chromium.orgd0b272b2013-01-30 23:55:33 +0000289 # Make any deps_file path platform-appropriate.
John Budorick0f7b2002018-01-19 15:46:17 -0800290 if self._deps_file:
291 for sep in ['/', '\\']:
292 self._deps_file = self._deps_file.replace(sep, os.sep)
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000293
294 @property
295 def deps_file(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000296 return self._deps_file
297
298 @property
299 def managed(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000300 return self._managed
301
302 @property
303 def parent(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000304 return self._parent
305
306 @property
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000307 def root(self):
308 """Returns the root node, a GClient object."""
309 if not self.parent:
310 # This line is to signal pylint that it could be a GClient instance.
311 return self or GClient(None, None)
312 return self.parent.root
313
314 @property
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000315 def should_process(self):
316 """True if this dependency should be processed, i.e. checked out."""
317 return self._should_process
318
319 @property
320 def custom_vars(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000321 return self._custom_vars.copy()
322
323 @property
324 def custom_deps(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000325 return self._custom_deps.copy()
326
maruel@chromium.org064186c2011-09-27 23:53:33 +0000327 @property
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000328 def custom_hooks(self):
329 return self._custom_hooks[:]
330
331 @property
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200332 def raw_url(self):
333 """URL before variable expansion."""
334 return self._raw_url
335
336 @property
maruel@chromium.org064186c2011-09-27 23:53:33 +0000337 def url(self):
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200338 """URL after variable expansion."""
maruel@chromium.org064186c2011-09-27 23:53:33 +0000339 return self._url
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000340
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000341 @property
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200342 def condition(self):
343 return self._condition
344
345 @property
346 def condition_value(self):
347 return self._condition_value
348
349 @property
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +0000350 def target_os(self):
351 if self.local_target_os is not None:
352 return tuple(set(self.local_target_os).union(self.parent.target_os))
353 else:
354 return self.parent.target_os
355
Tom Andersonc31ae0b2018-02-06 14:48:56 -0800356 @property
357 def target_cpu(self):
358 return self.parent.target_cpu
359
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000360 def get_custom_deps(self, name, url):
361 """Returns a custom deps if applicable."""
362 if self.parent:
363 url = self.parent.get_custom_deps(name, url)
364 # None is a valid return value to disable a dependency.
365 return self.custom_deps.get(name, url)
366
maruel@chromium.org064186c2011-09-27 23:53:33 +0000367
368class Dependency(gclient_utils.WorkItem, DependencySettings):
maruel@chromium.org54a07a22010-06-14 19:07:39 +0000369 """Object that represents a dependency checkout."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +0000370
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200371 def __init__(self, parent, name, raw_url, url, managed, custom_deps,
agabledce6ddc2016-09-08 10:02:16 -0700372 custom_vars, custom_hooks, deps_file, should_process,
Edward Lemur231f5ea2018-01-31 19:02:36 +0100373 relative, condition, condition_value, print_outbuf=False):
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000374 gclient_utils.WorkItem.__init__(self, name)
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000375 DependencySettings.__init__(
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200376 self, parent, raw_url, url, managed, custom_deps, custom_vars,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200377 custom_hooks, deps_file, should_process, relative,
378 condition, condition_value)
maruel@chromium.org68988972011-09-20 14:11:42 +0000379
380 # This is in both .gclient and DEPS files:
maruel@chromium.org064186c2011-09-27 23:53:33 +0000381 self._deps_hooks = []
maruel@chromium.org68988972011-09-20 14:11:42 +0000382
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000383 self._pre_deps_hooks = []
384
maruel@chromium.org68988972011-09-20 14:11:42 +0000385 # Calculates properties:
maruel@chromium.org064186c2011-09-27 23:53:33 +0000386 self._parsed_url = None
maruel@chromium.org4bdd5fd2011-09-26 19:41:17 +0000387 self._dependencies = []
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200388 self._vars = {}
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +0200389 self._os_dependencies = {}
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +0200390 self._os_deps_hooks = {}
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200391
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000392 # A cache of the files affected by the current operation, necessary for
393 # hooks.
maruel@chromium.org861fd0f2010-07-23 03:05:05 +0000394 self._file_list = []
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000395 # List of host names from which dependencies are allowed.
396 # Default is an empty set, meaning unspecified in DEPS file, and hence all
397 # hosts will be allowed. Non-empty set means whitelist of hosts.
398 # allowed_hosts var is scoped to its DEPS file, and so it isn't recursive.
399 self._allowed_hosts = frozenset()
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200400 # Spec for .gni output to write (if any).
401 self._gn_args_file = None
402 self._gn_args = []
maruel@chromium.org85c2a192010-07-22 21:14:43 +0000403 # If it is not set to True, the dependency wasn't processed for its child
404 # dependency, i.e. its DEPS wasn't read.
maruel@chromium.org064186c2011-09-27 23:53:33 +0000405 self._deps_parsed = False
maruel@chromium.orgf3abb802010-08-10 17:19:56 +0000406 # This dependency has been processed, i.e. checked out
maruel@chromium.org064186c2011-09-27 23:53:33 +0000407 self._processed = False
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000408 # This dependency had its pre-DEPS hooks run
409 self._pre_deps_hooks_ran = False
maruel@chromium.orgf3abb802010-08-10 17:19:56 +0000410 # This dependency had its hook run
maruel@chromium.org064186c2011-09-27 23:53:33 +0000411 self._hooks_ran = False
kustermann@google.coma692e8f2013-04-18 08:32:04 +0000412 # This is the scm used to checkout self.url. It may be used by dependencies
413 # to get the datetime of the revision we checked out.
414 self._used_scm = None
szager@chromium.org4ad264b2014-05-20 04:43:47 +0000415 self._used_revision = None
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000416 # The actual revision we ended up getting, or None if that information is
417 # unavailable
418 self._got_revision = None
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000419
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000420 # This is a mutable value that overrides the normal recursion limit for this
421 # dependency. It is read from the actual DEPS file so cannot be set on
422 # class instantiation.
423 self.recursion_override = None
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000424 # recursedeps is a mutable value that selectively overrides the default
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000425 # 'no recursion' setting on a dep-by-dep basis. It will replace
426 # recursion_override.
iannucci@chromium.orgafa11ac2016-05-04 22:17:34 +0000427 #
428 # It will be a dictionary of {deps_name: {"deps_file": depfile_name}} or
429 # None.
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000430 self.recursedeps = None
hinoka885e5b12016-06-08 14:40:09 -0700431 # This is inherited from WorkItem. We want the URL to be a resource.
432 if url and isinstance(url, basestring):
433 # The url is usually given to gclient either as https://blah@123
qyearsley12fa6ff2016-08-24 09:18:40 -0700434 # or just https://blah. The @123 portion is irrelevant.
hinoka885e5b12016-06-08 14:40:09 -0700435 self.resources.append(url.split('@')[0])
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000436
Edward Lemur231f5ea2018-01-31 19:02:36 +0100437 # Controls whether we want to print git's output when we first clone the
438 # dependency
439 self.print_outbuf = print_outbuf
440
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000441 if not self.name and self.parent:
442 raise gclient_utils.Error('Dependency without name')
443
John Budorick0f7b2002018-01-19 15:46:17 -0800444 def ToLines(self):
445 s = []
446 condition_part = ([' "condition": %r,' % self.condition]
447 if self.condition else [])
448 s.extend([
449 ' # %s' % self.hierarchy(include_url=False),
450 ' "%s": {' % (self.name,),
451 ' "url": "%s",' % (self.raw_url,),
452 ] + condition_part + [
453 ' },',
454 '',
455 ])
456 return s
457
maruel@chromium.org470b5432011-10-11 18:18:19 +0000458 @property
459 def requirements(self):
460 """Calculate the list of requirements."""
461 requirements = set()
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000462 # self.parent is implicitly a requirement. This will be recursive by
463 # definition.
464 if self.parent and self.parent.name:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000465 requirements.add(self.parent.name)
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000466
467 # For a tree with at least 2 levels*, the leaf node needs to depend
468 # on the level higher up in an orderly way.
469 # This becomes messy for >2 depth as the DEPS file format is a dictionary,
470 # thus unsorted, while the .gclient format is a list thus sorted.
471 #
472 # * _recursion_limit is hard coded 2 and there is no hope to change this
473 # value.
474 #
475 # Interestingly enough, the following condition only works in the case we
476 # want: self is a 2nd level node. 3nd level node wouldn't need this since
477 # they already have their parent as a requirement.
maruel@chromium.org470b5432011-10-11 18:18:19 +0000478 if self.parent and self.parent.parent and not self.parent.parent.parent:
479 requirements |= set(i.name for i in self.root.dependencies if i.name)
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000480
maruel@chromium.org470b5432011-10-11 18:18:19 +0000481 if self.name:
482 requirements |= set(
483 obj.name for obj in self.root.subtree(False)
484 if (obj is not self
485 and obj.name and
486 self.name.startswith(posixpath.join(obj.name, ''))))
487 requirements = tuple(sorted(requirements))
488 logging.info('Dependency(%s).requirements = %s' % (self.name, requirements))
489 return requirements
490
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000491 @property
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000492 def try_recursedeps(self):
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000493 """Returns False if recursion_override is ever specified."""
494 if self.recursion_override is not None:
495 return False
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000496 return self.parent.try_recursedeps
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000497
498 @property
499 def recursion_limit(self):
500 """Returns > 0 if this dependency is not too recursed to be processed."""
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000501 # We continue to support the absence of recursedeps until tools and DEPS
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000502 # using recursion_override are updated.
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000503 if self.try_recursedeps and self.parent.recursedeps != None:
504 if self.name in self.parent.recursedeps:
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000505 return 1
506
507 if self.recursion_override is not None:
508 return self.recursion_override
509 return max(self.parent.recursion_limit - 1, 0)
510
maruel@chromium.org470b5432011-10-11 18:18:19 +0000511 def verify_validity(self):
512 """Verifies that this Dependency is fine to add as a child of another one.
513
514 Returns True if this entry should be added, False if it is a duplicate of
515 another entry.
516 """
517 logging.info('Dependency(%s).verify_validity()' % self.name)
518 if self.name in [s.name for s in self.parent.dependencies]:
519 raise gclient_utils.Error(
520 'The same name "%s" appears multiple times in the deps section' %
521 self.name)
522 if not self.should_process:
523 # Return early, no need to set requirements.
524 return True
525
526 # This require a full tree traversal with locks.
527 siblings = [d for d in self.root.subtree(False) if d.name == self.name]
528 for sibling in siblings:
maruel@chromium.orgb848d5b2012-10-10 23:25:50 +0000529 self_url = self.LateOverride(self.url)
530 sibling_url = sibling.LateOverride(sibling.url)
531 # Allow to have only one to be None or ''.
532 if self_url != sibling_url and bool(self_url) == bool(sibling_url):
maruel@chromium.org470b5432011-10-11 18:18:19 +0000533 raise gclient_utils.Error(
maruel@chromium.orgb848d5b2012-10-10 23:25:50 +0000534 ('Dependency %s specified more than once:\n'
535 ' %s [%s]\n'
536 'vs\n'
537 ' %s [%s]') % (
538 self.name,
539 sibling.hierarchy(),
540 sibling_url,
541 self.hierarchy(),
542 self_url))
maruel@chromium.org470b5432011-10-11 18:18:19 +0000543 # In theory we could keep it as a shadow of the other one. In
544 # practice, simply ignore it.
545 logging.warn('Won\'t process duplicate dependency %s' % sibling)
546 return False
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000547 return True
maruel@chromium.org064186c2011-09-27 23:53:33 +0000548
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000549 def LateOverride(self, url):
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200550 """Resolves the parsed url from url."""
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000551 assert self.parsed_url == None or not self.should_process, self.parsed_url
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000552 parsed_url = self.get_custom_deps(self.name, url)
553 if parsed_url != url:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000554 logging.info(
555 'Dependency(%s).LateOverride(%s) -> %s' %
556 (self.name, url, parsed_url))
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000557 return parsed_url
558
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000559 if isinstance(url, basestring):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000560 parsed_url = urlparse.urlparse(url)
scr@chromium.orgf1eccaf2014-04-11 15:51:33 +0000561 if (not parsed_url[0] and
562 not re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2])):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000563 # A relative url. Fetch the real base.
564 path = parsed_url[2]
565 if not path.startswith('/'):
566 raise gclient_utils.Error(
567 'relative DEPS entry \'%s\' must begin with a slash' % url)
568 # Create a scm just to query the full url.
569 parent_url = self.parent.parsed_url
John Budorick0f7b2002018-01-19 15:46:17 -0800570 scm = self.CreateSCM(
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000571 parent_url, self.root.root_dir, None, self.outbuf)
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000572 parsed_url = scm.FullUrlForRelativeUrl(url)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000573 else:
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000574 parsed_url = url
maruel@chromium.org470b5432011-10-11 18:18:19 +0000575 logging.info(
576 'Dependency(%s).LateOverride(%s) -> %s' %
577 (self.name, url, parsed_url))
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000578 return parsed_url
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000579
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000580 if url is None:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000581 logging.info(
582 'Dependency(%s).LateOverride(%s) -> %s' % (self.name, url, url))
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000583 return url
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000584
585 raise gclient_utils.Error('Unknown url type')
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000586
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000587 @staticmethod
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +0200588 def MergeWithOsDeps(deps, deps_os, target_os_list, process_all_deps):
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000589 """Returns a new "deps" structure that is the deps sent in updated
590 with information from deps_os (the deps_os section of the DEPS
591 file) that matches the list of target os."""
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000592 new_deps = deps.copy()
Paweł Hajdan, Jrfd0057e2017-06-21 14:20:21 +0200593 for dep_os, os_deps in deps_os.iteritems():
594 for key, value in os_deps.iteritems():
595 if value is None:
596 # Make this condition very visible, so it's not a silent failure.
597 # It's unclear how to support None override in deps_os.
598 logging.error('Ignoring %r:%r in %r deps_os', key, value, dep_os)
599 continue
600
601 # Normalize value to be a dict which contains |should_process| metadata.
602 if isinstance(value, basestring):
603 value = {'url': value}
604 assert isinstance(value, collections.Mapping), (key, value)
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +0200605 value['should_process'] = dep_os in target_os_list or process_all_deps
Paweł Hajdan, Jrfd0057e2017-06-21 14:20:21 +0200606
607 # Handle collisions/overrides.
608 if key in new_deps and new_deps[key] != value:
609 # Normalize the existing new_deps entry.
610 if isinstance(new_deps[key], basestring):
611 new_deps[key] = {'url': new_deps[key]}
612 assert isinstance(new_deps[key],
613 collections.Mapping), (key, new_deps[key])
614
615 # It's OK if the "override" sets the key to the same value.
616 # This is mostly for legacy reasons to keep existing DEPS files
617 # working. Often mac/ios and unix/android will do this.
618 if value['url'] != new_deps[key]['url']:
619 raise gclient_utils.Error(
620 ('Value from deps_os (%r; %r: %r) conflicts with existing deps '
621 'entry (%r).') % (dep_os, key, value, new_deps[key]))
622
623 # We'd otherwise overwrite |should_process| metadata, but a dep should
624 # be processed if _any_ of its references call for that.
625 value['should_process'] = (
626 value['should_process'] or
627 new_deps[key].get('should_process', True))
628
629 new_deps[key] = value
630
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000631 return new_deps
632
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200633 def _postprocess_deps(self, deps, rel_prefix):
634 """Performs post-processing of deps compared to what's in the DEPS file."""
Paweł Hajdan, Jr4426eaf2017-06-13 12:25:47 +0200635 # Make sure the dict is mutable, e.g. in case it's frozen.
636 deps = dict(deps)
637
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200638 # If a line is in custom_deps, but not in the solution, we want to append
639 # this line to the solution.
640 for d in self.custom_deps:
641 if d not in deps:
642 deps[d] = self.custom_deps[d]
Michael Moss42d02c22018-02-05 10:32:24 -0800643 # Make child deps conditional on any parent conditions. This ensures that,
644 # when flattened, recursed entries have the correct restrictions, even if
645 # not explicitly set in the recursed DEPS file. For instance, if
646 # "src/ios_foo" is conditional on "checkout_ios=True", then anything
647 # recursively included by "src/ios_foo/DEPS" should also require
648 # "checkout_ios=True".
649 if self.condition:
650 for dname, dval in deps.iteritems():
651 if isinstance(dval, basestring):
652 dval = {'url': dval}
653 deps[dname] = dval
654 else:
655 assert isinstance(dval, collections.Mapping)
656 if dval.get('condition'):
657 dval['condition'] = '(%s) and (%s)' % (
658 dval['condition'], self.condition)
659 else:
660 dval['condition'] = self.condition
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200661
662 if rel_prefix:
663 logging.warning('use_relative_paths enabled.')
664 rel_deps = {}
665 for d, url in deps.items():
666 # normpath is required to allow DEPS to use .. in their
667 # dependency local path.
668 rel_deps[os.path.normpath(os.path.join(rel_prefix, d))] = url
669 logging.warning('Updating deps by prepending %s.', rel_prefix)
670 deps = rel_deps
671
672 return deps
673
674 def _deps_to_objects(self, deps, use_relative_paths):
675 """Convert a deps dict to a dict of Dependency objects."""
676 deps_to_add = []
677 for name, dep_value in deps.iteritems():
678 should_process = self.recursion_limit and self.should_process
679 deps_file = self.deps_file
680 if self.recursedeps is not None:
681 ent = self.recursedeps.get(name)
682 if ent is not None:
683 deps_file = ent['deps_file']
684 if dep_value is None:
685 continue
John Budorick0f7b2002018-01-19 15:46:17 -0800686
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200687 condition = None
688 condition_value = True
689 if isinstance(dep_value, basestring):
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200690 raw_url = dep_value
John Budorick0f7b2002018-01-19 15:46:17 -0800691 dep_type = None
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200692 else:
693 # This should be guaranteed by schema checking in gclient_eval.
694 assert isinstance(dep_value, collections.Mapping)
John Budorick0f7b2002018-01-19 15:46:17 -0800695 raw_url = dep_value.get('url')
Paweł Hajdan, Jrfd0057e2017-06-21 14:20:21 +0200696 # Take into account should_process metadata set by MergeWithOsDeps.
697 should_process = (should_process and
698 dep_value.get('should_process', True))
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200699 condition = dep_value.get('condition')
John Budorick0f7b2002018-01-19 15:46:17 -0800700 dep_type = dep_value.get('dep_type')
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200701
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200702 if condition:
Paweł Hajdan, Jrd3790252017-07-03 21:06:24 +0200703 condition_value = gclient_eval.EvaluateCondition(
704 condition, self.get_vars())
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +0200705 if not self._get_option('process_all_deps', False):
706 should_process = should_process and condition_value
John Budorick0f7b2002018-01-19 15:46:17 -0800707
708 if dep_type == 'cipd':
John Budorickd3ba72b2018-03-20 12:27:42 -0700709 cipd_root = self.GetCipdRoot()
John Budorick0f7b2002018-01-19 15:46:17 -0800710 for package in dep_value.get('packages', []):
Shenghua Zhangf438ff72018-02-23 14:20:19 -0800711 if 'version' in package:
712 # Matches version to vars value.
713 raw_version = package['version']
714 version = raw_version.format(**self.get_vars())
715 package['version'] = version
John Budorick0f7b2002018-01-19 15:46:17 -0800716 deps_to_add.append(
717 CipdDependency(
718 self, name, package, cipd_root,
719 self.custom_vars, should_process, use_relative_paths,
720 condition, condition_value))
721 elif dep_type == 'git':
722 url = raw_url.format(**self.get_vars())
723 deps_to_add.append(
724 GitDependency(
725 self, name, raw_url, url, None, None, self.custom_vars, None,
726 deps_file, should_process, use_relative_paths, condition,
727 condition_value))
728 else:
Michael Moss012013e2018-03-30 17:03:19 -0700729 url = raw_url.format(**self.get_vars()) if raw_url else None
John Budorick0f7b2002018-01-19 15:46:17 -0800730 deps_to_add.append(
731 Dependency(
732 self, name, raw_url, url, None, None, self.custom_vars, None,
733 deps_file, should_process, use_relative_paths, condition,
734 condition_value))
735
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200736 deps_to_add.sort(key=lambda x: x.name)
737 return deps_to_add
738
Edward Lesmes6c24d372018-03-28 12:52:29 -0400739 def ParseDepsFile(self, expand_vars=True):
maruel@chromium.org271375b2010-06-23 19:17:38 +0000740 """Parses the DEPS file for this dependency."""
maruel@chromium.org3223edd2011-10-10 23:17:39 +0000741 assert not self.deps_parsed
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000742 assert not self.dependencies
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000743
744 deps_content = None
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000745
746 # First try to locate the configured deps file. If it's missing, fallback
747 # to DEPS.
748 deps_files = [self.deps_file]
749 if 'DEPS' not in deps_files:
750 deps_files.append('DEPS')
751 for deps_file in deps_files:
752 filepath = os.path.join(self.root.root_dir, self.name, deps_file)
753 if os.path.isfile(filepath):
754 logging.info(
755 'ParseDepsFile(%s): %s file found at %s', self.name, deps_file,
756 filepath)
757 break
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000758 logging.info(
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000759 'ParseDepsFile(%s): No %s file found at %s', self.name, deps_file,
760 filepath)
761
762 if os.path.isfile(filepath):
maruel@chromium.org46304292010-10-28 11:42:00 +0000763 deps_content = gclient_utils.FileRead(filepath)
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000764 logging.debug('ParseDepsFile(%s) read:\n%s', self.name, deps_content)
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000765
766 local_scope = {}
767 if deps_content:
maruel@chromium.org46304292010-10-28 11:42:00 +0000768 try:
Edward Lesmes6c24d372018-03-28 12:52:29 -0400769 vars_override = {}
770 if self.parent:
771 vars_override = self.parent.get_vars()
772 vars_override.update(self.get_vars())
773 local_scope = gclient_eval.Parse(
774 deps_content, expand_vars,
775 self._get_option('validate_syntax', False),
776 filepath, vars_override)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +0000777 except SyntaxError as e:
maruel@chromium.org46304292010-10-28 11:42:00 +0000778 gclient_utils.SyntaxErrorToError(filepath, e)
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000779
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000780 if 'allowed_hosts' in local_scope:
781 try:
782 self._allowed_hosts = frozenset(local_scope.get('allowed_hosts'))
783 except TypeError: # raised if non-iterable
784 pass
785 if not self._allowed_hosts:
786 logging.warning("allowed_hosts is specified but empty %s",
787 self._allowed_hosts)
788 raise gclient_utils.Error(
789 'ParseDepsFile(%s): allowed_hosts must be absent '
790 'or a non-empty iterable' % self.name)
791
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200792 self._gn_args_file = local_scope.get('gclient_gn_args_file')
793 self._gn_args = local_scope.get('gclient_gn_args', [])
794
Edward Lesmes0b899352018-03-19 21:59:55 +0000795 self._vars = local_scope.get('vars', {})
Paweł Hajdan, Jr1407d002017-08-01 20:01:01 +0200796 if self.parent:
797 for key, value in self.parent.get_vars().iteritems():
798 if key in self._vars:
799 self._vars[key] = value
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200800 # Since we heavily post-process things, freeze ones which should
801 # reflect original state of DEPS.
Paweł Hajdan, Jr1407d002017-08-01 20:01:01 +0200802 self._vars = gclient_utils.freeze(self._vars)
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200803
804 # If use_relative_paths is set in the DEPS file, regenerate
805 # the dictionary using paths relative to the directory containing
806 # the DEPS file. Also update recursedeps if use_relative_paths is
807 # enabled.
808 # If the deps file doesn't set use_relative_paths, but the parent did
809 # (and therefore set self.relative on this Dependency object), then we
810 # want to modify the deps and recursedeps by prepending the parent
811 # directory of this dependency.
812 use_relative_paths = local_scope.get('use_relative_paths', False)
813 rel_prefix = None
814 if use_relative_paths:
815 rel_prefix = self.name
816 elif self._relative:
817 rel_prefix = os.path.dirname(self.name)
818
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +0200819 deps = {}
820 for key, value in local_scope.get('deps', {}).iteritems():
821 deps[key.format(**self.get_vars())] = value
822
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200823 if 'recursion' in local_scope:
824 self.recursion_override = local_scope.get('recursion')
825 logging.warning(
826 'Setting %s recursion to %d.', self.name, self.recursion_limit)
827 self.recursedeps = None
828 if 'recursedeps' in local_scope:
829 self.recursedeps = {}
830 for ent in local_scope['recursedeps']:
831 if isinstance(ent, basestring):
832 self.recursedeps[ent] = {"deps_file": self.deps_file}
833 else: # (depname, depsfilename)
834 self.recursedeps[ent[0]] = {"deps_file": ent[1]}
835 logging.warning('Found recursedeps %r.', repr(self.recursedeps))
836
837 if rel_prefix:
838 logging.warning('Updating recursedeps by prepending %s.', rel_prefix)
839 rel_deps = {}
840 for depname, options in self.recursedeps.iteritems():
841 rel_deps[
842 os.path.normpath(os.path.join(rel_prefix, depname))] = options
843 self.recursedeps = rel_deps
844
845 # If present, save 'target_os' in the local_target_os property.
846 if 'target_os' in local_scope:
847 self.local_target_os = local_scope['target_os']
848 # load os specific dependencies if defined. these dependencies may
849 # override or extend the values defined by the 'deps' member.
850 target_os_list = self.target_os
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +0200851 if 'deps_os' in local_scope:
852 for dep_os, os_deps in local_scope['deps_os'].iteritems():
853 self._os_dependencies[dep_os] = self._deps_to_objects(
854 self._postprocess_deps(os_deps, rel_prefix), use_relative_paths)
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +0200855 if target_os_list and not self._get_option(
856 'do_not_merge_os_specific_entries', False):
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +0200857 deps = self.MergeWithOsDeps(
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +0200858 deps, local_scope['deps_os'], target_os_list,
859 self._get_option('process_all_deps', False))
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200860
861 deps_to_add = self._deps_to_objects(
862 self._postprocess_deps(deps, rel_prefix), use_relative_paths)
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000863
864 # override named sets of hooks by the custom hooks
865 hooks_to_run = []
866 hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks]
867 for hook in local_scope.get('hooks', []):
868 if hook.get('name', '') not in hook_names_to_suppress:
869 hooks_to_run.append(hook)
Scott Grahamc4826742017-05-11 16:59:23 -0700870 if 'hooks_os' in local_scope and target_os_list:
871 hooks_os = local_scope['hooks_os']
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +0200872
873 # Keep original contents of hooks_os for flatten.
874 for hook_os, os_hooks in hooks_os.iteritems():
875 self._os_deps_hooks[hook_os] = [
Michael Moss42d02c22018-02-05 10:32:24 -0800876 Hook.from_dict(hook, variables=self.get_vars(), verbose=True,
877 conditions=self.condition)
Paweł Hajdan, Jrd3790252017-07-03 21:06:24 +0200878 for hook in os_hooks]
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +0200879
Scott Grahamc4826742017-05-11 16:59:23 -0700880 # Specifically append these to ensure that hooks_os run after hooks.
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +0200881 if not self._get_option('do_not_merge_os_specific_entries', False):
882 for the_target_os in target_os_list:
883 the_target_os_hooks = hooks_os.get(the_target_os, [])
884 hooks_to_run.extend(the_target_os_hooks)
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000885
886 # add the replacements and any additions
887 for hook in self.custom_hooks:
888 if 'action' in hook:
889 hooks_to_run.append(hook)
890
Dirk Prankeda3a29e2017-02-27 15:29:36 -0800891 if self.recursion_limit:
Paweł Hajdan, Jr032d5452017-06-22 20:43:53 +0200892 self._pre_deps_hooks = [
Michael Moss42d02c22018-02-05 10:32:24 -0800893 Hook.from_dict(hook, variables=self.get_vars(), verbose=True,
894 conditions=self.condition)
Daniel Chenga0c5f082017-10-19 13:35:19 -0700895 for hook in local_scope.get('pre_deps_hooks', [])
896 ]
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000897
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +0200898 self.add_dependencies_and_close(deps_to_add, hooks_to_run)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000899 logging.info('ParseDepsFile(%s) done' % self.name)
900
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200901 def _get_option(self, attr, default):
902 obj = self
903 while not hasattr(obj, '_options'):
904 obj = obj.parent
905 return getattr(obj._options, attr, default)
906
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +0200907 def add_dependencies_and_close(self, deps_to_add, hooks):
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000908 """Adds the dependencies, hooks and mark the parsing as done."""
maruel@chromium.orgb9be0652011-10-14 18:05:40 +0000909 for dep in deps_to_add:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000910 if dep.verify_validity():
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000911 self.add_dependency(dep)
Daniel Chenga0c5f082017-10-19 13:35:19 -0700912 self._mark_as_parsed([
913 Hook.from_dict(
Michael Moss42d02c22018-02-05 10:32:24 -0800914 h, variables=self.get_vars(), verbose=self.root._options.verbose,
915 conditions=self.condition)
Daniel Chenga0c5f082017-10-19 13:35:19 -0700916 for h in hooks
917 ])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000918
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000919 def findDepsFromNotAllowedHosts(self):
920 """Returns a list of depenecies from not allowed hosts.
921
922 If allowed_hosts is not set, allows all hosts and returns empty list.
923 """
924 if not self._allowed_hosts:
925 return []
926 bad_deps = []
927 for dep in self._dependencies:
szager@chromium.orgbd772dd2014-11-05 18:43:08 +0000928 # Don't enforce this for custom_deps.
929 if dep.name in self._custom_deps:
930 continue
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000931 if isinstance(dep.url, basestring):
932 parsed_url = urlparse.urlparse(dep.url)
933 if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
934 bad_deps.append(dep)
935 return bad_deps
936
Edward Lesmesbb16e332018-03-30 17:54:51 -0400937 def FuzzyMatchUrl(self, parsed_url, candidates):
938 """Attempts to find this dependency in the list of candidates.
939
940 It looks first for the URL of this dependency (parsed_url) in the list of
941 candidates. If it doesn't succeed, and the URL ends in '.git', it will try
942 looking for the URL minus '.git'. Finally it will try to look for the name
943 of the dependency.
944
945 Args:
946 parsed_url: str. The parsed URL for this dependency. Something like
947 "https://example.com/src.git@revision"
948 candidates: list, dict. The list of candidates in which to look for this
949 dependency. It can contain URLs as above, or dependency names like
950 "src/some/dep".
951
952 Returns:
953 If this dependency is not found in the list of candidates, returns None.
954 Otherwise, it returns under which name did we find this dependency:
955 - Its parsed url: "https://example.com/src.git'
956 - Its parsed url minus '.git': "https://example.com/src"
957 - Its name: "src"
958 """
959 if parsed_url:
960 origin, _ = gclient_utils.SplitUrlRevision(parsed_url)
961 if origin in candidates:
962 return origin
963 if origin.endswith('.git') and origin[:-len('.git')] in candidates:
964 return origin[:-len('.git')]
965 if self.name in candidates:
966 return self.name
967 return None
968
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000969 # Arguments number differs from overridden method
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800970 # pylint: disable=arguments-differ
Edward Lesmesc621b212018-03-21 20:26:56 -0400971 def run(self, revision_overrides, command, args, work_queue, options,
972 patch_refs):
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000973 """Runs |command| then parse the DEPS file."""
maruel@chromium.org470b5432011-10-11 18:18:19 +0000974 logging.info('Dependency(%s).run()' % self.name)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +0000975 assert self._file_list == []
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000976 if not self.should_process:
977 return
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000978 # When running runhooks, there's no need to consult the SCM.
979 # All known hooks are expected to run unconditionally regardless of working
980 # copy state, so skip the SCM status check.
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +0200981 run_scm = command not in (
982 'flatten', 'runhooks', 'recurse', 'validate', None)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000983 parsed_url = self.LateOverride(self.url)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000984 file_list = [] if not options.nohooks else None
Edward Lesmesbb16e332018-03-30 17:54:51 -0400985 revision_override = revision_overrides.pop(
986 self.FuzzyMatchUrl(parsed_url, revision_overrides), None)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000987 if run_scm and parsed_url:
agabled437d762016-10-17 09:35:11 -0700988 # Create a shallow copy to mutate revision.
989 options = copy.copy(options)
990 options.revision = revision_override
991 self._used_revision = options.revision
John Budorick0f7b2002018-01-19 15:46:17 -0800992 self._used_scm = self.CreateSCM(
agabled437d762016-10-17 09:35:11 -0700993 parsed_url, self.root.root_dir, self.name, self.outbuf,
994 out_cb=work_queue.out_cb)
995 self._got_revision = self._used_scm.RunCommand(command, options, args,
996 file_list)
Edward Lesmesc621b212018-03-21 20:26:56 -0400997
998 patch_repo = parsed_url.split('@')[0]
Edward Lesmesbb16e332018-03-30 17:54:51 -0400999 patch_ref = patch_refs.pop(
1000 self.FuzzyMatchUrl(parsed_url, patch_refs), None)
Edward Lesmesc621b212018-03-21 20:26:56 -04001001 if command == 'update' and patch_ref is not None:
1002 self._used_scm.apply_patch_ref(patch_repo, patch_ref, options,
Edward Lesmesbb16e332018-03-30 17:54:51 -04001003 file_list)
Edward Lesmesc621b212018-03-21 20:26:56 -04001004
agabled437d762016-10-17 09:35:11 -07001005 if file_list:
1006 file_list = [os.path.join(self.name, f.strip()) for f in file_list]
maruel@chromium.org68988972011-09-20 14:11:42 +00001007
1008 # TODO(phajdan.jr): We should know exactly when the paths are absolute.
1009 # Convert all absolute paths to relative.
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001010 for i in range(len(file_list or [])):
maruel@chromium.org68988972011-09-20 14:11:42 +00001011 # It depends on the command being executed (like runhooks vs sync).
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001012 if not os.path.isabs(file_list[i]):
maruel@chromium.org68988972011-09-20 14:11:42 +00001013 continue
1014 prefix = os.path.commonprefix(
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001015 [self.root.root_dir.lower(), file_list[i].lower()])
1016 file_list[i] = file_list[i][len(prefix):]
maruel@chromium.org68988972011-09-20 14:11:42 +00001017 # Strip any leading path separators.
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001018 while file_list[i].startswith(('\\', '/')):
1019 file_list[i] = file_list[i][1:]
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001020
1021 # Always parse the DEPS file.
Edward Lesmes6c24d372018-03-28 12:52:29 -04001022 self.ParseDepsFile(expand_vars=(command != 'flatten'))
iannucci@chromium.org396e1a62013-07-03 19:41:04 +00001023 self._run_is_done(file_list or [], parsed_url)
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001024 if command in ('update', 'revert') and not options.noprehooks:
1025 self.RunPreDepsHooks()
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001026
1027 if self.recursion_limit:
1028 # Parse the dependencies of this dependency.
1029 for s in self.dependencies:
Paweł Hajdan, Jr4baaa112017-07-04 19:09:32 +02001030 if s.should_process:
1031 work_queue.enqueue(s)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001032
1033 if command == 'recurse':
agabled437d762016-10-17 09:35:11 -07001034 # Skip file only checkout.
John Budorick0f7b2002018-01-19 15:46:17 -08001035 scm = self.GetScmName(parsed_url)
agabled437d762016-10-17 09:35:11 -07001036 if not options.scm or scm in options.scm:
1037 cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
1038 # Pass in the SCM type as an env variable. Make sure we don't put
1039 # unicode strings in the environment.
1040 env = os.environ.copy()
1041 if scm:
1042 env['GCLIENT_SCM'] = str(scm)
1043 if parsed_url:
1044 env['GCLIENT_URL'] = str(parsed_url)
1045 env['GCLIENT_DEP_PATH'] = str(self.name)
1046 if options.prepend_dir and scm == 'git':
1047 print_stdout = False
1048 def filter_fn(line):
1049 """Git-specific path marshaling. It is optimized for git-grep."""
ilevy@chromium.org0233ac22012-11-28 20:27:02 +00001050
agabled437d762016-10-17 09:35:11 -07001051 def mod_path(git_pathspec):
1052 match = re.match('^(\\S+?:)?([^\0]+)$', git_pathspec)
1053 modified_path = os.path.join(self.name, match.group(2))
1054 branch = match.group(1) or ''
1055 return '%s%s' % (branch, modified_path)
ilevy@chromium.org0233ac22012-11-28 20:27:02 +00001056
agabled437d762016-10-17 09:35:11 -07001057 match = re.match('^Binary file ([^\0]+) matches$', line)
1058 if match:
1059 print('Binary file %s matches\n' % mod_path(match.group(1)))
1060 return
ilevy@chromium.org0233ac22012-11-28 20:27:02 +00001061
agabled437d762016-10-17 09:35:11 -07001062 items = line.split('\0')
1063 if len(items) == 2 and items[1]:
1064 print('%s : %s' % (mod_path(items[0]), items[1]))
1065 elif len(items) >= 2:
1066 # Multiple null bytes or a single trailing null byte indicate
1067 # git is likely displaying filenames only (such as with -l)
1068 print('\n'.join(mod_path(path) for path in items if path))
1069 else:
1070 print(line)
1071 else:
1072 print_stdout = True
1073 filter_fn = None
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001074
agabled437d762016-10-17 09:35:11 -07001075 if parsed_url is None:
1076 print('Skipped omitted dependency %s' % cwd, file=sys.stderr)
1077 elif os.path.isdir(cwd):
1078 try:
1079 gclient_utils.CheckCallAndFilter(
1080 args, cwd=cwd, env=env, print_stdout=print_stdout,
1081 filter_fn=filter_fn,
1082 )
1083 except subprocess2.CalledProcessError:
1084 if not options.ignore:
1085 raise
1086 else:
1087 print('Skipped missing %s' % cwd, file=sys.stderr)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001088
John Budorick0f7b2002018-01-19 15:46:17 -08001089 def GetScmName(self, url):
1090 """Get the name of the SCM for the given URL.
1091
1092 While we currently support both git and cipd as SCM implementations,
1093 this currently cannot return 'cipd', regardless of the URL, as CIPD
1094 has no canonical URL format. If you want to use CIPD as an SCM, you
1095 must currently do so by explicitly using a CipdDependency.
1096 """
1097 if not url:
1098 return None
1099 url, _ = gclient_utils.SplitUrlRevision(url)
1100 if url.endswith('.git'):
1101 return 'git'
1102 protocol = url.split('://')[0]
1103 if protocol in (
1104 'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'):
1105 return 'git'
1106 return None
1107
1108 def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
1109 out_cb=None):
1110 SCM_MAP = {
1111 'cipd': gclient_scm.CipdWrapper,
1112 'git': gclient_scm.GitWrapper,
1113 }
1114
1115 scm_name = self.GetScmName(url)
1116 if not scm_name in SCM_MAP:
1117 raise gclient_utils.Error('No SCM found for url %s' % url)
1118 scm_class = SCM_MAP[scm_name]
1119 if not scm_class.BinaryExists():
1120 raise gclient_utils.Error('%s command not found' % scm_name)
Edward Lemur231f5ea2018-01-31 19:02:36 +01001121 return scm_class(url, root_dir, relpath, out_fh, out_cb, self.print_outbuf)
John Budorick0f7b2002018-01-19 15:46:17 -08001122
Dirk Pranke9f20d022017-10-11 18:36:54 -07001123 def HasGNArgsFile(self):
1124 return self._gn_args_file is not None
1125
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +02001126 def WriteGNArgsFile(self):
1127 lines = ['# Generated from %r' % self.deps_file]
Paweł Hajdan, Jrb495bf52017-09-25 19:33:50 +02001128 variables = self.get_vars()
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +02001129 for arg in self._gn_args:
Paweł Hajdan, Jre0214742017-09-28 12:21:01 +02001130 value = variables[arg]
1131 if isinstance(value, basestring):
1132 value = gclient_eval.EvaluateCondition(value, variables)
Paweł Hajdan, Jrb495bf52017-09-25 19:33:50 +02001133 lines.append('%s = %s' % (arg, ToGNString(value)))
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +02001134 with open(os.path.join(self.root.root_dir, self._gn_args_file), 'w') as f:
1135 f.write('\n'.join(lines))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001136
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001137 @gclient_utils.lockedmethod
1138 def _run_is_done(self, file_list, parsed_url):
1139 # Both these are kept for hooks that are run as a separate tree traversal.
1140 self._file_list = file_list
1141 self._parsed_url = parsed_url
1142 self._processed = True
1143
szager@google.comb9a78d32012-03-13 18:46:21 +00001144 def GetHooks(self, options):
1145 """Evaluates all hooks, and return them in a flat list.
1146
1147 RunOnDeps() must have been called before to load the DEPS.
1148 """
1149 result = []
maruel@chromium.org68988972011-09-20 14:11:42 +00001150 if not self.should_process or not self.recursion_limit:
maruel@chromium.orgf50907b2010-08-12 17:05:48 +00001151 # Don't run the hook when it is above recursion_limit.
szager@google.comb9a78d32012-03-13 18:46:21 +00001152 return result
maruel@chromium.orgdc7445d2010-07-09 21:05:29 +00001153 # If "--force" was specified, run all hooks regardless of what files have
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001154 # changed.
maruel@chromium.orgf50907b2010-08-12 17:05:48 +00001155 if self.deps_hooks:
agabled437d762016-10-17 09:35:11 -07001156 # TODO(maruel): If the user is using git, then we don't know
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001157 # what files have changed so we always run all hooks. It'd be nice to fix
1158 # that.
1159 if (options.force or
John Budorick0f7b2002018-01-19 15:46:17 -08001160 self.GetScmName(self.parsed_url) in ('git', None) or
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001161 os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02001162 result.extend(self.deps_hooks)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001163 else:
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02001164 for hook in self.deps_hooks:
1165 if hook.matches(self.file_list_and_children):
1166 result.append(hook)
maruel@chromium.orgf50907b2010-08-12 17:05:48 +00001167 for s in self.dependencies:
szager@google.comb9a78d32012-03-13 18:46:21 +00001168 result.extend(s.GetHooks(options))
1169 return result
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001170
Dirk Pranke9f20d022017-10-11 18:36:54 -07001171 def WriteGNArgsFilesRecursively(self, dependencies):
1172 for dep in dependencies:
1173 if dep.HasGNArgsFile():
1174 dep.WriteGNArgsFile()
1175 self.WriteGNArgsFilesRecursively(dep.dependencies)
1176
Daniel Chenga0c5f082017-10-19 13:35:19 -07001177 def RunHooksRecursively(self, options, progress):
szager@google.comb9a78d32012-03-13 18:46:21 +00001178 assert self.hooks_ran == False
maruel@chromium.org064186c2011-09-27 23:53:33 +00001179 self._hooks_ran = True
Daniel Chenga0c5f082017-10-19 13:35:19 -07001180 hooks = self.GetHooks(options)
1181 if progress:
1182 progress._total = len(hooks)
1183 for hook in hooks:
Daniel Chenga0c5f082017-10-19 13:35:19 -07001184 if progress:
1185 progress.update(extra=hook.name or '')
Daniel Cheng93c5d602017-10-20 11:40:17 -07001186 hook.run(self.root.root_dir)
Daniel Chenga0c5f082017-10-19 13:35:19 -07001187 if progress:
1188 progress.end()
maruel@chromium.orgeaf61062010-07-07 18:42:39 +00001189
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001190 def RunPreDepsHooks(self):
1191 assert self.processed
1192 assert self.deps_parsed
1193 assert not self.pre_deps_hooks_ran
1194 assert not self.hooks_ran
1195 for s in self.dependencies:
1196 assert not s.processed
1197 self._pre_deps_hooks_ran = True
1198 for hook in self.pre_deps_hooks:
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02001199 hook.run(self.root.root_dir)
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001200
John Budorickd3ba72b2018-03-20 12:27:42 -07001201 def GetCipdRoot(self):
1202 if self.root is self:
1203 # Let's not infinitely recurse. If this is root and isn't an
1204 # instance of GClient, do nothing.
1205 return None
1206 return self.root.GetCipdRoot()
1207
maruel@chromium.org0d812442010-08-10 12:41:08 +00001208 def subtree(self, include_all):
maruel@chromium.orgad3287e2011-10-03 19:15:10 +00001209 """Breadth first recursion excluding root node."""
maruel@chromium.orgf13a4182011-09-22 00:26:15 +00001210 dependencies = self.dependencies
1211 for d in dependencies:
maruel@chromium.orgf50907b2010-08-12 17:05:48 +00001212 if d.should_process or include_all:
maruel@chromium.orgad3287e2011-10-03 19:15:10 +00001213 yield d
maruel@chromium.orgf13a4182011-09-22 00:26:15 +00001214 for d in dependencies:
maruel@chromium.orgad3287e2011-10-03 19:15:10 +00001215 for i in d.subtree(include_all):
1216 yield i
1217
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001218 @gclient_utils.lockedmethod
1219 def add_dependency(self, new_dep):
1220 self._dependencies.append(new_dep)
1221
1222 @gclient_utils.lockedmethod
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +02001223 def _mark_as_parsed(self, new_hooks):
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001224 self._deps_hooks.extend(new_hooks)
1225 self._deps_parsed = True
1226
maruel@chromium.org68988972011-09-20 14:11:42 +00001227 @property
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001228 @gclient_utils.lockedmethod
maruel@chromium.org4bdd5fd2011-09-26 19:41:17 +00001229 def dependencies(self):
1230 return tuple(self._dependencies)
1231
1232 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001233 @gclient_utils.lockedmethod
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001234 def os_dependencies(self):
1235 return dict(self._os_dependencies)
1236
1237 @property
1238 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001239 def deps_hooks(self):
1240 return tuple(self._deps_hooks)
1241
1242 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001243 @gclient_utils.lockedmethod
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02001244 def os_deps_hooks(self):
1245 return dict(self._os_deps_hooks)
1246
1247 @property
1248 @gclient_utils.lockedmethod
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001249 def pre_deps_hooks(self):
1250 return tuple(self._pre_deps_hooks)
1251
1252 @property
1253 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001254 def parsed_url(self):
1255 return self._parsed_url
1256
1257 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001258 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001259 def deps_parsed(self):
maruel@chromium.org3223edd2011-10-10 23:17:39 +00001260 """This is purely for debugging purposes. It's not used anywhere."""
maruel@chromium.org064186c2011-09-27 23:53:33 +00001261 return self._deps_parsed
1262
1263 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001264 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001265 def processed(self):
1266 return self._processed
1267
1268 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001269 @gclient_utils.lockedmethod
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001270 def pre_deps_hooks_ran(self):
1271 return self._pre_deps_hooks_ran
1272
1273 @property
1274 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001275 def hooks_ran(self):
1276 return self._hooks_ran
1277
1278 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001279 @gclient_utils.lockedmethod
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00001280 def allowed_hosts(self):
1281 return self._allowed_hosts
1282
1283 @property
1284 @gclient_utils.lockedmethod
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001285 def file_list(self):
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001286 return tuple(self._file_list)
1287
1288 @property
kustermann@google.coma692e8f2013-04-18 08:32:04 +00001289 def used_scm(self):
1290 """SCMWrapper instance for this dependency or None if not processed yet."""
1291 return self._used_scm
1292
1293 @property
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00001294 @gclient_utils.lockedmethod
1295 def got_revision(self):
1296 return self._got_revision
1297
1298 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001299 def file_list_and_children(self):
1300 result = list(self.file_list)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001301 for d in self.dependencies:
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001302 result.extend(d.file_list_and_children)
maruel@chromium.org68988972011-09-20 14:11:42 +00001303 return tuple(result)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001304
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001305 def __str__(self):
1306 out = []
agablea98a6cd2016-11-15 14:30:10 -08001307 for i in ('name', 'url', 'parsed_url', 'custom_deps',
maruel@chromium.org3c74bc92011-09-15 19:17:21 +00001308 'custom_vars', 'deps_hooks', 'file_list', 'should_process',
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00001309 'processed', 'hooks_ran', 'deps_parsed', 'requirements',
1310 'allowed_hosts'):
maruel@chromium.org3c74bc92011-09-15 19:17:21 +00001311 # First try the native property if it exists.
1312 if hasattr(self, '_' + i):
1313 value = getattr(self, '_' + i, False)
1314 else:
1315 value = getattr(self, i, False)
1316 if value:
1317 out.append('%s: %s' % (i, value))
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001318
1319 for d in self.dependencies:
1320 out.extend([' ' + x for x in str(d).splitlines()])
1321 out.append('')
1322 return '\n'.join(out)
1323
1324 def __repr__(self):
1325 return '%s: %s' % (self.name, self.url)
1326
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001327 def hierarchy(self, include_url=True):
maruel@chromium.orgbc2d2f92010-07-22 21:26:48 +00001328 """Returns a human-readable hierarchical reference to a Dependency."""
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001329 def format_name(d):
1330 if include_url:
1331 return '%s(%s)' % (d.name, d.url)
1332 return d.name
1333 out = format_name(self)
maruel@chromium.orgbffb9042010-07-22 20:59:36 +00001334 i = self.parent
1335 while i and i.name:
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001336 out = '%s -> %s' % (format_name(i), out)
maruel@chromium.orgbffb9042010-07-22 20:59:36 +00001337 i = i.parent
1338 return out
1339
Michael Mossfe68c912018-03-22 19:19:35 -07001340 def hierarchy_data(self):
1341 """Returns a machine-readable hierarchical reference to a Dependency."""
1342 d = self
1343 out = []
1344 while d and d.name:
1345 out.insert(0, (d.name, d.url))
1346 d = d.parent
1347 return tuple(out)
1348
Paweł Hajdan, Jrd3790252017-07-03 21:06:24 +02001349 def get_vars(self):
1350 """Returns a dictionary of effective variable values
1351 (DEPS file contents with applied custom_vars overrides)."""
Paweł Hajdan, Jra3b67ae2017-08-30 15:18:21 +02001352 # Provide some built-in variables.
1353 result = {
Paweł Hajdan, Jrd325eb32017-10-03 17:43:37 +02001354 'checkout_android': 'android' in self.target_os,
Benjamin Pastene6fe29412018-01-23 15:35:58 -08001355 'checkout_chromeos': 'chromeos' in self.target_os,
Paweł Hajdan, Jrd325eb32017-10-03 17:43:37 +02001356 'checkout_fuchsia': 'fuchsia' in self.target_os,
1357 'checkout_ios': 'ios' in self.target_os,
1358 'checkout_linux': 'unix' in self.target_os,
1359 'checkout_mac': 'mac' in self.target_os,
1360 'checkout_win': 'win' in self.target_os,
1361 'host_os': _detect_host_os(),
Tom Andersonc31ae0b2018-02-06 14:48:56 -08001362
1363 'checkout_arm': 'arm' in self.target_cpu,
1364 'checkout_arm64': 'arm64' in self.target_cpu,
1365 'checkout_x86': 'x86' in self.target_cpu,
1366 'checkout_mips': 'mips' in self.target_cpu,
1367 'checkout_ppc': 'ppc' in self.target_cpu,
1368 'checkout_s390': 's390' in self.target_cpu,
1369 'checkout_x64': 'x64' in self.target_cpu,
1370 'host_cpu': detect_host_arch.HostArch(),
Paweł Hajdan, Jra3b67ae2017-08-30 15:18:21 +02001371 }
1372 # Variables defined in DEPS file override built-in ones.
1373 result.update(self._vars)
Paweł Hajdan, Jrd3790252017-07-03 21:06:24 +02001374 result.update(self.custom_vars or {})
1375 return result
1376
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001377
Paweł Hajdan, Jra3b67ae2017-08-30 15:18:21 +02001378_PLATFORM_MAPPING = {
1379 'cygwin': 'win',
1380 'darwin': 'mac',
1381 'linux2': 'linux',
1382 'win32': 'win',
Jaideep Bajwad05f3582017-09-11 12:31:48 -04001383 'aix6': 'aix',
Paweł Hajdan, Jra3b67ae2017-08-30 15:18:21 +02001384}
1385
1386
1387def _detect_host_os():
1388 return _PLATFORM_MAPPING[sys.platform]
1389
1390
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001391class GClient(Dependency):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001392 """Object that represent a gclient checkout. A tree of Dependency(), one per
1393 solution or DEPS entry."""
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001394
1395 DEPS_OS_CHOICES = {
Jaideep Bajwad05f3582017-09-11 12:31:48 -04001396 "aix6": "unix",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001397 "win32": "win",
1398 "win": "win",
1399 "cygwin": "win",
1400 "darwin": "mac",
1401 "mac": "mac",
1402 "unix": "unix",
1403 "linux": "unix",
1404 "linux2": "unix",
maruel@chromium.org244e3442011-06-12 15:20:55 +00001405 "linux3": "unix",
szager@chromium.orgf8c95cd2012-06-01 22:26:52 +00001406 "android": "android",
Michael Mossc54fa812017-08-17 11:27:58 -07001407 "ios": "ios",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001408 }
1409
1410 DEFAULT_CLIENT_FILE_TEXT = ("""\
1411solutions = [
smutae7ea312016-07-18 11:59:41 -07001412 { "name" : "%(solution_name)s",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001413 "url" : "%(solution_url)s",
nsylvain@google.comefc80932011-05-31 21:27:56 +00001414 "deps_file" : "%(deps_file)s",
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001415 "managed" : %(managed)s,
smutae7ea312016-07-18 11:59:41 -07001416 "custom_deps" : {
1417 },
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02001418 "custom_vars": %(custom_vars)r,
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001419 },
1420]
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001421cache_dir = %(cache_dir)r
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001422""")
1423
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001424 DEFAULT_SNAPSHOT_FILE_TEXT = ("""\
1425# Snapshot generated with gclient revinfo --snapshot
Edward Lesmesc2960242018-03-06 20:50:15 -05001426solutions = %(solution_list)s
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001427""")
1428
1429 def __init__(self, root_dir, options):
maruel@chromium.org0d812442010-08-10 12:41:08 +00001430 # Do not change previous behavior. Only solution level and immediate DEPS
1431 # are processed.
1432 self._recursion_limit = 2
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +02001433 Dependency.__init__(self, None, None, None, None, True, None, None, None,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001434 'unused', True, None, None, True)
maruel@chromium.org0d425922010-06-21 19:22:24 +00001435 self._options = options
maruel@chromium.org271375b2010-06-23 19:17:38 +00001436 if options.deps_os:
1437 enforced_os = options.deps_os.split(',')
1438 else:
1439 enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')]
1440 if 'all' in enforced_os:
1441 enforced_os = self.DEPS_OS_CHOICES.itervalues()
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001442 self._enforced_os = tuple(set(enforced_os))
Tom Andersonc31ae0b2018-02-06 14:48:56 -08001443 self._enforced_cpu = detect_host_arch.HostArch(),
maruel@chromium.org271375b2010-06-23 19:17:38 +00001444 self._root_dir = root_dir
John Budorickd3ba72b2018-03-20 12:27:42 -07001445 self._cipd_root = None
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001446 self.config_content = None
1447
borenet@google.com88d10082014-03-21 17:24:48 +00001448 def _CheckConfig(self):
1449 """Verify that the config matches the state of the existing checked-out
1450 solutions."""
1451 for dep in self.dependencies:
1452 if dep.managed and dep.url:
John Budorick0f7b2002018-01-19 15:46:17 -08001453 scm = self.CreateSCM(
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001454 dep.url, self.root_dir, dep.name, self.outbuf)
smut@google.comd33eab32014-07-07 19:35:18 +00001455 actual_url = scm.GetActualRemoteURL(self._options)
borenet@google.com4e9be262014-04-08 19:40:30 +00001456 if actual_url and not scm.DoesRemoteURLMatch(self._options):
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001457 mirror = scm.GetCacheMirror()
1458 if mirror:
1459 mirror_string = '%s (exists=%s)' % (mirror.mirror_path,
1460 mirror.exists())
1461 else:
1462 mirror_string = 'not used'
borenet@google.com0a427372014-04-02 19:12:13 +00001463 raise gclient_utils.Error('''
borenet@google.com88d10082014-03-21 17:24:48 +00001464Your .gclient file seems to be broken. The requested URL is different from what
borenet@google.com0a427372014-04-02 19:12:13 +00001465is actually checked out in %(checkout_path)s.
borenet@google.com88d10082014-03-21 17:24:48 +00001466
borenet@google.com97882362014-04-07 20:06:02 +00001467The .gclient file contains:
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001468URL: %(expected_url)s (%(expected_scm)s)
1469Cache mirror: %(mirror_string)s
borenet@google.com97882362014-04-07 20:06:02 +00001470
1471The local checkout in %(checkout_path)s reports:
1472%(actual_url)s (%(actual_scm)s)
borenet@google.com88d10082014-03-21 17:24:48 +00001473
1474You should ensure that the URL listed in .gclient is correct and either change
agabled437d762016-10-17 09:35:11 -07001475it or fix the checkout.
borenet@google.com88d10082014-03-21 17:24:48 +00001476''' % {'checkout_path': os.path.join(self.root_dir, dep.name),
1477 'expected_url': dep.url,
John Budorick0f7b2002018-01-19 15:46:17 -08001478 'expected_scm': self.GetScmName(dep.url),
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001479 'mirror_string' : mirror_string,
borenet@google.com88d10082014-03-21 17:24:48 +00001480 'actual_url': actual_url,
John Budorick0f7b2002018-01-19 15:46:17 -08001481 'actual_scm': self.GetScmName(actual_url)})
borenet@google.com88d10082014-03-21 17:24:48 +00001482
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001483 def SetConfig(self, content):
maruel@chromium.orgf13a4182011-09-22 00:26:15 +00001484 assert not self.dependencies
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001485 config_dict = {}
1486 self.config_content = content
1487 try:
1488 exec(content, config_dict)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00001489 except SyntaxError as e:
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001490 gclient_utils.SyntaxErrorToError('.gclient', e)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001491
peter@chromium.org1efccc82012-04-27 16:34:38 +00001492 # Append any target OS that is not already being enforced to the tuple.
1493 target_os = config_dict.get('target_os', [])
stuartmorgan@chromium.org18a4f6a2012-11-02 13:37:24 +00001494 if config_dict.get('target_os_only', False):
1495 self._enforced_os = tuple(set(target_os))
1496 else:
1497 self._enforced_os = tuple(set(self._enforced_os).union(target_os))
1498
Tom Andersonc31ae0b2018-02-06 14:48:56 -08001499 # Append any target CPU that is not already being enforced to the tuple.
1500 target_cpu = config_dict.get('target_cpu', [])
1501 if config_dict.get('target_cpu_only', False):
1502 self._enforced_cpu = tuple(set(target_cpu))
1503 else:
1504 self._enforced_cpu = tuple(set(self._enforced_cpu).union(target_cpu))
1505
Aleksandr Derbenev9e8fb0e2017-08-01 20:18:31 +03001506 cache_dir = config_dict.get('cache_dir', self._options.cache_dir)
dyen@chromium.orgd915cca2014-08-07 21:41:37 +00001507 if cache_dir:
1508 cache_dir = os.path.join(self.root_dir, cache_dir)
1509 cache_dir = os.path.abspath(cache_dir)
Andrii Shyshkalov77ce4bd2017-11-27 12:38:18 -08001510
dyen@chromium.orgd915cca2014-08-07 21:41:37 +00001511 gclient_scm.GitWrapper.cache_dir = cache_dir
1512 git_cache.Mirror.SetCachePath(cache_dir)
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001513
stuartmorgan@chromium.org18a4f6a2012-11-02 13:37:24 +00001514 if not target_os and config_dict.get('target_os_only', False):
1515 raise gclient_utils.Error('Can\'t use target_os_only if target_os is '
1516 'not specified')
peter@chromium.org1efccc82012-04-27 16:34:38 +00001517
Tom Andersonc31ae0b2018-02-06 14:48:56 -08001518 if not target_cpu and config_dict.get('target_cpu_only', False):
1519 raise gclient_utils.Error('Can\'t use target_cpu_only if target_cpu is '
1520 'not specified')
1521
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001522 deps_to_add = []
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001523 for s in config_dict.get('solutions', []):
maruel@chromium.org81843b82010-06-28 16:49:26 +00001524 try:
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001525 deps_to_add.append(Dependency(
Paweł Hajdan, Jrfc6196b2017-07-27 13:15:25 +02001526 self, s['name'], s['url'], s['url'],
smutae7ea312016-07-18 11:59:41 -07001527 s.get('managed', True),
maruel@chromium.org81843b82010-06-28 16:49:26 +00001528 s.get('custom_deps', {}),
maruel@chromium.org0d812442010-08-10 12:41:08 +00001529 s.get('custom_vars', {}),
petermayo@chromium.orge79161a2013-07-09 14:40:37 +00001530 s.get('custom_hooks', []),
nsylvain@google.comefc80932011-05-31 21:27:56 +00001531 s.get('deps_file', 'DEPS'),
agabledce6ddc2016-09-08 10:02:16 -07001532 True,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001533 None,
1534 None,
Edward Lemur231f5ea2018-01-31 19:02:36 +01001535 True,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001536 True))
maruel@chromium.org81843b82010-06-28 16:49:26 +00001537 except KeyError:
1538 raise gclient_utils.Error('Invalid .gclient file. Solution is '
1539 'incomplete: %s' % s)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001540 self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', []))
1541 logging.info('SetConfig() done')
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001542
1543 def SaveConfig(self):
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001544 gclient_utils.FileWrite(os.path.join(self.root_dir,
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001545 self._options.config_filename),
1546 self.config_content)
1547
1548 @staticmethod
1549 def LoadCurrentConfig(options):
1550 """Searches for and loads a .gclient file relative to the current working
1551 dir. Returns a GClient object."""
szager@chromium.orge2e03202012-07-31 18:05:16 +00001552 if options.spec:
1553 client = GClient('.', options)
1554 client.SetConfig(options.spec)
1555 else:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00001556 if options.verbose:
1557 print('Looking for %s starting from %s\n' % (
1558 options.config_filename, os.getcwd()))
szager@chromium.orge2e03202012-07-31 18:05:16 +00001559 path = gclient_utils.FindGclientRoot(os.getcwd(), options.config_filename)
1560 if not path:
Michael Achenbachb3ce73d2017-10-11 16:41:27 +02001561 if options.verbose:
1562 print('Couldn\'t find configuration file.')
szager@chromium.orge2e03202012-07-31 18:05:16 +00001563 return None
1564 client = GClient(path, options)
1565 client.SetConfig(gclient_utils.FileRead(
1566 os.path.join(path, options.config_filename)))
maruel@chromium.org69392e72011-10-13 22:09:00 +00001567
1568 if (options.revisions and
1569 len(client.dependencies) > 1 and
1570 any('@' not in r for r in options.revisions)):
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001571 print(
1572 ('You must specify the full solution name like --revision %s@%s\n'
1573 'when you have multiple solutions setup in your .gclient file.\n'
1574 'Other solutions present are: %s.') % (
maruel@chromium.org69392e72011-10-13 22:09:00 +00001575 client.dependencies[0].name,
1576 options.revisions[0],
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001577 ', '.join(s.name for s in client.dependencies[1:])),
1578 file=sys.stderr)
maruel@chromium.org15804092010-09-02 17:07:37 +00001579 return client
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001580
nsylvain@google.comefc80932011-05-31 21:27:56 +00001581 def SetDefaultConfig(self, solution_name, deps_file, solution_url,
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02001582 managed=True, cache_dir=None, custom_vars=None):
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001583 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
1584 'solution_name': solution_name,
1585 'solution_url': solution_url,
nsylvain@google.comefc80932011-05-31 21:27:56 +00001586 'deps_file': deps_file,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001587 'managed': managed,
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001588 'cache_dir': cache_dir,
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02001589 'custom_vars': custom_vars or {},
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001590 })
1591
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001592 def _SaveEntries(self):
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001593 """Creates a .gclient_entries file to record the list of unique checkouts.
1594
1595 The .gclient_entries file lives in the same directory as .gclient.
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001596 """
1597 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
1598 # makes testing a bit too fun.
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001599 result = 'entries = {\n'
maruel@chromium.org68988972011-09-20 14:11:42 +00001600 for entry in self.root.subtree(False):
agabled437d762016-10-17 09:35:11 -07001601 result += ' %s: %s,\n' % (pprint.pformat(entry.name),
1602 pprint.pformat(entry.parsed_url))
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001603 result += '}\n'
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001604 file_path = os.path.join(self.root_dir, self._options.entries_filename)
maruel@chromium.org1333cb32011-10-04 23:40:16 +00001605 logging.debug(result)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001606 gclient_utils.FileWrite(file_path, result)
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001607
1608 def _ReadEntries(self):
1609 """Read the .gclient_entries file for the given client.
1610
1611 Returns:
1612 A sequence of solution names, which will be empty if there is the
1613 entries file hasn't been created yet.
1614 """
1615 scope = {}
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001616 filename = os.path.join(self.root_dir, self._options.entries_filename)
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001617 if not os.path.exists(filename):
maruel@chromium.org73e21142010-07-05 13:32:01 +00001618 return {}
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001619 try:
1620 exec(gclient_utils.FileRead(filename), scope)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00001621 except SyntaxError as e:
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001622 gclient_utils.SyntaxErrorToError(filename, e)
Aaron Gable3721ee92017-04-03 14:53:14 -07001623 return scope.get('entries', {})
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001624
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001625 def _EnforceRevisions(self):
maruel@chromium.org918a9ae2010-05-28 15:50:30 +00001626 """Checks for revision overrides."""
1627 revision_overrides = {}
smutae7ea312016-07-18 11:59:41 -07001628 if self._options.head:
1629 return revision_overrides
joi@chromium.org792ea882010-11-10 02:37:27 +00001630 if not self._options.revisions:
1631 for s in self.dependencies:
smutae7ea312016-07-18 11:59:41 -07001632 if not s.managed:
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001633 self._options.revisions.append('%s@unmanaged' % s.name)
maruel@chromium.org307d1792010-05-31 20:03:13 +00001634 if not self._options.revisions:
1635 return revision_overrides
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001636 solutions_names = [s.name for s in self.dependencies]
smutae7ea312016-07-18 11:59:41 -07001637 index = 0
1638 for revision in self._options.revisions:
1639 if not '@' in revision:
maruel@chromium.org307d1792010-05-31 20:03:13 +00001640 # Support for --revision 123
smutae7ea312016-07-18 11:59:41 -07001641 revision = '%s@%s' % (solutions_names[index], revision)
1642 name, rev = revision.split('@', 1)
szager@chromium.org4ad264b2014-05-20 04:43:47 +00001643 revision_overrides[name] = rev
smutae7ea312016-07-18 11:59:41 -07001644 index += 1
maruel@chromium.org918a9ae2010-05-28 15:50:30 +00001645 return revision_overrides
1646
Edward Lesmesc621b212018-03-21 20:26:56 -04001647 def _EnforcePatchRefs(self):
1648 """Checks for patch refs."""
1649 patch_refs = {}
1650 if not self._options.patch_refs:
1651 return patch_refs
1652 for given_patch_ref in self._options.patch_refs:
1653 patch_repo, _, patch_ref = given_patch_ref.partition('@')
1654 if not patch_repo or not patch_ref:
1655 raise gclient_utils.Error(
1656 'Wrong revision format: %s should be of the form '
1657 'patch_repo@patch_ref.' % given_patch_ref)
1658 patch_refs[patch_repo] = patch_ref
1659 return patch_refs
1660
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001661 def RunOnDeps(self, command, args, ignore_requirements=False, progress=True):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001662 """Runs a command on each dependency in a client and its dependencies.
1663
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001664 Args:
1665 command: The command to use (e.g., 'status' or 'diff')
1666 args: list of str - extra arguments to add to the command line.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001667 """
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001668 if not self.dependencies:
maruel@chromium.org73e21142010-07-05 13:32:01 +00001669 raise gclient_utils.Error('No solution specified')
borenet@google.com0a427372014-04-02 19:12:13 +00001670
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001671 revision_overrides = {}
Edward Lesmesc621b212018-03-21 20:26:56 -04001672 patch_refs = {}
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001673 # It's unnecessary to check for revision overrides for 'recurse'.
1674 # Save a few seconds by not calling _EnforceRevisions() in that case.
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02001675 if command not in ('diff', 'recurse', 'runhooks', 'status', 'revert',
1676 'validate'):
szager@chromium.org5273b8a2014-08-21 15:10:10 +00001677 self._CheckConfig()
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001678 revision_overrides = self._EnforceRevisions()
Edward Lesmesc621b212018-03-21 20:26:56 -04001679
1680 if command == 'update':
1681 patch_refs = self._EnforcePatchRefs()
Daniel Chenga21b5b32017-10-19 20:07:48 +00001682 # Disable progress for non-tty stdout.
Daniel Chenga0c5f082017-10-19 13:35:19 -07001683 should_show_progress = (
1684 setup_color.IS_TTY and not self._options.verbose and progress)
1685 pm = None
1686 if should_show_progress:
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001687 if command in ('update', 'revert'):
1688 pm = Progress('Syncing projects', 1)
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02001689 elif command in ('recurse', 'validate'):
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001690 pm = Progress(' '.join(args), 1)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001691 work_queue = gclient_utils.ExecutionQueue(
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001692 self._options.jobs, pm, ignore_requirements=ignore_requirements,
1693 verbose=self._options.verbose)
maruel@chromium.org049bced2010-08-12 13:37:20 +00001694 for s in self.dependencies:
Paweł Hajdan, Jr4baaa112017-07-04 19:09:32 +02001695 if s.should_process:
1696 work_queue.enqueue(s)
Edward Lesmesc621b212018-03-21 20:26:56 -04001697 work_queue.flush(revision_overrides, command, args, options=self._options,
1698 patch_refs=patch_refs)
1699
szager@chromium.org4ad264b2014-05-20 04:43:47 +00001700 if revision_overrides:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001701 print('Please fix your script, having invalid --revision flags will soon '
Edward Lesmesc621b212018-03-21 20:26:56 -04001702 'be considered an error.', file=sys.stderr)
1703
1704 if patch_refs:
1705 raise gclient_utils.Error(
1706 'The following --patch-ref flags were not used. Please fix it:\n%s' %
1707 ('\n'.join(
1708 patch_repo + '@' + patch_ref
1709 for patch_repo, patch_ref in patch_refs.iteritems())))
piman@chromium.org6f363722010-04-27 00:41:09 +00001710
John Budorickd3ba72b2018-03-20 12:27:42 -07001711 if self._cipd_root:
1712 self._cipd_root.run(command)
1713
Dirk Pranke9f20d022017-10-11 18:36:54 -07001714 # Once all the dependencies have been processed, it's now safe to write
1715 # out any gn_args_files and run the hooks.
1716 if command == 'update':
1717 self.WriteGNArgsFilesRecursively(self.dependencies)
1718
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001719 if not self._options.nohooks:
Daniel Chenga0c5f082017-10-19 13:35:19 -07001720 if should_show_progress:
1721 pm = Progress('Running hooks', 1)
1722 self.RunHooksRecursively(self._options, pm)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001723
1724 if command == 'update':
ajwong@chromium.orgcdcee802009-06-23 15:30:42 +00001725 # Notify the user if there is an orphaned entry in their working copy.
1726 # Only delete the directory if there are no changes in it, and
1727 # delete_unversioned_trees is set to true.
maruel@chromium.org68988972011-09-20 14:11:42 +00001728 entries = [i.name for i in self.root.subtree(False) if i.url]
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001729 full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep))
1730 for e in entries]
1731
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001732 for entry, prev_url in self._ReadEntries().iteritems():
maruel@chromium.org04dd7de2010-10-14 13:25:49 +00001733 if not prev_url:
1734 # entry must have been overridden via .gclient custom_deps
1735 continue
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001736 # Fix path separator on Windows.
1737 entry_fixed = entry.replace('/', os.path.sep)
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001738 e_dir = os.path.join(self.root_dir, entry_fixed)
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001739 # Use entry and not entry_fixed there.
jochen@chromium.orga78e5532013-03-11 13:33:03 +00001740 if (entry not in entries and
1741 (not any(path.startswith(entry + '/') for path in entries)) and
jochen@chromium.orgcc475722013-03-11 13:07:40 +00001742 os.path.exists(e_dir)):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001743 # The entry has been removed from DEPS.
John Budorick0f7b2002018-01-19 15:46:17 -08001744 scm = self.CreateSCM(
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001745 prev_url, self.root_dir, entry_fixed, self.outbuf)
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001746
1747 # Check to see if this directory is now part of a higher-up checkout.
borenet@google.com359bb642014-05-13 17:28:19 +00001748 scm_root = None
agabled437d762016-10-17 09:35:11 -07001749 try:
1750 scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(scm.checkout_path)
1751 except subprocess2.CalledProcessError:
1752 pass
1753 if not scm_root:
borenet@google.com359bb642014-05-13 17:28:19 +00001754 logging.warning('Could not find checkout root for %s. Unable to '
1755 'determine whether it is part of a higher-level '
1756 'checkout, so not removing.' % entry)
1757 continue
primiano@chromium.org1c127382015-02-17 11:15:40 +00001758
1759 # This is to handle the case of third_party/WebKit migrating from
1760 # being a DEPS entry to being part of the main project.
1761 # If the subproject is a Git project, we need to remove its .git
1762 # folder. Otherwise git operations on that folder will have different
1763 # effects depending on the current working directory.
agabled437d762016-10-17 09:35:11 -07001764 if os.path.abspath(scm_root) == os.path.abspath(e_dir):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001765 e_par_dir = os.path.join(e_dir, os.pardir)
agabled437d762016-10-17 09:35:11 -07001766 if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir):
1767 par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(e_par_dir)
primiano@chromium.org1c127382015-02-17 11:15:40 +00001768 # rel_e_dir : relative path of entry w.r.t. its parent repo.
1769 rel_e_dir = os.path.relpath(e_dir, par_scm_root)
agabled437d762016-10-17 09:35:11 -07001770 if gclient_scm.scm.GIT.IsDirectoryVersioned(
1771 par_scm_root, rel_e_dir):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001772 save_dir = scm.GetGitBackupDirPath()
1773 # Remove any eventual stale backup dir for the same project.
1774 if os.path.exists(save_dir):
1775 gclient_utils.rmtree(save_dir)
1776 os.rename(os.path.join(e_dir, '.git'), save_dir)
1777 # When switching between the two states (entry/ is a subproject
1778 # -> entry/ is part of the outer project), it is very likely
1779 # that some files are changed in the checkout, unless we are
1780 # jumping *exactly* across the commit which changed just DEPS.
1781 # In such case we want to cleanup any eventual stale files
1782 # (coming from the old subproject) in order to end up with a
1783 # clean checkout.
agabled437d762016-10-17 09:35:11 -07001784 gclient_scm.scm.GIT.CleanupDir(par_scm_root, rel_e_dir)
primiano@chromium.org1c127382015-02-17 11:15:40 +00001785 assert not os.path.exists(os.path.join(e_dir, '.git'))
1786 print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
1787 'level checkout. The git folder containing all the local'
1788 ' branches has been saved to %s.\n'
1789 'If you don\'t care about its state you can safely '
1790 'remove that folder to free up space.') %
1791 (entry, save_dir))
1792 continue
1793
borenet@google.com359bb642014-05-13 17:28:19 +00001794 if scm_root in full_entries:
primiano@chromium.org1c127382015-02-17 11:15:40 +00001795 logging.info('%s is part of a higher level checkout, not removing',
1796 scm.GetCheckoutRoot())
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001797 continue
1798
1799 file_list = []
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001800 scm.status(self._options, [], file_list)
1801 modified_files = file_list != []
maruel@chromium.org28d14bd2010-11-11 20:37:09 +00001802 if (not self._options.delete_unversioned_trees or
1803 (modified_files and not self._options.force)):
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001804 # There are modified files in this entry. Keep warning until
1805 # removed.
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001806 print(('\nWARNING: \'%s\' is no longer part of this client. '
1807 'It is recommended that you manually remove it.\n') %
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001808 entry_fixed)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001809 else:
1810 # Delete the entry
maruel@chromium.org73e21142010-07-05 13:32:01 +00001811 print('\n________ deleting \'%s\' in \'%s\'' % (
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001812 entry_fixed, self.root_dir))
digit@chromium.orgdc112ac2013-04-24 13:00:19 +00001813 gclient_utils.rmtree(e_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001814 # record the current list of entries for next time
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001815 self._SaveEntries()
maruel@chromium.org17cdf762010-05-28 17:30:52 +00001816 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001817
1818 def PrintRevInfo(self):
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001819 if not self.dependencies:
maruel@chromium.org73e21142010-07-05 13:32:01 +00001820 raise gclient_utils.Error('No solution specified')
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001821 # Load all the settings.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001822 work_queue = gclient_utils.ExecutionQueue(
1823 self._options.jobs, None, False, verbose=self._options.verbose)
maruel@chromium.org049bced2010-08-12 13:37:20 +00001824 for s in self.dependencies:
Paweł Hajdan, Jr4baaa112017-07-04 19:09:32 +02001825 if s.should_process:
1826 work_queue.enqueue(s)
Edward Lesmesc621b212018-03-21 20:26:56 -04001827 work_queue.flush({}, None, [], options=self._options, patch_refs=None)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001828
Edward Lesmesbb16e332018-03-30 17:54:51 -04001829 def ShouldPrintRevision(dep, rev):
1830 return (not self._options.filter
1831 or d.FuzzyMatchUrl(rev, self._options.filter))
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001832
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001833 def GetURLAndRev(dep):
1834 """Returns the revision-qualified SCM url for a Dependency."""
1835 if dep.parsed_url is None:
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001836 return None
agabled437d762016-10-17 09:35:11 -07001837 url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url)
John Budorick0f7b2002018-01-19 15:46:17 -08001838 scm = dep.CreateSCM(
agabled437d762016-10-17 09:35:11 -07001839 dep.parsed_url, self.root_dir, dep.name, self.outbuf)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001840 if not os.path.isdir(scm.checkout_path):
1841 return None
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001842 return '%s@%s' % (url, scm.revinfo(self._options, [], None))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001843
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001844 if self._options.snapshot:
Edward Lesmesc2960242018-03-06 20:50:15 -05001845 json_output = []
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001846 # First level at .gclient
1847 for d in self.dependencies:
1848 entries = {}
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001849 def GrabDeps(dep):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001850 """Recursively grab dependencies."""
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001851 for d in dep.dependencies:
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001852 rev = GetURLAndRev(d)
Edward Lesmesbb16e332018-03-30 17:54:51 -04001853 if ShouldPrintRevision(d, rev):
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001854 entries[d.name] = rev
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001855 GrabDeps(d)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001856 GrabDeps(d)
Edward Lesmesc2960242018-03-06 20:50:15 -05001857 json_output.append({
1858 'name': d.name,
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001859 'solution_url': d.url,
nsylvain@google.comefc80932011-05-31 21:27:56 +00001860 'deps_file': d.deps_file,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001861 'managed': d.managed,
Edward Lesmesc2960242018-03-06 20:50:15 -05001862 'custom_deps': entries,
1863 })
1864 if self._options.output_json == '-':
1865 print(json.dumps(json_output, indent=2, separators=(',', ': ')))
1866 elif self._options.output_json:
1867 with open(self._options.output_json, 'w') as f:
1868 json.dump(json_output, f)
1869 else:
1870 # Print the snapshot configuration file
1871 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {
1872 'solution_list': pprint.pformat(json_output, indent=2),
1873 })
nasser@codeaurora.orgde8f3522010-03-11 23:47:44 +00001874 else:
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00001875 entries = {}
maruel@chromium.org68988972011-09-20 14:11:42 +00001876 for d in self.root.subtree(False):
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00001877 if self._options.actual:
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001878 rev = GetURLAndRev(d)
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00001879 else:
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001880 rev = d.parsed_url
Edward Lesmesbb16e332018-03-30 17:54:51 -04001881 if ShouldPrintRevision(d, rev):
Edward Lesmesdaa76d22018-03-06 14:56:57 -05001882 entries[d.name] = rev
Edward Lesmesc2960242018-03-06 20:50:15 -05001883 if self._options.output_json:
1884 json_output = {
1885 name: {
Michael Moss012013e2018-03-30 17:03:19 -07001886 'url': rev.split('@')[0] if rev else None,
1887 'rev': rev.split('@')[1] if rev and '@' in rev else None,
Edward Lesmesc2960242018-03-06 20:50:15 -05001888 }
1889 for name, rev in entries.iteritems()
1890 }
1891 if self._options.output_json == '-':
1892 print(json.dumps(json_output, indent=2, separators=(',', ': ')))
1893 else:
1894 with open(self._options.output_json, 'w') as f:
1895 json.dump(json_output, f)
1896 else:
1897 keys = sorted(entries.keys())
1898 for x in keys:
1899 print('%s: %s' % (x, entries[x]))
maruel@chromium.orgdde32ee2010-08-10 17:44:05 +00001900 logging.info(str(self))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001901
Edward Lesmes6c24d372018-03-28 12:52:29 -04001902 def ParseDepsFile(self, expand_vars=None):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001903 """No DEPS to parse for a .gclient file."""
maruel@chromium.org049bced2010-08-12 13:37:20 +00001904 raise gclient_utils.Error('Internal error')
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001905
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00001906 def PrintLocationAndContents(self):
1907 # Print out the .gclient file. This is longer than if we just printed the
1908 # client dict, but more legible, and it might contain helpful comments.
1909 print('Loaded .gclient config in %s:\n%s' % (
1910 self.root_dir, self.config_content))
1911
John Budorickd3ba72b2018-03-20 12:27:42 -07001912 def GetCipdRoot(self):
1913 if not self._cipd_root:
1914 self._cipd_root = gclient_scm.CipdRoot(
1915 self.root_dir,
1916 # TODO(jbudorick): Support other service URLs as necessary.
1917 # Service URLs should be constant over the scope of a cipd
1918 # root, so a var per DEPS file specifying the service URL
1919 # should suffice.
1920 'https://chrome-infra-packages.appspot.com')
1921 return self._cipd_root
1922
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001923 @property
maruel@chromium.org75a59272010-06-11 22:34:03 +00001924 def root_dir(self):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001925 """Root directory of gclient checkout."""
maruel@chromium.org75a59272010-06-11 22:34:03 +00001926 return self._root_dir
1927
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001928 @property
maruel@chromium.org271375b2010-06-23 19:17:38 +00001929 def enforced_os(self):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001930 """What deps_os entries that are to be parsed."""
maruel@chromium.org271375b2010-06-23 19:17:38 +00001931 return self._enforced_os
1932
maruel@chromium.org68988972011-09-20 14:11:42 +00001933 @property
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001934 def recursion_limit(self):
1935 """How recursive can each dependencies in DEPS file can load DEPS file."""
1936 return self._recursion_limit
1937
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +00001938 @property
cmp@chromium.orgc401ad12014-07-02 23:20:08 +00001939 def try_recursedeps(self):
1940 """Whether to attempt using recursedeps-style recursion processing."""
cmp@chromium.orge84ac912014-06-30 23:14:35 +00001941 return True
1942
1943 @property
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +00001944 def target_os(self):
1945 return self._enforced_os
1946
Tom Andersonc31ae0b2018-02-06 14:48:56 -08001947 @property
1948 def target_cpu(self):
1949 return self._enforced_cpu
1950
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001951
John Budorick0f7b2002018-01-19 15:46:17 -08001952class GitDependency(Dependency):
1953 """A Dependency object that represents a single git checkout."""
1954
1955 #override
1956 def GetScmName(self, url):
1957 """Always 'git'."""
1958 del url
1959 return 'git'
1960
1961 #override
1962 def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
1963 out_cb=None):
1964 """Create a Wrapper instance suitable for handling this git dependency."""
1965 return gclient_scm.GitWrapper(url, root_dir, relpath, out_fh, out_cb)
1966
1967
1968class CipdDependency(Dependency):
1969 """A Dependency object that represents a single CIPD package."""
1970
1971 def __init__(
1972 self, parent, name, dep_value, cipd_root,
1973 custom_vars, should_process, relative, condition, condition_value):
1974 package = dep_value['package']
1975 version = dep_value['version']
1976 url = urlparse.urljoin(
1977 cipd_root.service_url, '%s@%s' % (package, version))
1978 super(CipdDependency, self).__init__(
Shenghua Zhang6f830312018-02-26 11:45:07 -08001979 parent, name + ':' + package, url, url, None, None, custom_vars,
John Budorick0f7b2002018-01-19 15:46:17 -08001980 None, None, should_process, relative, condition, condition_value)
1981 if relative:
1982 # TODO(jbudorick): Implement relative if necessary.
1983 raise gclient_utils.Error(
1984 'Relative CIPD dependencies are not currently supported.')
John Budorickd3ba72b2018-03-20 12:27:42 -07001985 self._cipd_package = None
John Budorick0f7b2002018-01-19 15:46:17 -08001986 self._cipd_root = cipd_root
John Budorick0f7b2002018-01-19 15:46:17 -08001987 self._cipd_subdir = os.path.relpath(
Shenghua Zhang6f830312018-02-26 11:45:07 -08001988 os.path.join(self.root.root_dir, name), cipd_root.root_dir)
John Budorickd3ba72b2018-03-20 12:27:42 -07001989 self._package_name = package
1990 self._package_version = version
1991
1992 #override
Edward Lesmesc621b212018-03-21 20:26:56 -04001993 def run(self, revision_overrides, command, args, work_queue, options,
1994 patch_refs):
John Budorickd3ba72b2018-03-20 12:27:42 -07001995 """Runs |command| then parse the DEPS file."""
1996 logging.info('CipdDependency(%s).run()' % self.name)
1997 if not self.should_process:
1998 return
1999 self._CreatePackageIfNecessary()
2000 super(CipdDependency, self).run(revision_overrides, command, args,
Edward Lesmesc621b212018-03-21 20:26:56 -04002001 work_queue, options, patch_refs)
John Budorickd3ba72b2018-03-20 12:27:42 -07002002
2003 def _CreatePackageIfNecessary(self):
2004 # We lazily create the CIPD package to make sure that only packages
2005 # that we want (as opposed to all packages defined in all DEPS files
2006 # we parse) get added to the root and subsequently ensured.
2007 if not self._cipd_package:
2008 self._cipd_package = self._cipd_root.add_package(
2009 self._cipd_subdir, self._package_name, self._package_version)
John Budorick0f7b2002018-01-19 15:46:17 -08002010
Edward Lesmes6c24d372018-03-28 12:52:29 -04002011 def ParseDepsFile(self, expand_vars=None):
John Budorick0f7b2002018-01-19 15:46:17 -08002012 """CIPD dependencies are not currently allowed to have nested deps."""
2013 self.add_dependencies_and_close([], [])
2014
2015 #override
Shenghua Zhang6f830312018-02-26 11:45:07 -08002016 def verify_validity(self):
2017 """CIPD dependencies allow duplicate name for packages in same directory."""
2018 logging.info('Dependency(%s).verify_validity()' % self.name)
2019 return True
2020
2021 #override
John Budorick0f7b2002018-01-19 15:46:17 -08002022 def GetScmName(self, url):
2023 """Always 'cipd'."""
2024 del url
2025 return 'cipd'
2026
2027 #override
2028 def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
2029 out_cb=None):
2030 """Create a Wrapper instance suitable for handling this CIPD dependency."""
John Budorickd3ba72b2018-03-20 12:27:42 -07002031 self._CreatePackageIfNecessary()
John Budorick0f7b2002018-01-19 15:46:17 -08002032 return gclient_scm.CipdWrapper(
2033 url, root_dir, relpath, out_fh, out_cb,
2034 root=self._cipd_root,
2035 package=self._cipd_package)
2036
2037 def ToLines(self):
2038 """Return a list of lines representing this in a DEPS file."""
2039 s = []
John Budorickd3ba72b2018-03-20 12:27:42 -07002040 self._CreatePackageIfNecessary()
John Budorick0f7b2002018-01-19 15:46:17 -08002041 if self._cipd_package.authority_for_subdir:
2042 condition_part = ([' "condition": %r,' % self.condition]
2043 if self.condition else [])
2044 s.extend([
2045 ' # %s' % self.hierarchy(include_url=False),
John Budorickd3ba72b2018-03-20 12:27:42 -07002046 ' "%s": {' % (self.name.split(':')[0],),
John Budorick0f7b2002018-01-19 15:46:17 -08002047 ' "packages": [',
2048 ])
2049 for p in self._cipd_root.packages(self._cipd_subdir):
2050 s.extend([
John Budorick64e33cb2018-02-20 09:40:30 -08002051 ' {',
2052 ' "package": "%s",' % p.name,
2053 ' "version": "%s",' % p.version,
2054 ' },',
John Budorick0f7b2002018-01-19 15:46:17 -08002055 ])
John Budorickd3ba72b2018-03-20 12:27:42 -07002056
John Budorick0f7b2002018-01-19 15:46:17 -08002057 s.extend([
2058 ' ],',
2059 ' "dep_type": "cipd",',
2060 ] + condition_part + [
2061 ' },',
2062 '',
2063 ])
2064 return s
2065
2066
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002067#### gclient commands.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002068
2069
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002070@subcommand.usage('[command] [args ...]')
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002071def CMDrecurse(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002072 """Operates [command args ...] on all the dependencies.
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002073
2074 Runs a shell command on all entries.
qyearsley12fa6ff2016-08-24 09:18:40 -07002075 Sets GCLIENT_DEP_PATH environment variable as the dep's relative location to
ilevy@chromium.org37116242012-11-28 01:32:48 +00002076 root directory of the checkout.
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002077 """
2078 # Stop parsing at the first non-arg so that these go through to the command
2079 parser.disable_interspersed_args()
2080 parser.add_option('-s', '--scm', action='append', default=[],
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002081 help='Choose scm types to operate upon.')
maruel@chromium.org288054d2012-03-05 00:43:07 +00002082 parser.add_option('-i', '--ignore', action='store_true',
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002083 help='Ignore non-zero return codes from subcommands.')
2084 parser.add_option('--prepend-dir', action='store_true',
2085 help='Prepend relative dir for use with git <cmd> --null.')
2086 parser.add_option('--no-progress', action='store_true',
2087 help='Disable progress bar that shows sub-command updates')
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002088 options, args = parser.parse_args(args)
maruel@chromium.org45e9f2d2010-10-18 13:33:46 +00002089 if not args:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002090 print('Need to supply a command!', file=sys.stderr)
maruel@chromium.org45e9f2d2010-10-18 13:33:46 +00002091 return 1
maruel@chromium.org78cba522010-10-18 13:32:05 +00002092 root_and_entries = gclient_utils.GetGClientRootAndEntries()
2093 if not root_and_entries:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002094 print(
maruel@chromium.org78cba522010-10-18 13:32:05 +00002095 'You need to run gclient sync at least once to use \'recurse\'.\n'
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002096 'This is because .gclient_entries needs to exist and be up to date.',
2097 file=sys.stderr)
maruel@chromium.org78cba522010-10-18 13:32:05 +00002098 return 1
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00002099
2100 # Normalize options.scm to a set()
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002101 scm_set = set()
2102 for scm in options.scm:
2103 scm_set.update(scm.split(','))
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00002104 options.scm = scm_set
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002105
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00002106 options.nohooks = True
2107 client = GClient.LoadCurrentConfig(options)
Marc-Antoine Ruele6e06412017-10-18 13:47:02 -04002108 if not client:
2109 raise gclient_utils.Error('client not configured; see \'gclient config\'')
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002110 return client.RunOnDeps('recurse', args, ignore_requirements=True,
2111 progress=not options.no_progress)
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00002112
2113
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002114@subcommand.usage('[args ...]')
davidbarr@chromium.org12f944e2012-03-01 02:18:31 +00002115def CMDfetch(parser, args):
2116 """Fetches upstream commits for all modules.
2117
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002118 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
2119 """
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00002120 (options, args) = parser.parse_args(args)
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002121 return CMDrecurse(OptionParser(), [
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002122 '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args)
2123
2124
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002125class Flattener(object):
2126 """Flattens a gclient solution."""
2127
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002128 def __init__(self, client, pin_all_deps=False):
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002129 """Constructor.
2130
2131 Arguments:
2132 client (GClient): client to flatten
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002133 pin_all_deps (bool): whether to pin all deps, even if they're not pinned
2134 in DEPS
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002135 """
2136 self._client = client
2137
2138 self._deps_string = None
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002139 self._deps_files = set()
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002140
2141 self._allowed_hosts = set()
2142 self._deps = {}
2143 self._deps_os = {}
2144 self._hooks = []
2145 self._hooks_os = {}
2146 self._pre_deps_hooks = []
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002147 self._vars = {}
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002148
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002149 self._flatten(pin_all_deps=pin_all_deps)
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002150
2151 @property
2152 def deps_string(self):
2153 assert self._deps_string is not None
2154 return self._deps_string
2155
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002156 @property
2157 def deps_files(self):
2158 return self._deps_files
2159
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002160 def _pin_dep(self, dep):
2161 """Pins a dependency to specific full revision sha.
2162
2163 Arguments:
2164 dep (Dependency): dependency to process
2165 """
2166 if dep.parsed_url is None:
2167 return
2168
2169 # Make sure the revision is always fully specified (a hash),
2170 # as opposed to refs or tags which might change. Similarly,
2171 # shortened shas might become ambiguous; make sure to always
2172 # use full one for pinning.
2173 url, revision = gclient_utils.SplitUrlRevision(dep.parsed_url)
2174 if revision and gclient_utils.IsFullGitSha(revision):
2175 return
2176
John Budorick0f7b2002018-01-19 15:46:17 -08002177 scm = dep.CreateSCM(
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002178 dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf)
2179 revinfo = scm.revinfo(self._client._options, [], None)
2180
2181 dep._parsed_url = dep._url = '%s@%s' % (url, revinfo)
2182 raw_url, _ = gclient_utils.SplitUrlRevision(dep._raw_url)
2183 dep._raw_url = '%s@%s' % (raw_url, revinfo)
2184
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002185 def _flatten(self, pin_all_deps=False):
2186 """Runs the flattener. Saves resulting DEPS string.
2187
2188 Arguments:
2189 pin_all_deps (bool): whether to pin all deps, even if they're not pinned
2190 in DEPS
2191 """
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002192 for solution in self._client.dependencies:
Paweł Hajdan, Jr11eb7152017-08-10 12:50:11 +02002193 self._add_dep(solution)
Paweł Hajdan, Jrb0ad16e2017-08-03 15:33:21 +02002194 self._flatten_dep(solution)
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002195
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002196 if pin_all_deps:
2197 for dep in self._deps.itervalues():
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002198 self._pin_dep(dep)
Paweł Hajdan, Jr39300ba2017-08-11 16:52:38 +02002199
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002200 for os_deps in self._deps_os.itervalues():
2201 for dep in os_deps.itervalues():
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002202 self._pin_dep(dep)
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002203
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002204 def add_deps_file(dep):
Paweł Hajdan, Jr0870df22017-08-23 17:59:29 +02002205 # Only include DEPS files referenced by recursedeps.
2206 if not (dep.parent is None or
2207 (dep.name in (dep.parent.recursedeps or {}))):
2208 return
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002209 deps_file = dep.deps_file
2210 deps_path = os.path.join(self._client.root_dir, dep.name, deps_file)
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002211 if not os.path.exists(deps_path):
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002212 # gclient has a fallback that if deps_file doesn't exist, it'll try
2213 # DEPS. Do the same here.
2214 deps_file = 'DEPS'
2215 deps_path = os.path.join(self._client.root_dir, dep.name, deps_file)
2216 if not os.path.exists(deps_path):
2217 return
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002218 assert dep.parsed_url
Michael Mossfe68c912018-03-22 19:19:35 -07002219 self._deps_files.add((dep.parsed_url, deps_file, dep.hierarchy_data()))
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002220 for dep in self._deps.itervalues():
2221 add_deps_file(dep)
2222 for os_deps in self._deps_os.itervalues():
2223 for dep in os_deps.itervalues():
2224 add_deps_file(dep)
2225
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002226 self._deps_string = '\n'.join(
2227 _GNSettingsToLines(
2228 self._client.dependencies[0]._gn_args_file,
2229 self._client.dependencies[0]._gn_args) +
2230 _AllowedHostsToLines(self._allowed_hosts) +
2231 _DepsToLines(self._deps) +
2232 _DepsOsToLines(self._deps_os) +
2233 _HooksToLines('hooks', self._hooks) +
2234 _HooksToLines('pre_deps_hooks', self._pre_deps_hooks) +
2235 _HooksOsToLines(self._hooks_os) +
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002236 _VarsToLines(self._vars) +
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002237 ['# %s, %s' % (url, deps_file)
Michael Mossfe68c912018-03-22 19:19:35 -07002238 for url, deps_file, _ in sorted(self._deps_files)] +
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002239 ['']) # Ensure newline at end of file.
2240
Paweł Hajdan, Jr11eb7152017-08-10 12:50:11 +02002241 def _add_dep(self, dep):
2242 """Helper to add a dependency to flattened DEPS.
2243
2244 Arguments:
2245 dep (Dependency): dependency to add
2246 """
2247 assert dep.name not in self._deps or self._deps.get(dep.name) == dep, (
2248 dep.name, self._deps.get(dep.name))
Paweł Hajdan, Jr9a289022017-08-10 16:04:24 +02002249 if dep.url:
2250 self._deps[dep.name] = dep
Paweł Hajdan, Jr11eb7152017-08-10 12:50:11 +02002251
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002252 def _add_os_dep(self, os_dep, dep_os):
2253 """Helper to add an OS-specific dependency to flattened DEPS.
2254
2255 Arguments:
2256 os_dep (Dependency): dependency to add
2257 dep_os (str): name of the OS
2258 """
2259 assert (
2260 os_dep.name not in self._deps_os.get(dep_os, {}) or
2261 self._deps_os.get(dep_os, {}).get(os_dep.name) == os_dep), (
2262 os_dep.name, self._deps_os.get(dep_os, {}).get(os_dep.name))
2263 if os_dep.url:
Paweł Hajdan, Jr76a9d042017-08-18 20:05:41 +02002264 # OS-specific deps need to have their full URL resolved manually.
2265 assert not os_dep.parsed_url, (os_dep, os_dep.parsed_url)
2266 os_dep._parsed_url = os_dep.LateOverride(os_dep.url)
2267
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002268 self._deps_os.setdefault(dep_os, {})[os_dep.name] = os_dep
2269
2270 def _flatten_dep(self, dep, dep_os=None):
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002271 """Visits a dependency in order to flatten it (see CMDflatten).
2272
2273 Arguments:
2274 dep (Dependency): dependency to process
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002275 dep_os (str or None): name of the OS |dep| is specific to
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002276 """
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002277 logging.debug('_flatten_dep(%s, %s)', dep.name, dep_os)
2278
Paweł Hajdan, Jrc69b32e2017-08-17 18:47:48 +02002279 if not dep.deps_parsed:
2280 dep.ParseDepsFile()
2281
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002282 self._allowed_hosts.update(dep.allowed_hosts)
2283
Michael Mossce9f17f2018-01-31 13:16:35 -08002284 # Only include vars explicitly listed in the DEPS files or gclient solution,
2285 # not automatic, local overrides (i.e. not all of dep.get_vars()).
2286 hierarchy = dep.hierarchy(include_url=False)
Paweł Hajdan, Jra3b67ae2017-08-30 15:18:21 +02002287 for key, value in dep._vars.iteritems():
Paweł Hajdan, Jrc9353602017-08-02 17:52:08 +02002288 # Make sure there are no conflicting variables. It is fine however
2289 # to use same variable name, as long as the value is consistent.
2290 assert key not in self._vars or self._vars[key][1] == value
Michael Mossce9f17f2018-01-31 13:16:35 -08002291 self._vars[key] = (hierarchy, value)
2292 # Override explicit custom variables.
2293 for key, value in dep.custom_vars.iteritems():
2294 # Do custom_vars that don't correspond to DEPS vars ever make sense? DEPS
2295 # conditionals shouldn't be using vars that aren't also defined in the
2296 # DEPS (presubmit actually disallows this), so any new custom_var must be
2297 # unused in the DEPS, so no need to add it to the flattened output either.
2298 if key not in self._vars:
2299 continue
2300 # Don't "override" existing vars if it's actually the same value.
2301 elif self._vars[key][1] == value:
2302 continue
2303 # Anything else is overriding a default value from the DEPS.
2304 self._vars[key] = (hierarchy + ' [custom_var override]', value)
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002305
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002306 self._pre_deps_hooks.extend([(dep, hook) for hook in dep.pre_deps_hooks])
2307
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002308 if dep_os:
2309 if dep.deps_hooks:
2310 self._hooks_os.setdefault(dep_os, []).extend(
2311 [(dep, hook) for hook in dep.deps_hooks])
2312 else:
2313 self._hooks.extend([(dep, hook) for hook in dep.deps_hooks])
2314
Paweł Hajdan, Jrb0ad16e2017-08-03 15:33:21 +02002315 for sub_dep in dep.dependencies:
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002316 if dep_os:
2317 self._add_os_dep(sub_dep, dep_os)
2318 else:
2319 self._add_dep(sub_dep)
Paweł Hajdan, Jrb0ad16e2017-08-03 15:33:21 +02002320
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002321 for hook_os, os_hooks in dep.os_deps_hooks.iteritems():
2322 self._hooks_os.setdefault(hook_os, []).extend(
2323 [(dep, hook) for hook in os_hooks])
2324
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002325 for sub_dep_os, os_deps in dep.os_dependencies.iteritems():
Paweł Hajdan, Jre2deb1e2017-08-09 17:29:21 +02002326 for os_dep in os_deps:
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002327 self._add_os_dep(os_dep, sub_dep_os)
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002328
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002329 # Process recursedeps. |deps_by_name| is a map where keys are dependency
2330 # names, and values are maps of OS names to |Dependency| instances.
2331 # |None| in place of OS name means the dependency is not OS-specific.
2332 deps_by_name = dict((d.name, {None: d}) for d in dep.dependencies)
2333 for sub_dep_os, os_deps in dep.os_dependencies.iteritems():
Paweł Hajdan, Jrc9353602017-08-02 17:52:08 +02002334 for os_dep in os_deps:
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002335 assert sub_dep_os not in deps_by_name.get(os_dep.name, {}), (
2336 os_dep.name, sub_dep_os)
2337 deps_by_name.setdefault(os_dep.name, {})[sub_dep_os] = os_dep
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002338 for recurse_dep_name in (dep.recursedeps or []):
Paweł Hajdan, Jradae2a62017-08-18 16:49:57 +02002339 dep_info = deps_by_name[recurse_dep_name]
2340 for sub_dep_os, os_dep in dep_info.iteritems():
2341 self._flatten_dep(os_dep, dep_os=(sub_dep_os or dep_os))
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002342
2343
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002344def CMDflatten(parser, args):
2345 """Flattens the solutions into a single DEPS file."""
2346 parser.add_option('--output-deps', help='Path to the output DEPS file')
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002347 parser.add_option(
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002348 '--output-deps-files',
2349 help=('Path to the output metadata about DEPS files referenced by '
2350 'recursedeps.'))
2351 parser.add_option(
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002352 '--pin-all-deps', action='store_true',
2353 help=('Pin all deps, even if not pinned in DEPS. CAVEAT: only does so '
2354 'for checked out deps, NOT deps_os.'))
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002355 options, args = parser.parse_args(args)
2356
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +02002357 options.do_not_merge_os_specific_entries = True
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002358 options.nohooks = True
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002359 options.process_all_deps = True
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002360 client = GClient.LoadCurrentConfig(options)
2361
2362 # Only print progress if we're writing to a file. Otherwise, progress updates
2363 # could obscure intended output.
2364 code = client.RunOnDeps('flatten', args, progress=options.output_deps)
2365 if code != 0:
2366 return code
2367
Paweł Hajdan, Jr271a1682017-07-06 20:54:30 +02002368 flattener = Flattener(client, pin_all_deps=options.pin_all_deps)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002369
2370 if options.output_deps:
2371 with open(options.output_deps, 'w') as f:
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002372 f.write(flattener.deps_string)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002373 else:
Paweł Hajdan, Jraaf93f42017-07-06 17:37:46 +02002374 print(flattener.deps_string)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002375
Michael Mossfe68c912018-03-22 19:19:35 -07002376 deps_files = [{'url': d[0], 'deps_file': d[1], 'hierarchy': d[2]}
Paweł Hajdan, Jrfeb01642017-09-12 15:50:46 +02002377 for d in sorted(flattener.deps_files)]
2378 if options.output_deps_files:
2379 with open(options.output_deps_files, 'w') as f:
2380 json.dump(deps_files, f)
2381
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002382 return 0
2383
2384
Paweł Hajdan, Jr3c2aa832017-06-07 20:22:16 +02002385def _GNSettingsToLines(gn_args_file, gn_args):
2386 s = []
2387 if gn_args_file:
2388 s.extend([
2389 'gclient_gn_args_file = "%s"' % gn_args_file,
2390 'gclient_gn_args = %r' % gn_args,
2391 ])
2392 return s
2393
2394
Paweł Hajdan, Jr6014b562017-06-30 17:43:42 +02002395def _AllowedHostsToLines(allowed_hosts):
2396 """Converts |allowed_hosts| set to list of lines for output."""
2397 if not allowed_hosts:
2398 return []
2399 s = ['allowed_hosts = [']
2400 for h in sorted(allowed_hosts):
2401 s.append(' "%s",' % h)
2402 s.extend([']', ''])
2403 return s
2404
2405
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002406def _DepsToLines(deps):
2407 """Converts |deps| dict to list of lines for output."""
Paweł Hajdan, Jr5b593352017-06-29 18:37:45 +02002408 if not deps:
2409 return []
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002410 s = ['deps = {']
John Budorick0f7b2002018-01-19 15:46:17 -08002411 for _, dep in sorted(deps.iteritems()):
2412 s.extend(dep.ToLines())
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002413 s.extend(['}', ''])
2414 return s
2415
2416
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02002417def _DepsOsToLines(deps_os):
2418 """Converts |deps_os| dict to list of lines for output."""
Paweł Hajdan, Jr5b593352017-06-29 18:37:45 +02002419 if not deps_os:
2420 return []
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02002421 s = ['deps_os = {']
2422 for dep_os, os_deps in sorted(deps_os.iteritems()):
2423 s.append(' "%s": {' % dep_os)
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +02002424 for name, dep in sorted(os_deps.iteritems()):
Paweł Hajdan, Jr78ce24e2017-10-03 17:09:13 +02002425 condition_part = ([' "condition": %r,' % dep.condition]
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +02002426 if dep.condition else [])
2427 s.extend([
2428 ' # %s' % dep.hierarchy(include_url=False),
2429 ' "%s": {' % (name,),
Paweł Hajdan, Jrde86ab32017-08-10 13:55:16 +02002430 ' "url": "%s",' % (dep.raw_url,),
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +02002431 ] + condition_part + [
2432 ' },',
2433 '',
2434 ])
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02002435 s.extend([' },', ''])
2436 s.extend(['}', ''])
2437 return s
2438
2439
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002440def _HooksToLines(name, hooks):
2441 """Converts |hooks| list to list of lines for output."""
Paweł Hajdan, Jr5b593352017-06-29 18:37:45 +02002442 if not hooks:
2443 return []
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002444 s = ['%s = [' % name]
2445 for dep, hook in hooks:
2446 s.extend([
2447 ' # %s' % dep.hierarchy(include_url=False),
2448 ' {',
2449 ])
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02002450 if hook.name is not None:
2451 s.append(' "name": "%s",' % hook.name)
2452 if hook.pattern is not None:
2453 s.append(' "pattern": "%s",' % hook.pattern)
Paweł Hajdan, Jrecf53fe2017-09-29 18:28:49 +02002454 if hook.condition is not None:
Paweł Hajdan, Jr78ce24e2017-10-03 17:09:13 +02002455 s.append(' "condition": %r,' % hook.condition)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002456 s.extend(
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +02002457 # Hooks run in the parent directory of their dep.
Paweł Hajdan, Jrad30de62017-06-26 18:51:58 +02002458 [' "cwd": "%s",' % os.path.normpath(os.path.dirname(dep.name))] +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002459 [' "action": ['] +
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02002460 [' "%s",' % arg for arg in hook.action] +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02002461 [' ]', ' },', '']
2462 )
2463 s.extend([']', ''])
2464 return s
2465
2466
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02002467def _HooksOsToLines(hooks_os):
2468 """Converts |hooks| list to list of lines for output."""
Paweł Hajdan, Jr5b593352017-06-29 18:37:45 +02002469 if not hooks_os:
2470 return []
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02002471 s = ['hooks_os = {']
2472 for hook_os, os_hooks in hooks_os.iteritems():
Michael Moss017bcf62017-06-28 15:26:38 -07002473 s.append(' "%s": [' % hook_os)
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02002474 for dep, hook in os_hooks:
2475 s.extend([
2476 ' # %s' % dep.hierarchy(include_url=False),
2477 ' {',
2478 ])
2479 if hook.name is not None:
2480 s.append(' "name": "%s",' % hook.name)
2481 if hook.pattern is not None:
2482 s.append(' "pattern": "%s",' % hook.pattern)
Paweł Hajdan, Jrecf53fe2017-09-29 18:28:49 +02002483 if hook.condition is not None:
Paweł Hajdan, Jr78ce24e2017-10-03 17:09:13 +02002484 s.append(' "condition": %r,' % hook.condition)
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02002485 s.extend(
2486 # Hooks run in the parent directory of their dep.
2487 [' "cwd": "%s",' % os.path.normpath(os.path.dirname(dep.name))] +
2488 [' "action": ['] +
2489 [' "%s",' % arg for arg in hook.action] +
2490 [' ]', ' },', '']
2491 )
Michael Moss017bcf62017-06-28 15:26:38 -07002492 s.extend([' ],', ''])
Paweł Hajdan, Jr96e1d782017-06-27 11:12:25 +02002493 s.extend(['}', ''])
2494 return s
2495
2496
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002497def _VarsToLines(variables):
2498 """Converts |variables| dict to list of lines for output."""
2499 if not variables:
2500 return []
2501 s = ['vars = {']
2502 for key, tup in sorted(variables.iteritems()):
Michael Mossce9f17f2018-01-31 13:16:35 -08002503 hierarchy, value = tup
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002504 s.extend([
Michael Mossce9f17f2018-01-31 13:16:35 -08002505 ' # %s' % hierarchy,
Paweł Hajdan, Jrfb022012017-07-06 18:00:08 +02002506 ' "%s": %r,' % (key, value),
2507 '',
2508 ])
2509 s.extend(['}', ''])
2510 return s
2511
2512
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002513def CMDgrep(parser, args):
2514 """Greps through git repos managed by gclient.
2515
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002516 Runs 'git grep [args...]' for each module.
2517 """
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002518 # We can't use optparse because it will try to parse arguments sent
2519 # to git grep and throw an error. :-(
2520 if not args or re.match('(-h|--help)$', args[0]):
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002521 print(
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002522 'Usage: gclient grep [-j <N>] git-grep-args...\n\n'
2523 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep '
2524 '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to '
2525 '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the'
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002526 ' end of your query.',
2527 file=sys.stderr)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00002528 return 1
2529
2530 jobs_arg = ['--jobs=1']
2531 if re.match(r'(-j|--jobs=)\d+$', args[0]):
2532 jobs_arg, args = args[:1], args[1:]
2533 elif re.match(r'(-j|--jobs)$', args[0]):
2534 jobs_arg, args = args[:2], args[2:]
2535
2536 return CMDrecurse(
2537 parser,
2538 jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git',
2539 'git', 'grep', '--null', '--color=Always'] + args)
davidbarr@chromium.org12f944e2012-03-01 02:18:31 +00002540
2541
stip@chromium.orga735da22015-04-29 23:18:20 +00002542def CMDroot(parser, args):
2543 """Outputs the solution root (or current dir if there isn't one)."""
2544 (options, args) = parser.parse_args(args)
2545 client = GClient.LoadCurrentConfig(options)
2546 if client:
2547 print(os.path.abspath(client.root_dir))
2548 else:
2549 print(os.path.abspath('.'))
2550
2551
agablea98a6cd2016-11-15 14:30:10 -08002552@subcommand.usage('[url]')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002553def CMDconfig(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002554 """Creates a .gclient file in the current directory.
maruel@chromium.orgddff62d2010-05-17 21:02:36 +00002555
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002556 This specifies the configuration for further commands. After update/sync,
2557 top-level DEPS files in each module are read to determine dependent
2558 modules to operate on as well. If optional [url] parameter is
2559 provided, then configuration is read from a specified Subversion server
2560 URL.
2561 """
szager@chromium.orge2e03202012-07-31 18:05:16 +00002562 # We do a little dance with the --gclientfile option. 'gclient config' is the
2563 # only command where it's acceptable to have both '--gclientfile' and '--spec'
2564 # arguments. So, we temporarily stash any --gclientfile parameter into
2565 # options.output_config_file until after the (gclientfile xor spec) error
2566 # check.
2567 parser.remove_option('--gclientfile')
2568 parser.add_option('--gclientfile', dest='output_config_file',
2569 help='Specify an alternate .gclient file')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002570 parser.add_option('--name',
2571 help='overrides the default name for the solution')
nsylvain@google.comefc80932011-05-31 21:27:56 +00002572 parser.add_option('--deps-file', default='DEPS',
David Benjamin105e11e2017-10-16 10:39:35 -04002573 help='overrides the default name for the DEPS file for the '
nsylvain@google.comefc80932011-05-31 21:27:56 +00002574 'main solutions and all sub-dependencies')
smutae7ea312016-07-18 11:59:41 -07002575 parser.add_option('--unmanaged', action='store_true', default=False,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00002576 help='overrides the default behavior to make it possible '
smutae7ea312016-07-18 11:59:41 -07002577 'to have the main solution untouched by gclient '
2578 '(gclient will check out unmanaged dependencies but '
2579 'will never sync them)')
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02002580 parser.add_option('--custom-var', action='append', dest='custom_vars',
2581 default=[],
2582 help='overrides variables; key=value syntax')
szager@chromium.orge2e03202012-07-31 18:05:16 +00002583 parser.set_defaults(config_filename=None)
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002584 (options, args) = parser.parse_args(args)
szager@chromium.orge2e03202012-07-31 18:05:16 +00002585 if options.output_config_file:
2586 setattr(options, 'config_filename', getattr(options, 'output_config_file'))
maruel@chromium.org5fc2a332010-05-26 19:37:15 +00002587 if ((options.spec and args) or len(args) > 2 or
2588 (not options.spec and not args)):
2589 parser.error('Inconsistent arguments. Use either --spec or one or 2 args')
2590
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02002591 custom_vars = {}
2592 for arg in options.custom_vars:
2593 kv = arg.split('=', 1)
2594 if len(kv) != 2:
2595 parser.error('Invalid --custom-var argument: %r' % arg)
2596 custom_vars[kv[0]] = gclient_eval.EvaluateCondition(kv[1], {})
2597
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002598 client = GClient('.', options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002599 if options.spec:
2600 client.SetConfig(options.spec)
2601 else:
maruel@chromium.org1ab7ffc2009-06-03 17:21:37 +00002602 base_url = args[0].rstrip('/')
iposva@chromium.org8cf7a392010-04-07 17:20:26 +00002603 if not options.name:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002604 name = base_url.split('/')[-1]
nsylvain@google.com12649ef2011-06-01 17:11:20 +00002605 if name.endswith('.git'):
2606 name = name[:-4]
iposva@chromium.org8cf7a392010-04-07 17:20:26 +00002607 else:
2608 # specify an alternate relpath for the given URL.
2609 name = options.name
agable@chromium.orgf2214672015-10-27 21:02:48 +00002610 if not os.path.abspath(os.path.join(os.getcwd(), name)).startswith(
2611 os.getcwd()):
2612 parser.error('Do not pass a relative path for --name.')
2613 if any(x in ('..', '.', '/', '\\') for x in name.split(os.sep)):
2614 parser.error('Do not include relative path components in --name.')
2615
nsylvain@google.comefc80932011-05-31 21:27:56 +00002616 deps_file = options.deps_file
agablea98a6cd2016-11-15 14:30:10 -08002617 client.SetDefaultConfig(name, deps_file, base_url,
smutae7ea312016-07-18 11:59:41 -07002618 managed=not options.unmanaged,
Paweł Hajdan, Jr3ba2a7c2017-10-04 19:24:46 +02002619 cache_dir=options.cache_dir,
2620 custom_vars=custom_vars)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002621 client.SaveConfig()
maruel@chromium.org79692d62010-05-14 18:57:13 +00002622 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002623
2624
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002625@subcommand.epilog("""Example:
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002626 gclient pack > patch.txt
2627 generate simple patch for configured client and dependences
2628""")
2629def CMDpack(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002630 """Generates a patch which can be applied at the root of the tree.
maruel@chromium.orgddff62d2010-05-17 21:02:36 +00002631
agabled437d762016-10-17 09:35:11 -07002632 Internally, runs 'git diff' on each checked out module and
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002633 dependencies, and performs minimal postprocessing of the output. The
2634 resulting patch is printed to stdout and can be applied to a freshly
2635 checked out tree via 'patch -p0 < patchfile'.
2636 """
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002637 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2638 help='override deps for the specified (comma-separated) '
2639 'platform(s); \'all\' will process all deps_os '
2640 'references')
haitao.feng@intel.com306080c2012-05-04 13:11:29 +00002641 parser.remove_option('--jobs')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002642 (options, args) = parser.parse_args(args)
iannucci@chromium.org50395ea2013-04-04 04:47:42 +00002643 # Force jobs to 1 so the stdout is not annotated with the thread ids
haitao.feng@intel.com306080c2012-05-04 13:11:29 +00002644 options.jobs = 1
kbr@google.comab318592009-09-04 00:54:55 +00002645 client = GClient.LoadCurrentConfig(options)
2646 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002647 raise gclient_utils.Error('client not configured; see \'gclient config\'')
kbr@google.comab318592009-09-04 00:54:55 +00002648 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002649 client.PrintLocationAndContents()
kbr@google.comab318592009-09-04 00:54:55 +00002650 return client.RunOnDeps('pack', args)
2651
2652
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002653def CMDstatus(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002654 """Shows modification status for every dependencies."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002655 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2656 help='override deps for the specified (comma-separated) '
2657 'platform(s); \'all\' will process all deps_os '
2658 'references')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002659 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002660 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002661 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002662 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002663 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002664 client.PrintLocationAndContents()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002665 return client.RunOnDeps('status', args)
2666
2667
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002668@subcommand.epilog("""Examples:
maruel@chromium.org79692d62010-05-14 18:57:13 +00002669 gclient sync
2670 update files from SCM according to current configuration,
2671 *for modules which have changed since last update or sync*
2672 gclient sync --force
2673 update files from SCM according to current configuration, for
2674 all modules (useful for recovering files deleted from local copy)
2675 gclient sync --revision src@31000
2676 update src directory to r31000
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002677
2678JSON output format:
2679If the --output-json option is specified, the following document structure will
2680be emitted to the provided file. 'null' entries may occur for subprojects which
2681are present in the gclient solution, but were not processed (due to custom_deps,
2682os_deps, etc.)
2683
2684{
2685 "solutions" : {
2686 "<name>": { # <name> is the posix-normalized path to the solution.
agabled437d762016-10-17 09:35:11 -07002687 "revision": [<git id hex string>|null],
2688 "scm": ["git"|null],
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002689 }
2690 }
2691}
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002692""")
2693def CMDsync(parser, args):
2694 """Checkout/update all modules."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002695 parser.add_option('-f', '--force', action='store_true',
2696 help='force update even for unchanged modules')
2697 parser.add_option('-n', '--nohooks', action='store_true',
2698 help='don\'t run hooks after the update is complete')
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00002699 parser.add_option('-p', '--noprehooks', action='store_true',
2700 help='don\'t run pre-DEPS hooks', default=False)
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002701 parser.add_option('-r', '--revision', action='append',
2702 dest='revisions', metavar='REV', default=[],
2703 help='Enforces revision/hash for the solutions with the '
2704 'format src@rev. The src@ part is optional and can be '
Edward Lesmes53014652018-03-07 18:01:40 -05002705 'skipped. You can also specify URLs instead of paths '
2706 'and gclient will find the solution corresponding to '
2707 'the given URL. If a path is also specified, the URL '
2708 'takes precedence. -r can be used multiple times when '
2709 '.gclient has multiple solutions configured, and will '
2710 'work even if the src@ part is skipped.')
Edward Lesmesc621b212018-03-21 20:26:56 -04002711 parser.add_option('--patch-ref', action='append',
2712 dest='patch_refs', metavar='GERRIT_REF', default=[],
2713 help='Patches the given reference with the format dep@ref. '
2714 'For dep, you can specify URLs as well as paths, with '
2715 'URLs taking preference. The reference will be '
2716 'applied to the necessary path, will be rebased on '
2717 'top what the dep was synced to, and then will do a '
2718 'soft reset. Use --no-rebase-patch-ref and '
2719 '--reset-patch-ref to disable this behavior.')
maruel@chromium.org794207e2013-03-08 15:29:43 +00002720 parser.add_option('--with_branch_heads', action='store_true',
2721 help='Clone git "branch_heads" refspecs in addition to '
2722 'the default refspecs. This adds about 1/2GB to a '
2723 'full checkout. (git only)')
szager@chromium.org8d3348f2014-08-19 22:49:16 +00002724 parser.add_option('--with_tags', action='store_true',
2725 help='Clone git tags in addition to the default refspecs.')
agable2697cd12016-06-28 10:23:53 -07002726 parser.add_option('-H', '--head', action='store_true',
agablea98a6cd2016-11-15 14:30:10 -08002727 help='DEPRECATED: only made sense with safesync urls.')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002728 parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002729 help='Deletes from the working copy any dependencies that '
2730 'have been removed since the last sync, as long as '
2731 'there are no local modifications. When used with '
2732 '--force, such dependencies are removed even if they '
2733 'have local modifications. When used with --reset, '
2734 'all untracked directories are removed from the '
tnagel@chromium.orga2aaa632014-02-28 21:47:27 +00002735 'working copy, excluding those which are explicitly '
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002736 'ignored in the repository.')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002737 parser.add_option('-R', '--reset', action='store_true',
2738 help='resets any local changes before updating (git only)')
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +00002739 parser.add_option('-M', '--merge', action='store_true',
2740 help='merge upstream changes instead of trying to '
2741 'fast-forward or rebase')
dnj@chromium.org5b23e872015-02-20 21:25:57 +00002742 parser.add_option('-A', '--auto_rebase', action='store_true',
2743 help='Automatically rebase repositories against local '
2744 'checkout during update (git only).')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002745 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2746 help='override deps for the specified (comma-separated) '
2747 'platform(s); \'all\' will process all deps_os '
2748 'references')
Paweł Hajdan, Jr357415c2017-07-24 14:35:28 +02002749 # TODO(phajdan.jr): use argparse.SUPPRESS to hide internal flags.
2750 parser.add_option('--do-not-merge-os-specific-entries', action='store_true',
2751 help='INTERNAL ONLY - disables merging of deps_os and '
2752 'hooks_os to dependencies and hooks')
Paweł Hajdan, Jr5ec77132017-08-16 19:21:06 +02002753 parser.add_option('--process-all-deps', action='store_true',
2754 help='Check out all deps, even for different OS-es, '
2755 'or with conditions evaluating to false')
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +00002756 parser.add_option('--upstream', action='store_true',
2757 help='Make repo state match upstream branch.')
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002758 parser.add_option('--output-json',
2759 help='Output a json document to this path containing '
2760 'summary information about the sync.')
primiano@chromium.org5439ea52014-08-06 17:18:18 +00002761 parser.add_option('--no-history', action='store_true',
2762 help='GIT ONLY - Reduces the size/time of the checkout at '
2763 'the cost of no history. Requires Git 1.9+')
hinoka@chromium.org46b87412014-05-15 00:42:05 +00002764 parser.add_option('--shallow', action='store_true',
2765 help='GIT ONLY - Do a shallow clone into the cache dir. '
2766 'Requires Git 1.9+')
e.hakkinen@samsung.come8bc1aa2015-04-08 08:00:37 +00002767 parser.add_option('--no_bootstrap', '--no-bootstrap',
2768 action='store_true',
2769 help='Don\'t bootstrap from Google Storage.')
Vadim Shtayura08049e22017-10-11 00:14:52 +00002770 parser.add_option('--ignore_locks', action='store_true',
2771 help='GIT ONLY - Ignore cache locks.')
iannucci@chromium.org30a07982016-04-07 21:35:19 +00002772 parser.add_option('--break_repo_locks', action='store_true',
2773 help='GIT ONLY - Forcibly remove repo locks (e.g. '
2774 'index.lock). This should only be used if you know for '
2775 'certain that this invocation of gclient is the only '
2776 'thing operating on the git repos (e.g. on a bot).')
Vadim Shtayura08049e22017-10-11 00:14:52 +00002777 parser.add_option('--lock_timeout', type='int', default=5000,
2778 help='GIT ONLY - Deadline (in seconds) to wait for git '
2779 'cache lock to become available. Default is %default.')
agabled437d762016-10-17 09:35:11 -07002780 # TODO(agable): Remove these when the oldest CrOS release milestone is M56.
2781 parser.add_option('-t', '--transitive', action='store_true',
2782 help='DEPRECATED: This is a no-op.')
sdefresne69b1be12016-10-18 05:48:02 -07002783 parser.add_option('-m', '--manually_grab_svn_rev', action='store_true',
agabled437d762016-10-17 09:35:11 -07002784 help='DEPRECATED: This is a no-op.')
Paweł Hajdan, Jr7c7b5592017-05-23 15:06:05 +02002785 # TODO(phajdan.jr): Remove validation options once default (crbug/570091).
Paweł Hajdan, Jr694773d2017-05-29 16:06:23 +02002786 parser.add_option('--validate-syntax', action='store_true', default=True,
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02002787 help='Validate the .gclient and DEPS syntax')
Paweł Hajdan, Jr7c7b5592017-05-23 15:06:05 +02002788 parser.add_option('--disable-syntax-validation', action='store_false',
2789 dest='validate_syntax',
2790 help='Disable validation of .gclient and DEPS syntax.')
Edward Lesmesc621b212018-03-21 20:26:56 -04002791 parser.add_option('--no-rebase-patch-ref', action='store_false',
2792 dest='rebase_patch_ref', default=True,
2793 help='Bypass rebase of the patch ref after checkout.')
2794 parser.add_option('--no-reset-patch-ref', action='store_false',
2795 dest='reset_patch_ref', default=True,
2796 help='Bypass calling reset after patching the ref.')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002797 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002798 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002799
2800 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002801 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002802
smutae7ea312016-07-18 11:59:41 -07002803 if options.revisions and options.head:
2804 # TODO(maruel): Make it a parser.error if it doesn't break any builder.
2805 print('Warning: you cannot use both --head and --revision')
2806
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002807 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002808 client.PrintLocationAndContents()
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002809 ret = client.RunOnDeps('update', args)
2810 if options.output_json:
2811 slns = {}
2812 for d in client.subtree(True):
2813 normed = d.name.replace('\\', '/').rstrip('/') + '/'
2814 slns[normed] = {
2815 'revision': d.got_revision,
2816 'scm': d.used_scm.name if d.used_scm else None,
hinoka@chromium.org17db9052014-05-10 01:11:29 +00002817 'url': str(d.url) if d.url else None,
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002818 }
2819 with open(options.output_json, 'wb') as f:
2820 json.dump({'solutions': slns}, f)
2821 return ret
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002822
2823
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002824CMDupdate = CMDsync
2825
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002826
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02002827def CMDvalidate(parser, args):
2828 """Validates the .gclient and DEPS syntax."""
2829 options, args = parser.parse_args(args)
2830 options.validate_syntax = True
2831 client = GClient.LoadCurrentConfig(options)
2832 rv = client.RunOnDeps('validate', args)
2833 if rv == 0:
2834 print('validate: SUCCESS')
2835 else:
2836 print('validate: FAILURE')
2837 return rv
2838
2839
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002840def CMDdiff(parser, args):
2841 """Displays local diff for every dependencies."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002842 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2843 help='override deps for the specified (comma-separated) '
2844 'platform(s); \'all\' will process all deps_os '
2845 'references')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002846 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002847 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002848 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002849 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002850 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002851 client.PrintLocationAndContents()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002852 return client.RunOnDeps('diff', args)
2853
2854
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002855def CMDrevert(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002856 """Reverts all modifications in every dependencies.
maruel@chromium.org28d14bd2010-11-11 20:37:09 +00002857
2858 That's the nuclear option to get back to a 'clean' state. It removes anything
agabled437d762016-10-17 09:35:11 -07002859 that shows up in git status."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002860 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2861 help='override deps for the specified (comma-separated) '
2862 'platform(s); \'all\' will process all deps_os '
2863 'references')
2864 parser.add_option('-n', '--nohooks', action='store_true',
2865 help='don\'t run hooks after the revert is complete')
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00002866 parser.add_option('-p', '--noprehooks', action='store_true',
2867 help='don\'t run pre-DEPS hooks', default=False)
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +00002868 parser.add_option('--upstream', action='store_true',
2869 help='Make repo state match upstream branch.')
iannucci@chromium.orgbf525dc2016-04-07 22:00:28 +00002870 parser.add_option('--break_repo_locks', action='store_true',
2871 help='GIT ONLY - Forcibly remove repo locks (e.g. '
2872 'index.lock). This should only be used if you know for '
2873 'certain that this invocation of gclient is the only '
2874 'thing operating on the git repos (e.g. on a bot).')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002875 (options, args) = parser.parse_args(args)
2876 # --force is implied.
2877 options.force = True
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002878 options.reset = False
2879 options.delete_unversioned_trees = False
agablec903d732016-07-26 09:07:24 -07002880 options.merge = False
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002881 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002882 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002883 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002884 return client.RunOnDeps('revert', args)
2885
2886
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002887def CMDrunhooks(parser, args):
2888 """Runs hooks for files that have been modified in the local working copy."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002889 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2890 help='override deps for the specified (comma-separated) '
2891 'platform(s); \'all\' will process all deps_os '
2892 'references')
2893 parser.add_option('-f', '--force', action='store_true', default=True,
2894 help='Deprecated. No effect.')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002895 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002896 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002897 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002898 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002899 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002900 client.PrintLocationAndContents()
maruel@chromium.org5df6a462009-08-28 18:52:26 +00002901 options.force = True
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002902 options.nohooks = False
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002903 return client.RunOnDeps('runhooks', args)
2904
2905
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002906def CMDrevinfo(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002907 """Outputs revision info mapping for the client and its dependencies.
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002908
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002909 This allows the capture of an overall 'revision' for the source tree that
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002910 can be used to reproduce the same tree in the future. It is only useful for
agabled437d762016-10-17 09:35:11 -07002911 'unpinned dependencies', i.e. DEPS/deps references without a git hash.
2912 A git branch name isn't 'pinned' since the actual commit can change.
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002913 """
2914 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2915 help='override deps for the specified (comma-separated) '
2916 'platform(s); \'all\' will process all deps_os '
2917 'references')
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00002918 parser.add_option('-a', '--actual', action='store_true',
2919 help='gets the actual checked out revisions instead of the '
2920 'ones specified in the DEPS and .gclient files')
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002921 parser.add_option('-s', '--snapshot', action='store_true',
2922 help='creates a snapshot .gclient file of the current '
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00002923 'version of all repositories to reproduce the tree, '
2924 'implies -a')
Edward Lesmesbb16e332018-03-30 17:54:51 -04002925 parser.add_option('--filter', action='append', dest='filter',
Edward Lesmesdaa76d22018-03-06 14:56:57 -05002926 help='Display revision information only for the specified '
Edward Lesmesbb16e332018-03-30 17:54:51 -04002927 'dependencies (filtered by URL or path).')
Edward Lesmesc2960242018-03-06 20:50:15 -05002928 parser.add_option('--output-json',
2929 help='Output a json document to this path containing '
2930 'information about the revisions.')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002931 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002932 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002933 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002934 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002935 client.PrintRevInfo()
maruel@chromium.org79692d62010-05-14 18:57:13 +00002936 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002937
2938
Edward Lesmes6f64a052018-03-20 17:35:49 -04002939def CMDsetdep(parser, args):
2940 parser.add_option('--var', action='append',
2941 dest='vars', metavar='VAR=VAL', default=[],
2942 help='Sets a variable to the given value with the format '
2943 'name=value.')
2944 parser.add_option('-r', '--revision', action='append',
2945 dest='revisions', metavar='DEP@REV', default=[],
2946 help='Sets the revision/version for the dependency with '
2947 'the format dep@rev. If it is a git dependency, dep '
2948 'must be a path and rev must be a git hash or '
2949 'reference (e.g. src/dep@deadbeef). If it is a CIPD '
2950 'dependency, dep must be of the form path:package and '
2951 'rev must be the package version '
2952 '(e.g. src/pkg:chromium/pkg@2.1-cr0).')
2953 parser.add_option('--deps-file', default='DEPS',
2954 # TODO(ehmaldonado): Try to find the DEPS file pointed by
2955 # .gclient first.
2956 help='The DEPS file to be edited. Defaults to the DEPS '
2957 'file in the current directory.')
2958 (options, args) = parser.parse_args(args)
2959
Edward Lesmes6f64a052018-03-20 17:35:49 -04002960 if not os.path.isfile(options.deps_file):
2961 raise gclient_utils.Error(
2962 'DEPS file %s does not exist.' % options.deps_file)
2963 with open(options.deps_file) as f:
2964 contents = f.read()
Edward Lesmes6c24d372018-03-28 12:52:29 -04002965 local_scope = gclient_eval.Parse(
2966 contents, expand_vars=True, validate_syntax=True,
2967 filename=options.deps_file)
Edward Lesmes6f64a052018-03-20 17:35:49 -04002968
2969 for var in options.vars:
2970 name, _, value = var.partition('=')
2971 if not name or not value:
2972 raise gclient_utils.Error(
2973 'Wrong var format: %s should be of the form name=value.' % var)
2974 gclient_eval.SetVar(local_scope, name, value)
2975
2976 for revision in options.revisions:
2977 name, _, value = revision.partition('@')
2978 if not name or not value:
2979 raise gclient_utils.Error(
2980 'Wrong dep format: %s should be of the form dep@rev.' % revision)
2981 if ':' in name:
2982 name, _, package = name.partition(':')
2983 if not name or not package:
2984 raise gclient_utils.Error(
2985 'Wrong CIPD format: %s:%s should be of the form path:pkg@version.'
2986 % (name, package))
2987 gclient_eval.SetCIPD(local_scope, name, package, value)
2988 else:
Edward Lesmes9f531292018-03-20 21:27:15 -04002989 gclient_eval.SetRevision(local_scope, name, value)
Edward Lesmes6f64a052018-03-20 17:35:49 -04002990
2991 with open(options.deps_file, 'w') as f:
2992 f.write(gclient_eval.RenderDEPSFile(local_scope))
2993
2994
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00002995def CMDverify(parser, args):
2996 """Verifies the DEPS file deps are only from allowed_hosts."""
2997 (options, args) = parser.parse_args(args)
2998 client = GClient.LoadCurrentConfig(options)
2999 if not client:
3000 raise gclient_utils.Error('client not configured; see \'gclient config\'')
3001 client.RunOnDeps(None, [])
3002 # Look at each first-level dependency of this gclient only.
3003 for dep in client.dependencies:
3004 bad_deps = dep.findDepsFromNotAllowedHosts()
3005 if not bad_deps:
3006 continue
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003007 print("There are deps from not allowed hosts in file %s" % dep.deps_file)
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00003008 for bad_dep in bad_deps:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003009 print("\t%s at %s" % (bad_dep.name, bad_dep.url))
3010 print("allowed_hosts:", ', '.join(dep.allowed_hosts))
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00003011 sys.stdout.flush()
3012 raise gclient_utils.Error(
3013 'dependencies from disallowed hosts; check your DEPS file.')
3014 return 0
3015
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003016class OptionParser(optparse.OptionParser):
szager@chromium.orge2e03202012-07-31 18:05:16 +00003017 gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003018
3019 def __init__(self, **kwargs):
3020 optparse.OptionParser.__init__(
3021 self, version='%prog ' + __version__, **kwargs)
3022
3023 # Some arm boards have issues with parallel sync.
3024 if platform.machine().startswith('arm'):
3025 jobs = 1
3026 else:
3027 jobs = max(8, gclient_utils.NumLocalCpus())
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003028
3029 self.add_option(
3030 '-j', '--jobs', default=jobs, type='int',
3031 help='Specify how many SCM commands can run in parallel; defaults to '
tnagel@chromium.orga2aaa632014-02-28 21:47:27 +00003032 '%default on this machine')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003033 self.add_option(
3034 '-v', '--verbose', action='count', default=0,
3035 help='Produces additional output for diagnostics. Can be used up to '
3036 'three times for more logging info.')
3037 self.add_option(
3038 '--gclientfile', dest='config_filename',
3039 help='Specify an alternate %s file' % self.gclientfile_default)
3040 self.add_option(
3041 '--spec',
3042 help='create a gclient file containing the provided string. Due to '
3043 'Cygwin/Python brokenness, it can\'t contain any newlines.')
3044 self.add_option(
Aleksandr Derbenev9e8fb0e2017-08-01 20:18:31 +03003045 '--cache-dir',
3046 help='(git only) Cache all git repos into this dir and do '
3047 'shared clones from the cache, instead of cloning '
3048 'directly from the remote. (experimental)',
3049 default=os.environ.get('GCLIENT_CACHE_DIR'))
3050 self.add_option(
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003051 '--no-nag-max', default=False, action='store_true',
scottmg@chromium.orgf547c802013-09-27 17:55:26 +00003052 help='Ignored for backwards compatibility.')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003053
3054 def parse_args(self, args=None, values=None):
3055 """Integrates standard options processing."""
3056 options, args = optparse.OptionParser.parse_args(self, args, values)
3057 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
3058 logging.basicConfig(
3059 level=levels[min(options.verbose, len(levels) - 1)],
maruel@chromium.org0895b752011-08-26 20:40:33 +00003060 format='%(module)s(%(lineno)d) %(funcName)s:%(message)s')
szager@chromium.orge2e03202012-07-31 18:05:16 +00003061 if options.config_filename and options.spec:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003062 self.error('Cannot specifiy both --gclientfile and --spec')
rdsmith@chromium.orgd9591f02014-02-05 19:28:20 +00003063 if (options.config_filename and
3064 options.config_filename != os.path.basename(options.config_filename)):
3065 self.error('--gclientfile target must be a filename, not a path')
szager@chromium.orge2e03202012-07-31 18:05:16 +00003066 if not options.config_filename:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003067 options.config_filename = self.gclientfile_default
maruel@chromium.org0895b752011-08-26 20:40:33 +00003068 options.entries_filename = options.config_filename + '_entries'
3069 if options.jobs < 1:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003070 self.error('--jobs must be 1 or higher')
maruel@chromium.org0895b752011-08-26 20:40:33 +00003071
3072 # These hacks need to die.
3073 if not hasattr(options, 'revisions'):
3074 # GClient.RunOnDeps expects it even if not applicable.
3075 options.revisions = []
smutae7ea312016-07-18 11:59:41 -07003076 if not hasattr(options, 'head'):
3077 options.head = None
maruel@chromium.org0895b752011-08-26 20:40:33 +00003078 if not hasattr(options, 'nohooks'):
3079 options.nohooks = True
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00003080 if not hasattr(options, 'noprehooks'):
3081 options.noprehooks = True
maruel@chromium.org0895b752011-08-26 20:40:33 +00003082 if not hasattr(options, 'deps_os'):
3083 options.deps_os = None
maruel@chromium.org0895b752011-08-26 20:40:33 +00003084 if not hasattr(options, 'force'):
3085 options.force = None
3086 return (options, args)
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00003087
maruel@chromium.org39c0b222013-08-17 16:57:01 +00003088
3089def disable_buffering():
3090 # Make stdout auto-flush so buildbot doesn't kill us during lengthy
3091 # operations. Python as a strong tendency to buffer sys.stdout.
3092 sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout)
3093 # Make stdout annotated with the thread ids.
3094 sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout)
maruel@chromium.org0895b752011-08-26 20:40:33 +00003095
3096
sbc@chromium.org013731e2015-02-26 18:28:43 +00003097def main(argv):
maruel@chromium.org5ca27692010-05-26 19:32:41 +00003098 """Doesn't parse the arguments here, just find the right subcommand to
3099 execute."""
maruel@chromium.org82798cb2012-02-23 18:16:12 +00003100 if sys.hexversion < 0x02060000:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003101 print(
maruel@chromium.org82798cb2012-02-23 18:16:12 +00003102 '\nYour python version %s is unsupported, please upgrade.\n' %
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003103 sys.version.split(' ', 1)[0],
3104 file=sys.stderr)
maruel@chromium.org82798cb2012-02-23 18:16:12 +00003105 return 2
bcwhite@chromium.org6683ab42013-02-11 16:13:47 +00003106 if not sys.executable:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003107 print(
3108 '\nPython cannot find the location of it\'s own executable.\n',
3109 file=sys.stderr)
bcwhite@chromium.org6683ab42013-02-11 16:13:47 +00003110 return 2
maruel@chromium.org39c0b222013-08-17 16:57:01 +00003111 fix_encoding.fix_encoding()
3112 disable_buffering()
iannucci@chromium.org596cd5c2016-04-04 21:34:39 +00003113 setup_color.init()
maruel@chromium.org39c0b222013-08-17 16:57:01 +00003114 dispatcher = subcommand.CommandDispatcher(__name__)
maruel@chromium.org6e29d572010-06-04 17:32:20 +00003115 try:
maruel@chromium.org39c0b222013-08-17 16:57:01 +00003116 return dispatcher.execute(OptionParser(), argv)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00003117 except KeyboardInterrupt:
3118 gclient_utils.GClientChildren.KillAllRemainingChildren()
3119 raise
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00003120 except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00003121 print('Error: %s' % str(e), file=sys.stderr)
maruel@chromium.org6e29d572010-06-04 17:32:20 +00003122 return 1
borenet@google.com6a9b1682014-03-24 18:35:23 +00003123 finally:
3124 gclient_utils.PrintWarnings()
sbc@chromium.org013731e2015-02-26 18:28:43 +00003125 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003126
3127
maruel@chromium.orgf0fc9912010-06-11 17:57:33 +00003128if '__main__' == __name__:
sbc@chromium.org013731e2015-02-26 18:28:43 +00003129 try:
3130 sys.exit(main(sys.argv[1:]))
3131 except KeyboardInterrupt:
3132 sys.stderr.write('interrupted\n')
3133 sys.exit(1)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003134
3135# vim: ts=2:sw=2:tw=80:et: