blob: 01c40ffc1bbf62eb78e8e2f4b73b89f177a4c103 [file] [log] [blame]
iannucci@chromium.org405b87e2015-11-12 18:08:34 +00001#!/usr/bin/env python
thakis@chromium.org4f474b62012-01-18 01:31:29 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgba551772010-02-03 18:21:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00005
agabled437d762016-10-17 09:35:11 -07006"""Meta checkout dependency manager for Git."""
maruel@chromium.org39c0b222013-08-17 16:57:01 +00007# Files
8# .gclient : Current client configuration, written by 'config' command.
9# Format is a Python script defining 'solutions', a list whose
10# entries each are maps binding the strings "name" and "url"
11# to strings specifying the name and location of the client
12# module, as well as "custom_deps" to a map similar to the
13# deps section of the DEPS file below, as well as
14# "custom_hooks" to a list similar to the hooks sections of
15# the DEPS file below.
16# .gclient_entries : A cache constructed by 'update' command. Format is a
17# Python script defining 'entries', a list of the names
18# of all modules in the client
19# <module>/DEPS : Python script defining var 'deps' as a map from each
20# requisite submodule name to a URL where it can be found (via
21# one SCM)
22#
23# Hooks
24# .gclient and DEPS files may optionally contain a list named "hooks" to
25# allow custom actions to be performed based on files that have changed in the
26# working copy as a result of a "sync"/"update" or "revert" operation. This
27# can be prevented by using --nohooks (hooks run by default). Hooks can also
28# be forced to run with the "runhooks" operation. If "sync" is run with
29# --force, all known but not suppressed hooks will run regardless of the state
30# of the working copy.
31#
32# Each item in a "hooks" list is a dict, containing these two keys:
33# "pattern" The associated value is a string containing a regular
34# expression. When a file whose pathname matches the expression
35# is checked out, updated, or reverted, the hook's "action" will
36# run.
37# "action" A list describing a command to run along with its arguments, if
38# any. An action command will run at most one time per gclient
39# invocation, regardless of how many files matched the pattern.
40# The action is executed in the same directory as the .gclient
41# file. If the first item in the list is the string "python",
42# the current Python interpreter (sys.executable) will be used
43# to run the command. If the list contains string
44# "$matching_files" it will be removed from the list and the list
45# will be extended by the list of matching files.
46# "name" An optional string specifying the group to which a hook belongs
47# for overriding and organizing.
48#
49# Example:
50# hooks = [
51# { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
52# "action": ["python", "image_indexer.py", "--all"]},
53# { "pattern": ".",
54# "name": "gyp",
55# "action": ["python", "src/build/gyp_chromium"]},
56# ]
57#
borenet@google.com2d1ee9e2013-10-15 08:13:16 +000058# Pre-DEPS Hooks
59# DEPS files may optionally contain a list named "pre_deps_hooks". These are
60# the same as normal hooks, except that they run before the DEPS are
61# processed. Pre-DEPS run with "sync" and "revert" unless the --noprehooks
62# flag is used.
rdsmith@chromium.orgd9591f02014-02-05 19:28:20 +000063#
maruel@chromium.org39c0b222013-08-17 16:57:01 +000064# Specifying a target OS
65# An optional key named "target_os" may be added to a gclient file to specify
66# one or more additional operating systems that should be considered when
Scott Grahamc4826742017-05-11 16:59:23 -070067# processing the deps_os/hooks_os dict of a DEPS file.
maruel@chromium.org39c0b222013-08-17 16:57:01 +000068#
69# Example:
70# target_os = [ "android" ]
71#
72# If the "target_os_only" key is also present and true, then *only* the
73# operating systems listed in "target_os" will be used.
74#
75# Example:
76# target_os = [ "ios" ]
77# target_os_only = True
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000078
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +000079from __future__ import print_function
80
maruel@chromium.org39c0b222013-08-17 16:57:01 +000081__version__ = '0.7'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000082
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +020083import collections
maruel@chromium.org9e5317a2010-08-13 20:35:11 +000084import copy
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +000085import json
maruel@chromium.org754960e2009-09-21 12:31:05 +000086import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000087import optparse
88import os
bradnelson@google.com4949dab2012-04-19 16:41:07 +000089import platform
maruel@chromium.org621939b2010-08-10 20:12:00 +000090import posixpath
msb@chromium.org2e38de72009-09-28 17:04:47 +000091import pprint
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000092import re
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000093import sys
ilevy@chromium.orgc28d3772013-07-12 19:42:37 +000094import time
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000095import urllib
bradnelson@google.com4949dab2012-04-19 16:41:07 +000096import urlparse
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000097
maruel@chromium.org35625c72011-03-23 17:34:02 +000098import fix_encoding
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +020099import gclient_eval
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000100import gclient_scm
101import gclient_utils
szager@chromium.org848fd492014-04-09 19:06:44 +0000102import git_cache
nasser@codeaurora.org1f7a3d12010-02-04 15:11:50 +0000103from third_party.repo.progress import Progress
maruel@chromium.org39c0b222013-08-17 16:57:01 +0000104import subcommand
maruel@chromium.org31cb48a2011-04-04 18:01:36 +0000105import subprocess2
iannucci@chromium.org596cd5c2016-04-04 21:34:39 +0000106import setup_color
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000107
108
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200109class GNException(Exception):
110 pass
111
112
113def ToGNString(value, allow_dicts = True):
114 """Returns a stringified GN equivalent of the Python value.
115
116 allow_dicts indicates if this function will allow converting dictionaries
117 to GN scopes. This is only possible at the top level, you can't nest a
118 GN scope in a list, so this should be set to False for recursive calls."""
119 if isinstance(value, basestring):
120 if value.find('\n') >= 0:
121 raise GNException("Trying to print a string with a newline in it.")
122 return '"' + \
123 value.replace('\\', '\\\\').replace('"', '\\"').replace('$', '\\$') + \
124 '"'
125
126 if isinstance(value, unicode):
127 return ToGNString(value.encode('utf-8'))
128
129 if isinstance(value, bool):
130 if value:
131 return "true"
132 return "false"
133
134 # NOTE: some type handling removed compared to chromium/src copy.
135
136 raise GNException("Unsupported type when printing to GN.")
137
138
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200139class Hook(object):
140 """Descriptor of command ran before/after sync or on demand."""
141
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200142 def __init__(self, action, pattern=None, name=None, cwd=None):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200143 """Constructor.
144
145 Arguments:
146 action (list of basestring): argv of the command to run
147 pattern (basestring regex): noop with git; deprecated
148 name (basestring): optional name; no effect on operation
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200149 cwd (basestring): working directory to use
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200150 """
151 self._action = gclient_utils.freeze(action)
152 self._pattern = pattern
153 self._name = name
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200154 self._cwd = cwd
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200155
156 @staticmethod
157 def from_dict(d):
158 """Creates a Hook instance from a dict like in the DEPS file."""
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200159 return Hook(d['action'], d.get('pattern'), d.get('name'), d.get('cwd'))
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200160
161 @property
162 def action(self):
163 return self._action
164
165 @property
166 def pattern(self):
167 return self._pattern
168
169 @property
170 def name(self):
171 return self._name
172
173 def matches(self, file_list):
174 """Returns true if the pattern matches any of files in the list."""
175 if not self._pattern:
176 return True
177 pattern = re.compile(self._pattern)
178 return bool([f for f in file_list if pattern.search(f)])
179
180 def run(self, root):
181 """Executes the hook's command."""
182 cmd = list(self._action)
183 if cmd[0] == 'python':
184 # If the hook specified "python" as the first item, the action is a
185 # Python script. Run it by starting a new copy of the same
186 # interpreter.
187 cmd[0] = sys.executable
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200188
189 cwd = root
190 if self._cwd:
191 cwd = os.path.join(cwd, self._cwd)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200192 try:
193 start_time = time.time()
194 gclient_utils.CheckCallAndFilterAndHeader(
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +0200195 cmd, cwd=cwd, always=True)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200196 except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
197 # Use a discrete exit status code of 2 to indicate that a hook action
198 # failed. Users of this script may wish to treat hook action failures
199 # differently from VC failures.
200 print('Error: %s' % str(e), file=sys.stderr)
201 sys.exit(2)
202 finally:
203 elapsed_time = time.time() - start_time
204 if elapsed_time > 10:
205 print("Hook '%s' took %.2f secs" % (
206 gclient_utils.CommandToStr(cmd), elapsed_time))
207
208
maruel@chromium.org116704f2010-06-11 17:34:38 +0000209class GClientKeywords(object):
maruel@chromium.org116704f2010-06-11 17:34:38 +0000210 class VarImpl(object):
211 def __init__(self, custom_vars, local_scope):
212 self._custom_vars = custom_vars
213 self._local_scope = local_scope
214
215 def Lookup(self, var_name):
216 """Implements the Var syntax."""
217 if var_name in self._custom_vars:
218 return self._custom_vars[var_name]
219 elif var_name in self._local_scope.get("vars", {}):
220 return self._local_scope["vars"][var_name]
221 raise gclient_utils.Error("Var is not defined: %s" % var_name)
222
223
maruel@chromium.org064186c2011-09-27 23:53:33 +0000224class DependencySettings(GClientKeywords):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000225 """Immutable configuration settings."""
226 def __init__(
agablea98a6cd2016-11-15 14:30:10 -0800227 self, parent, url, managed, custom_deps, custom_vars,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200228 custom_hooks, deps_file, should_process, relative,
229 condition, condition_value):
maruel@chromium.org064186c2011-09-27 23:53:33 +0000230 GClientKeywords.__init__(self)
231
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000232 # These are not mutable:
233 self._parent = parent
mmoss@chromium.org8f93f792014-08-26 23:24:09 +0000234 self._deps_file = deps_file
maruel@chromium.org064186c2011-09-27 23:53:33 +0000235 self._url = url
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200236 # The condition as string (or None). Useful to keep e.g. for flatten.
237 self._condition = condition
238 # Boolean value of the condition. If there's no condition, just True.
239 self._condition_value = condition_value
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000240 # 'managed' determines whether or not this dependency is synced/updated by
241 # gclient after gclient checks it out initially. The difference between
242 # 'managed' and 'should_process' is that the user specifies 'managed' via
smutae7ea312016-07-18 11:59:41 -0700243 # the --unmanaged command-line flag or a .gclient config, where
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000244 # 'should_process' is dynamically set by gclient if it goes over its
245 # recursion limit and controls gclient's behavior so it does not misbehave.
246 self._managed = managed
247 self._should_process = should_process
agabledce6ddc2016-09-08 10:02:16 -0700248 # If this is a recursed-upon sub-dependency, and the parent has
249 # use_relative_paths set, then this dependency should check out its own
250 # dependencies relative to that parent's path for this, rather than
251 # relative to the .gclient file.
252 self._relative = relative
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +0000253 # This is a mutable value which has the list of 'target_os' OSes listed in
254 # the current deps file.
255 self.local_target_os = None
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000256
257 # These are only set in .gclient and not in DEPS files.
258 self._custom_vars = custom_vars or {}
259 self._custom_deps = custom_deps or {}
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000260 self._custom_hooks = custom_hooks or []
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000261
maruel@chromium.org064186c2011-09-27 23:53:33 +0000262 # Post process the url to remove trailing slashes.
263 if isinstance(self._url, basestring):
264 # urls are sometime incorrectly written as proto://host/path/@rev. Replace
265 # it to proto://host/path@rev.
maruel@chromium.org064186c2011-09-27 23:53:33 +0000266 self._url = self._url.replace('/@', '@')
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200267 elif not isinstance(self._url, (None.__class__)):
maruel@chromium.org064186c2011-09-27 23:53:33 +0000268 raise gclient_utils.Error(
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200269 ('dependency url must be either string or None, '
270 'instead of %s') % self._url.__class__.__name__)
mmoss@chromium.orgd0b272b2013-01-30 23:55:33 +0000271 # Make any deps_file path platform-appropriate.
272 for sep in ['/', '\\']:
273 self._deps_file = self._deps_file.replace(sep, os.sep)
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000274
275 @property
276 def deps_file(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000277 return self._deps_file
278
279 @property
280 def managed(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000281 return self._managed
282
283 @property
284 def parent(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000285 return self._parent
286
287 @property
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000288 def root(self):
289 """Returns the root node, a GClient object."""
290 if not self.parent:
291 # This line is to signal pylint that it could be a GClient instance.
292 return self or GClient(None, None)
293 return self.parent.root
294
295 @property
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000296 def should_process(self):
297 """True if this dependency should be processed, i.e. checked out."""
298 return self._should_process
299
300 @property
301 def custom_vars(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000302 return self._custom_vars.copy()
303
304 @property
305 def custom_deps(self):
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000306 return self._custom_deps.copy()
307
maruel@chromium.org064186c2011-09-27 23:53:33 +0000308 @property
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000309 def custom_hooks(self):
310 return self._custom_hooks[:]
311
312 @property
maruel@chromium.org064186c2011-09-27 23:53:33 +0000313 def url(self):
314 return self._url
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000315
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000316 @property
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200317 def condition(self):
318 return self._condition
319
320 @property
321 def condition_value(self):
322 return self._condition_value
323
324 @property
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +0000325 def target_os(self):
326 if self.local_target_os is not None:
327 return tuple(set(self.local_target_os).union(self.parent.target_os))
328 else:
329 return self.parent.target_os
330
maruel@chromium.org8c0d9582011-10-03 21:36:01 +0000331 def get_custom_deps(self, name, url):
332 """Returns a custom deps if applicable."""
333 if self.parent:
334 url = self.parent.get_custom_deps(name, url)
335 # None is a valid return value to disable a dependency.
336 return self.custom_deps.get(name, url)
337
maruel@chromium.org064186c2011-09-27 23:53:33 +0000338
339class Dependency(gclient_utils.WorkItem, DependencySettings):
maruel@chromium.org54a07a22010-06-14 19:07:39 +0000340 """Object that represents a dependency checkout."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +0000341
agablea98a6cd2016-11-15 14:30:10 -0800342 def __init__(self, parent, name, url, managed, custom_deps,
agabledce6ddc2016-09-08 10:02:16 -0700343 custom_vars, custom_hooks, deps_file, should_process,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200344 relative, condition, condition_value):
maruel@chromium.org6ca8bf82011-09-19 23:04:30 +0000345 gclient_utils.WorkItem.__init__(self, name)
maruel@chromium.org8ac2b272011-09-26 18:49:49 +0000346 DependencySettings.__init__(
agablea98a6cd2016-11-15 14:30:10 -0800347 self, parent, url, managed, custom_deps, custom_vars,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +0200348 custom_hooks, deps_file, should_process, relative,
349 condition, condition_value)
maruel@chromium.org68988972011-09-20 14:11:42 +0000350
351 # This is in both .gclient and DEPS files:
maruel@chromium.org064186c2011-09-27 23:53:33 +0000352 self._deps_hooks = []
maruel@chromium.org68988972011-09-20 14:11:42 +0000353
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000354 self._pre_deps_hooks = []
355
maruel@chromium.org68988972011-09-20 14:11:42 +0000356 # Calculates properties:
maruel@chromium.org064186c2011-09-27 23:53:33 +0000357 self._parsed_url = None
maruel@chromium.org4bdd5fd2011-09-26 19:41:17 +0000358 self._dependencies = []
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200359 # Keep track of original values, before post-processing (e.g. deps_os).
360 self._orig_dependencies = []
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200361 self._vars = {}
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +0200362 self._os_dependencies = {}
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200363
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000364 # A cache of the files affected by the current operation, necessary for
365 # hooks.
maruel@chromium.org861fd0f2010-07-23 03:05:05 +0000366 self._file_list = []
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000367 # List of host names from which dependencies are allowed.
368 # Default is an empty set, meaning unspecified in DEPS file, and hence all
369 # hosts will be allowed. Non-empty set means whitelist of hosts.
370 # allowed_hosts var is scoped to its DEPS file, and so it isn't recursive.
371 self._allowed_hosts = frozenset()
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200372 # Spec for .gni output to write (if any).
373 self._gn_args_file = None
374 self._gn_args = []
maruel@chromium.org85c2a192010-07-22 21:14:43 +0000375 # If it is not set to True, the dependency wasn't processed for its child
376 # dependency, i.e. its DEPS wasn't read.
maruel@chromium.org064186c2011-09-27 23:53:33 +0000377 self._deps_parsed = False
maruel@chromium.orgf3abb802010-08-10 17:19:56 +0000378 # This dependency has been processed, i.e. checked out
maruel@chromium.org064186c2011-09-27 23:53:33 +0000379 self._processed = False
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000380 # This dependency had its pre-DEPS hooks run
381 self._pre_deps_hooks_ran = False
maruel@chromium.orgf3abb802010-08-10 17:19:56 +0000382 # This dependency had its hook run
maruel@chromium.org064186c2011-09-27 23:53:33 +0000383 self._hooks_ran = False
kustermann@google.coma692e8f2013-04-18 08:32:04 +0000384 # This is the scm used to checkout self.url. It may be used by dependencies
385 # to get the datetime of the revision we checked out.
386 self._used_scm = None
szager@chromium.org4ad264b2014-05-20 04:43:47 +0000387 self._used_revision = None
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +0000388 # The actual revision we ended up getting, or None if that information is
389 # unavailable
390 self._got_revision = None
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000391
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000392 # This is a mutable value that overrides the normal recursion limit for this
393 # dependency. It is read from the actual DEPS file so cannot be set on
394 # class instantiation.
395 self.recursion_override = None
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000396 # recursedeps is a mutable value that selectively overrides the default
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000397 # 'no recursion' setting on a dep-by-dep basis. It will replace
398 # recursion_override.
iannucci@chromium.orgafa11ac2016-05-04 22:17:34 +0000399 #
400 # It will be a dictionary of {deps_name: {"deps_file": depfile_name}} or
401 # None.
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000402 self.recursedeps = None
hinoka885e5b12016-06-08 14:40:09 -0700403 # This is inherited from WorkItem. We want the URL to be a resource.
404 if url and isinstance(url, basestring):
405 # The url is usually given to gclient either as https://blah@123
qyearsley12fa6ff2016-08-24 09:18:40 -0700406 # or just https://blah. The @123 portion is irrelevant.
hinoka885e5b12016-06-08 14:40:09 -0700407 self.resources.append(url.split('@')[0])
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000408
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000409 if not self.name and self.parent:
410 raise gclient_utils.Error('Dependency without name')
411
maruel@chromium.org470b5432011-10-11 18:18:19 +0000412 @property
413 def requirements(self):
414 """Calculate the list of requirements."""
415 requirements = set()
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000416 # self.parent is implicitly a requirement. This will be recursive by
417 # definition.
418 if self.parent and self.parent.name:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000419 requirements.add(self.parent.name)
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000420
421 # For a tree with at least 2 levels*, the leaf node needs to depend
422 # on the level higher up in an orderly way.
423 # This becomes messy for >2 depth as the DEPS file format is a dictionary,
424 # thus unsorted, while the .gclient format is a list thus sorted.
425 #
426 # * _recursion_limit is hard coded 2 and there is no hope to change this
427 # value.
428 #
429 # Interestingly enough, the following condition only works in the case we
430 # want: self is a 2nd level node. 3nd level node wouldn't need this since
431 # they already have their parent as a requirement.
maruel@chromium.org470b5432011-10-11 18:18:19 +0000432 if self.parent and self.parent.parent and not self.parent.parent.parent:
433 requirements |= set(i.name for i in self.root.dependencies if i.name)
maruel@chromium.org118fb1c2011-09-01 20:04:24 +0000434
maruel@chromium.org470b5432011-10-11 18:18:19 +0000435 if self.name:
436 requirements |= set(
437 obj.name for obj in self.root.subtree(False)
438 if (obj is not self
439 and obj.name and
440 self.name.startswith(posixpath.join(obj.name, ''))))
441 requirements = tuple(sorted(requirements))
442 logging.info('Dependency(%s).requirements = %s' % (self.name, requirements))
443 return requirements
444
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000445 @property
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000446 def try_recursedeps(self):
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000447 """Returns False if recursion_override is ever specified."""
448 if self.recursion_override is not None:
449 return False
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000450 return self.parent.try_recursedeps
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000451
452 @property
453 def recursion_limit(self):
454 """Returns > 0 if this dependency is not too recursed to be processed."""
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000455 # We continue to support the absence of recursedeps until tools and DEPS
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000456 # using recursion_override are updated.
cmp@chromium.orgc401ad12014-07-02 23:20:08 +0000457 if self.try_recursedeps and self.parent.recursedeps != None:
458 if self.name in self.parent.recursedeps:
cmp@chromium.orge84ac912014-06-30 23:14:35 +0000459 return 1
460
461 if self.recursion_override is not None:
462 return self.recursion_override
463 return max(self.parent.recursion_limit - 1, 0)
464
maruel@chromium.org470b5432011-10-11 18:18:19 +0000465 def verify_validity(self):
466 """Verifies that this Dependency is fine to add as a child of another one.
467
468 Returns True if this entry should be added, False if it is a duplicate of
469 another entry.
470 """
471 logging.info('Dependency(%s).verify_validity()' % self.name)
472 if self.name in [s.name for s in self.parent.dependencies]:
473 raise gclient_utils.Error(
474 'The same name "%s" appears multiple times in the deps section' %
475 self.name)
476 if not self.should_process:
477 # Return early, no need to set requirements.
478 return True
479
480 # This require a full tree traversal with locks.
481 siblings = [d for d in self.root.subtree(False) if d.name == self.name]
482 for sibling in siblings:
maruel@chromium.orgb848d5b2012-10-10 23:25:50 +0000483 self_url = self.LateOverride(self.url)
484 sibling_url = sibling.LateOverride(sibling.url)
485 # Allow to have only one to be None or ''.
486 if self_url != sibling_url and bool(self_url) == bool(sibling_url):
maruel@chromium.org470b5432011-10-11 18:18:19 +0000487 raise gclient_utils.Error(
maruel@chromium.orgb848d5b2012-10-10 23:25:50 +0000488 ('Dependency %s specified more than once:\n'
489 ' %s [%s]\n'
490 'vs\n'
491 ' %s [%s]') % (
492 self.name,
493 sibling.hierarchy(),
494 sibling_url,
495 self.hierarchy(),
496 self_url))
maruel@chromium.org470b5432011-10-11 18:18:19 +0000497 # In theory we could keep it as a shadow of the other one. In
498 # practice, simply ignore it.
499 logging.warn('Won\'t process duplicate dependency %s' % sibling)
500 return False
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000501 return True
maruel@chromium.org064186c2011-09-27 23:53:33 +0000502
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000503 def LateOverride(self, url):
Paweł Hajdan, Jr7e9303b2017-05-23 14:38:27 +0200504 """Resolves the parsed url from url."""
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000505 assert self.parsed_url == None or not self.should_process, self.parsed_url
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000506 parsed_url = self.get_custom_deps(self.name, url)
507 if parsed_url != url:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000508 logging.info(
509 'Dependency(%s).LateOverride(%s) -> %s' %
510 (self.name, url, parsed_url))
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000511 return parsed_url
512
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000513 if isinstance(url, basestring):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000514 parsed_url = urlparse.urlparse(url)
scr@chromium.orgf1eccaf2014-04-11 15:51:33 +0000515 if (not parsed_url[0] and
516 not re.match(r'^\w+\@[\w\.-]+\:[\w\/]+', parsed_url[2])):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000517 # A relative url. Fetch the real base.
518 path = parsed_url[2]
519 if not path.startswith('/'):
520 raise gclient_utils.Error(
521 'relative DEPS entry \'%s\' must begin with a slash' % url)
522 # Create a scm just to query the full url.
523 parent_url = self.parent.parsed_url
szager@chromium.orgfe0d1902014-04-08 20:50:44 +0000524 scm = gclient_scm.CreateSCM(
525 parent_url, self.root.root_dir, None, self.outbuf)
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000526 parsed_url = scm.FullUrlForRelativeUrl(url)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000527 else:
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000528 parsed_url = url
maruel@chromium.org470b5432011-10-11 18:18:19 +0000529 logging.info(
530 'Dependency(%s).LateOverride(%s) -> %s' %
531 (self.name, url, parsed_url))
maruel@chromium.orgda7a1f92010-08-10 17:19:02 +0000532 return parsed_url
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000533
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000534 if url is None:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000535 logging.info(
536 'Dependency(%s).LateOverride(%s) -> %s' % (self.name, url, url))
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000537 return url
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000538
539 raise gclient_utils.Error('Unknown url type')
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000540
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000541 @staticmethod
542 def MergeWithOsDeps(deps, deps_os, target_os_list):
543 """Returns a new "deps" structure that is the deps sent in updated
544 with information from deps_os (the deps_os section of the DEPS
545 file) that matches the list of target os."""
Andrii Shyshkalov806b7012017-06-21 11:48:10 +0000546 os_overrides = {}
547 for the_target_os in target_os_list:
548 the_target_os_deps = deps_os.get(the_target_os, {})
549 for os_dep_key, os_dep_value in the_target_os_deps.iteritems():
550 overrides = os_overrides.setdefault(os_dep_key, [])
551 overrides.append((the_target_os, os_dep_value))
552
553 # If any os didn't specify a value (we have fewer value entries
554 # than in the os list), then it wants to use the default value.
555 for os_dep_key, os_dep_value in os_overrides.iteritems():
556 if len(os_dep_value) != len(target_os_list):
557 # Record the default value too so that we don't accidentally
558 # set it to None or miss a conflicting DEPS.
559 if os_dep_key in deps:
560 os_dep_value.append(('default', deps[os_dep_key]))
561
562 target_os_deps = {}
563 for os_dep_key, os_dep_value in os_overrides.iteritems():
564 # os_dep_value is a list of (os, value) pairs.
565 possible_values = set(x[1] for x in os_dep_value if x[1] is not None)
566 if not possible_values:
567 target_os_deps[os_dep_key] = None
568 else:
569 if len(possible_values) > 1:
570 raise gclient_utils.Error(
571 'Conflicting dependencies for %s: %s. (target_os=%s)' % (
572 os_dep_key, os_dep_value, target_os_list))
573 # Sorting to get the same result every time in case of conflicts.
574 target_os_deps[os_dep_key] = sorted(possible_values)[0]
575
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000576 new_deps = deps.copy()
Andrii Shyshkalov806b7012017-06-21 11:48:10 +0000577 for key, value in target_os_deps.iteritems():
578 if key in new_deps:
579 raise gclient_utils.Error(
580 ('Value from deps_os (%r: %r) conflicts with existing deps '
581 'entry (%r).') % (key, value, new_deps[key]))
582 new_deps[key] = value
bratell@opera.comed2b4fe2013-12-16 14:34:12 +0000583 return new_deps
584
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200585 def _postprocess_deps(self, deps, rel_prefix):
586 """Performs post-processing of deps compared to what's in the DEPS file."""
Paweł Hajdan, Jr4426eaf2017-06-13 12:25:47 +0200587 # Make sure the dict is mutable, e.g. in case it's frozen.
588 deps = dict(deps)
589
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200590 # If a line is in custom_deps, but not in the solution, we want to append
591 # this line to the solution.
592 for d in self.custom_deps:
593 if d not in deps:
594 deps[d] = self.custom_deps[d]
595
596 if rel_prefix:
597 logging.warning('use_relative_paths enabled.')
598 rel_deps = {}
599 for d, url in deps.items():
600 # normpath is required to allow DEPS to use .. in their
601 # dependency local path.
602 rel_deps[os.path.normpath(os.path.join(rel_prefix, d))] = url
603 logging.warning('Updating deps by prepending %s.', rel_prefix)
604 deps = rel_deps
605
606 return deps
607
608 def _deps_to_objects(self, deps, use_relative_paths):
609 """Convert a deps dict to a dict of Dependency objects."""
610 deps_to_add = []
611 for name, dep_value in deps.iteritems():
612 should_process = self.recursion_limit and self.should_process
613 deps_file = self.deps_file
614 if self.recursedeps is not None:
615 ent = self.recursedeps.get(name)
616 if ent is not None:
617 deps_file = ent['deps_file']
618 if dep_value is None:
619 continue
620 condition = None
621 condition_value = True
622 if isinstance(dep_value, basestring):
623 url = dep_value
624 else:
625 # This should be guaranteed by schema checking in gclient_eval.
626 assert isinstance(dep_value, collections.Mapping)
627 url = dep_value['url']
628 condition = dep_value.get('condition')
629 if condition:
630 # TODO(phajdan.jr): should we also take custom vars into account?
631 condition_value = gclient_eval.EvaluateCondition(condition, self._vars)
632 should_process = should_process and condition_value
633 deps_to_add.append(Dependency(
634 self, name, url, None, None, self.custom_vars, None,
635 deps_file, should_process, use_relative_paths, condition,
636 condition_value))
637 deps_to_add.sort(key=lambda x: x.name)
638 return deps_to_add
639
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000640 def ParseDepsFile(self):
maruel@chromium.org271375b2010-06-23 19:17:38 +0000641 """Parses the DEPS file for this dependency."""
maruel@chromium.org3223edd2011-10-10 23:17:39 +0000642 assert not self.deps_parsed
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000643 assert not self.dependencies
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000644
645 deps_content = None
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000646
647 # First try to locate the configured deps file. If it's missing, fallback
648 # to DEPS.
649 deps_files = [self.deps_file]
650 if 'DEPS' not in deps_files:
651 deps_files.append('DEPS')
652 for deps_file in deps_files:
653 filepath = os.path.join(self.root.root_dir, self.name, deps_file)
654 if os.path.isfile(filepath):
655 logging.info(
656 'ParseDepsFile(%s): %s file found at %s', self.name, deps_file,
657 filepath)
658 break
maruel@chromium.org1333cb32011-10-04 23:40:16 +0000659 logging.info(
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000660 'ParseDepsFile(%s): No %s file found at %s', self.name, deps_file,
661 filepath)
662
663 if os.path.isfile(filepath):
maruel@chromium.org46304292010-10-28 11:42:00 +0000664 deps_content = gclient_utils.FileRead(filepath)
cmp@chromium.org76ce73c2014-07-02 00:13:18 +0000665 logging.debug('ParseDepsFile(%s) read:\n%s', self.name, deps_content)
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000666
667 local_scope = {}
668 if deps_content:
669 # One thing is unintuitive, vars = {} must happen before Var() use.
670 var = self.VarImpl(self.custom_vars, local_scope)
Paweł Hajdan, Jrf1587bf2017-06-20 21:19:07 +0200671 global_scope = {
672 'Var': var.Lookup,
673 'deps_os': {},
674 }
maruel@chromium.org46304292010-10-28 11:42:00 +0000675 # Eval the content.
676 try:
Paweł Hajdan, Jrc485d5a2017-06-02 12:08:09 +0200677 if self._get_option('validate_syntax', False):
678 gclient_eval.Exec(deps_content, global_scope, local_scope, filepath)
679 else:
680 exec(deps_content, global_scope, local_scope)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +0000681 except SyntaxError as e:
maruel@chromium.org46304292010-10-28 11:42:00 +0000682 gclient_utils.SyntaxErrorToError(filepath, e)
iannucci@chromium.org0c3f3052014-05-22 00:29:22 +0000683
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000684 if 'allowed_hosts' in local_scope:
685 try:
686 self._allowed_hosts = frozenset(local_scope.get('allowed_hosts'))
687 except TypeError: # raised if non-iterable
688 pass
689 if not self._allowed_hosts:
690 logging.warning("allowed_hosts is specified but empty %s",
691 self._allowed_hosts)
692 raise gclient_utils.Error(
693 'ParseDepsFile(%s): allowed_hosts must be absent '
694 'or a non-empty iterable' % self.name)
695
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200696 self._gn_args_file = local_scope.get('gclient_gn_args_file')
697 self._gn_args = local_scope.get('gclient_gn_args', [])
698
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200699 # Since we heavily post-process things, freeze ones which should
700 # reflect original state of DEPS.
701 self._vars = gclient_utils.freeze(local_scope.get('vars', {}))
702
703 # If use_relative_paths is set in the DEPS file, regenerate
704 # the dictionary using paths relative to the directory containing
705 # the DEPS file. Also update recursedeps if use_relative_paths is
706 # enabled.
707 # If the deps file doesn't set use_relative_paths, but the parent did
708 # (and therefore set self.relative on this Dependency object), then we
709 # want to modify the deps and recursedeps by prepending the parent
710 # directory of this dependency.
711 use_relative_paths = local_scope.get('use_relative_paths', False)
712 rel_prefix = None
713 if use_relative_paths:
714 rel_prefix = self.name
715 elif self._relative:
716 rel_prefix = os.path.dirname(self.name)
717
718 deps = local_scope.get('deps', {})
719 orig_deps = gclient_utils.freeze(deps)
720 if 'recursion' in local_scope:
721 self.recursion_override = local_scope.get('recursion')
722 logging.warning(
723 'Setting %s recursion to %d.', self.name, self.recursion_limit)
724 self.recursedeps = None
725 if 'recursedeps' in local_scope:
726 self.recursedeps = {}
727 for ent in local_scope['recursedeps']:
728 if isinstance(ent, basestring):
729 self.recursedeps[ent] = {"deps_file": self.deps_file}
730 else: # (depname, depsfilename)
731 self.recursedeps[ent[0]] = {"deps_file": ent[1]}
732 logging.warning('Found recursedeps %r.', repr(self.recursedeps))
733
734 if rel_prefix:
735 logging.warning('Updating recursedeps by prepending %s.', rel_prefix)
736 rel_deps = {}
737 for depname, options in self.recursedeps.iteritems():
738 rel_deps[
739 os.path.normpath(os.path.join(rel_prefix, depname))] = options
740 self.recursedeps = rel_deps
741
742 # If present, save 'target_os' in the local_target_os property.
743 if 'target_os' in local_scope:
744 self.local_target_os = local_scope['target_os']
745 # load os specific dependencies if defined. these dependencies may
746 # override or extend the values defined by the 'deps' member.
747 target_os_list = self.target_os
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +0200748 if 'deps_os' in local_scope:
749 for dep_os, os_deps in local_scope['deps_os'].iteritems():
750 self._os_dependencies[dep_os] = self._deps_to_objects(
751 self._postprocess_deps(os_deps, rel_prefix), use_relative_paths)
752 if target_os_list:
753 deps = self.MergeWithOsDeps(
754 deps, local_scope['deps_os'], target_os_list)
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200755
756 deps_to_add = self._deps_to_objects(
757 self._postprocess_deps(deps, rel_prefix), use_relative_paths)
758 orig_deps_to_add = self._deps_to_objects(
759 self._postprocess_deps(orig_deps, rel_prefix), use_relative_paths)
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000760
761 # override named sets of hooks by the custom hooks
762 hooks_to_run = []
763 hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks]
764 for hook in local_scope.get('hooks', []):
765 if hook.get('name', '') not in hook_names_to_suppress:
766 hooks_to_run.append(hook)
Scott Grahamc4826742017-05-11 16:59:23 -0700767 if 'hooks_os' in local_scope and target_os_list:
768 hooks_os = local_scope['hooks_os']
769 # Specifically append these to ensure that hooks_os run after hooks.
770 for the_target_os in target_os_list:
771 the_target_os_hooks = hooks_os.get(the_target_os, [])
772 hooks_to_run.extend(the_target_os_hooks)
petermayo@chromium.orge79161a2013-07-09 14:40:37 +0000773
774 # add the replacements and any additions
775 for hook in self.custom_hooks:
776 if 'action' in hook:
777 hooks_to_run.append(hook)
778
Dirk Prankeda3a29e2017-02-27 15:29:36 -0800779 if self.recursion_limit:
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200780 self._pre_deps_hooks = [Hook.from_dict(hook) for hook in
Dirk Prankeda3a29e2017-02-27 15:29:36 -0800781 local_scope.get('pre_deps_hooks', [])]
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000782
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200783 self.add_dependencies_and_close(
784 deps_to_add, hooks_to_run, orig_deps_to_add=orig_deps_to_add)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000785 logging.info('ParseDepsFile(%s) done' % self.name)
786
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +0200787 def _get_option(self, attr, default):
788 obj = self
789 while not hasattr(obj, '_options'):
790 obj = obj.parent
791 return getattr(obj._options, attr, default)
792
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200793 def add_dependencies_and_close(
794 self, deps_to_add, hooks, orig_deps_to_add=None):
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000795 """Adds the dependencies, hooks and mark the parsing as done."""
maruel@chromium.orgb9be0652011-10-14 18:05:40 +0000796 for dep in deps_to_add:
maruel@chromium.org470b5432011-10-11 18:18:19 +0000797 if dep.verify_validity():
maruel@chromium.org0bcfd182011-10-10 20:06:09 +0000798 self.add_dependency(dep)
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +0200799 for dep in (orig_deps_to_add or deps_to_add):
800 self.add_orig_dependency(dep)
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200801 self._mark_as_parsed([Hook.from_dict(h) for h in hooks])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000802
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000803 def findDepsFromNotAllowedHosts(self):
804 """Returns a list of depenecies from not allowed hosts.
805
806 If allowed_hosts is not set, allows all hosts and returns empty list.
807 """
808 if not self._allowed_hosts:
809 return []
810 bad_deps = []
811 for dep in self._dependencies:
szager@chromium.orgbd772dd2014-11-05 18:43:08 +0000812 # Don't enforce this for custom_deps.
813 if dep.name in self._custom_deps:
814 continue
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +0000815 if isinstance(dep.url, basestring):
816 parsed_url = urlparse.urlparse(dep.url)
817 if parsed_url.netloc and parsed_url.netloc not in self._allowed_hosts:
818 bad_deps.append(dep)
819 return bad_deps
820
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000821 # Arguments number differs from overridden method
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800822 # pylint: disable=arguments-differ
maruel@chromium.org3742c842010-09-09 19:27:14 +0000823 def run(self, revision_overrides, command, args, work_queue, options):
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000824 """Runs |command| then parse the DEPS file."""
maruel@chromium.org470b5432011-10-11 18:18:19 +0000825 logging.info('Dependency(%s).run()' % self.name)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +0000826 assert self._file_list == []
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000827 if not self.should_process:
828 return
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000829 # When running runhooks, there's no need to consult the SCM.
830 # All known hooks are expected to run unconditionally regardless of working
831 # copy state, so skip the SCM status check.
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +0200832 run_scm = command not in (
833 'flatten', 'runhooks', 'recurse', 'validate', None)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000834 parsed_url = self.LateOverride(self.url)
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000835 file_list = [] if not options.nohooks else None
szager@chromium.org3a3608d2014-10-22 21:13:52 +0000836 revision_override = revision_overrides.pop(self.name, None)
Dave Tubbda9712017-06-01 15:10:53 -0700837 if not revision_override and parsed_url:
838 revision_override = revision_overrides.get(parsed_url.split('@')[0], None)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000839 if run_scm and parsed_url:
agabled437d762016-10-17 09:35:11 -0700840 # Create a shallow copy to mutate revision.
841 options = copy.copy(options)
842 options.revision = revision_override
843 self._used_revision = options.revision
844 self._used_scm = gclient_scm.CreateSCM(
845 parsed_url, self.root.root_dir, self.name, self.outbuf,
846 out_cb=work_queue.out_cb)
847 self._got_revision = self._used_scm.RunCommand(command, options, args,
848 file_list)
849 if file_list:
850 file_list = [os.path.join(self.name, f.strip()) for f in file_list]
maruel@chromium.org68988972011-09-20 14:11:42 +0000851
852 # TODO(phajdan.jr): We should know exactly when the paths are absolute.
853 # Convert all absolute paths to relative.
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000854 for i in range(len(file_list or [])):
maruel@chromium.org68988972011-09-20 14:11:42 +0000855 # It depends on the command being executed (like runhooks vs sync).
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000856 if not os.path.isabs(file_list[i]):
maruel@chromium.org68988972011-09-20 14:11:42 +0000857 continue
858 prefix = os.path.commonprefix(
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000859 [self.root.root_dir.lower(), file_list[i].lower()])
860 file_list[i] = file_list[i][len(prefix):]
maruel@chromium.org68988972011-09-20 14:11:42 +0000861 # Strip any leading path separators.
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000862 while file_list[i].startswith(('\\', '/')):
863 file_list[i] = file_list[i][1:]
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000864
865 # Always parse the DEPS file.
866 self.ParseDepsFile()
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200867 if self._gn_args_file and command == 'update':
868 self.WriteGNArgsFile()
iannucci@chromium.org396e1a62013-07-03 19:41:04 +0000869 self._run_is_done(file_list or [], parsed_url)
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000870 if command in ('update', 'revert') and not options.noprehooks:
871 self.RunPreDepsHooks()
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000872
873 if self.recursion_limit:
874 # Parse the dependencies of this dependency.
875 for s in self.dependencies:
876 work_queue.enqueue(s)
877
878 if command == 'recurse':
agabled437d762016-10-17 09:35:11 -0700879 # Skip file only checkout.
880 scm = gclient_scm.GetScmName(parsed_url)
881 if not options.scm or scm in options.scm:
882 cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
883 # Pass in the SCM type as an env variable. Make sure we don't put
884 # unicode strings in the environment.
885 env = os.environ.copy()
886 if scm:
887 env['GCLIENT_SCM'] = str(scm)
888 if parsed_url:
889 env['GCLIENT_URL'] = str(parsed_url)
890 env['GCLIENT_DEP_PATH'] = str(self.name)
891 if options.prepend_dir and scm == 'git':
892 print_stdout = False
893 def filter_fn(line):
894 """Git-specific path marshaling. It is optimized for git-grep."""
ilevy@chromium.org0233ac22012-11-28 20:27:02 +0000895
agabled437d762016-10-17 09:35:11 -0700896 def mod_path(git_pathspec):
897 match = re.match('^(\\S+?:)?([^\0]+)$', git_pathspec)
898 modified_path = os.path.join(self.name, match.group(2))
899 branch = match.group(1) or ''
900 return '%s%s' % (branch, modified_path)
ilevy@chromium.org0233ac22012-11-28 20:27:02 +0000901
agabled437d762016-10-17 09:35:11 -0700902 match = re.match('^Binary file ([^\0]+) matches$', line)
903 if match:
904 print('Binary file %s matches\n' % mod_path(match.group(1)))
905 return
ilevy@chromium.org0233ac22012-11-28 20:27:02 +0000906
agabled437d762016-10-17 09:35:11 -0700907 items = line.split('\0')
908 if len(items) == 2 and items[1]:
909 print('%s : %s' % (mod_path(items[0]), items[1]))
910 elif len(items) >= 2:
911 # Multiple null bytes or a single trailing null byte indicate
912 # git is likely displaying filenames only (such as with -l)
913 print('\n'.join(mod_path(path) for path in items if path))
914 else:
915 print(line)
916 else:
917 print_stdout = True
918 filter_fn = None
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +0000919
agabled437d762016-10-17 09:35:11 -0700920 if parsed_url is None:
921 print('Skipped omitted dependency %s' % cwd, file=sys.stderr)
922 elif os.path.isdir(cwd):
923 try:
924 gclient_utils.CheckCallAndFilter(
925 args, cwd=cwd, env=env, print_stdout=print_stdout,
926 filter_fn=filter_fn,
927 )
928 except subprocess2.CalledProcessError:
929 if not options.ignore:
930 raise
931 else:
932 print('Skipped missing %s' % cwd, file=sys.stderr)
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000933
Paweł Hajdan, Jr57253732017-06-06 23:49:11 +0200934 def WriteGNArgsFile(self):
935 lines = ['# Generated from %r' % self.deps_file]
936 for arg in self._gn_args:
937 lines.append('%s = %s' % (arg, ToGNString(self._vars[arg])))
938 with open(os.path.join(self.root.root_dir, self._gn_args_file), 'w') as f:
939 f.write('\n'.join(lines))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000940
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +0000941 @gclient_utils.lockedmethod
942 def _run_is_done(self, file_list, parsed_url):
943 # Both these are kept for hooks that are run as a separate tree traversal.
944 self._file_list = file_list
945 self._parsed_url = parsed_url
946 self._processed = True
947
szager@google.comb9a78d32012-03-13 18:46:21 +0000948 def GetHooks(self, options):
949 """Evaluates all hooks, and return them in a flat list.
950
951 RunOnDeps() must have been called before to load the DEPS.
952 """
953 result = []
maruel@chromium.org68988972011-09-20 14:11:42 +0000954 if not self.should_process or not self.recursion_limit:
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000955 # Don't run the hook when it is above recursion_limit.
szager@google.comb9a78d32012-03-13 18:46:21 +0000956 return result
maruel@chromium.orgdc7445d2010-07-09 21:05:29 +0000957 # If "--force" was specified, run all hooks regardless of what files have
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000958 # changed.
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000959 if self.deps_hooks:
agabled437d762016-10-17 09:35:11 -0700960 # TODO(maruel): If the user is using git, then we don't know
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000961 # what files have changed so we always run all hooks. It'd be nice to fix
962 # that.
963 if (options.force or
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000964 gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +0000965 os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200966 result.extend(self.deps_hooks)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +0000967 else:
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200968 for hook in self.deps_hooks:
969 if hook.matches(self.file_list_and_children):
970 result.append(hook)
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000971 for s in self.dependencies:
szager@google.comb9a78d32012-03-13 18:46:21 +0000972 result.extend(s.GetHooks(options))
973 return result
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000974
szager@google.comb9a78d32012-03-13 18:46:21 +0000975 def RunHooksRecursively(self, options):
976 assert self.hooks_ran == False
maruel@chromium.org064186c2011-09-27 23:53:33 +0000977 self._hooks_ran = True
szager@google.comb9a78d32012-03-13 18:46:21 +0000978 for hook in self.GetHooks(options):
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200979 hook.run(self.root.root_dir)
maruel@chromium.orgeaf61062010-07-07 18:42:39 +0000980
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000981 def RunPreDepsHooks(self):
982 assert self.processed
983 assert self.deps_parsed
984 assert not self.pre_deps_hooks_ran
985 assert not self.hooks_ran
986 for s in self.dependencies:
987 assert not s.processed
988 self._pre_deps_hooks_ran = True
989 for hook in self.pre_deps_hooks:
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +0200990 hook.run(self.root.root_dir)
borenet@google.com2d1ee9e2013-10-15 08:13:16 +0000991
992
maruel@chromium.org0d812442010-08-10 12:41:08 +0000993 def subtree(self, include_all):
maruel@chromium.orgad3287e2011-10-03 19:15:10 +0000994 """Breadth first recursion excluding root node."""
maruel@chromium.orgf13a4182011-09-22 00:26:15 +0000995 dependencies = self.dependencies
996 for d in dependencies:
maruel@chromium.orgf50907b2010-08-12 17:05:48 +0000997 if d.should_process or include_all:
maruel@chromium.orgad3287e2011-10-03 19:15:10 +0000998 yield d
maruel@chromium.orgf13a4182011-09-22 00:26:15 +0000999 for d in dependencies:
maruel@chromium.orgad3287e2011-10-03 19:15:10 +00001000 for i in d.subtree(include_all):
1001 yield i
1002
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001003 @gclient_utils.lockedmethod
1004 def add_dependency(self, new_dep):
1005 self._dependencies.append(new_dep)
1006
1007 @gclient_utils.lockedmethod
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +02001008 def add_orig_dependency(self, new_dep):
1009 self._orig_dependencies.append(new_dep)
1010
1011 @gclient_utils.lockedmethod
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001012 def _mark_as_parsed(self, new_hooks):
1013 self._deps_hooks.extend(new_hooks)
1014 self._deps_parsed = True
1015
maruel@chromium.org68988972011-09-20 14:11:42 +00001016 @property
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001017 @gclient_utils.lockedmethod
maruel@chromium.org4bdd5fd2011-09-26 19:41:17 +00001018 def dependencies(self):
1019 return tuple(self._dependencies)
1020
1021 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001022 @gclient_utils.lockedmethod
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +02001023 def orig_dependencies(self):
1024 return tuple(self._orig_dependencies)
1025
1026 @property
1027 @gclient_utils.lockedmethod
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001028 def os_dependencies(self):
1029 return dict(self._os_dependencies)
1030
1031 @property
1032 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001033 def deps_hooks(self):
1034 return tuple(self._deps_hooks)
1035
1036 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001037 @gclient_utils.lockedmethod
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001038 def pre_deps_hooks(self):
1039 return tuple(self._pre_deps_hooks)
1040
1041 @property
1042 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001043 def parsed_url(self):
1044 return self._parsed_url
1045
1046 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001047 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001048 def deps_parsed(self):
maruel@chromium.org3223edd2011-10-10 23:17:39 +00001049 """This is purely for debugging purposes. It's not used anywhere."""
maruel@chromium.org064186c2011-09-27 23:53:33 +00001050 return self._deps_parsed
1051
1052 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001053 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001054 def processed(self):
1055 return self._processed
1056
1057 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001058 @gclient_utils.lockedmethod
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00001059 def pre_deps_hooks_ran(self):
1060 return self._pre_deps_hooks_ran
1061
1062 @property
1063 @gclient_utils.lockedmethod
maruel@chromium.org064186c2011-09-27 23:53:33 +00001064 def hooks_ran(self):
1065 return self._hooks_ran
1066
1067 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001068 @gclient_utils.lockedmethod
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00001069 def allowed_hosts(self):
1070 return self._allowed_hosts
1071
1072 @property
1073 @gclient_utils.lockedmethod
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001074 def file_list(self):
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001075 return tuple(self._file_list)
1076
1077 @property
kustermann@google.coma692e8f2013-04-18 08:32:04 +00001078 def used_scm(self):
1079 """SCMWrapper instance for this dependency or None if not processed yet."""
1080 return self._used_scm
1081
1082 @property
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00001083 @gclient_utils.lockedmethod
1084 def got_revision(self):
1085 return self._got_revision
1086
1087 @property
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001088 def file_list_and_children(self):
1089 result = list(self.file_list)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001090 for d in self.dependencies:
maruel@chromium.orgbaa7be32011-10-10 20:49:47 +00001091 result.extend(d.file_list_and_children)
maruel@chromium.org68988972011-09-20 14:11:42 +00001092 return tuple(result)
maruel@chromium.org861fd0f2010-07-23 03:05:05 +00001093
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001094 def __str__(self):
1095 out = []
agablea98a6cd2016-11-15 14:30:10 -08001096 for i in ('name', 'url', 'parsed_url', 'custom_deps',
maruel@chromium.org3c74bc92011-09-15 19:17:21 +00001097 'custom_vars', 'deps_hooks', 'file_list', 'should_process',
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00001098 'processed', 'hooks_ran', 'deps_parsed', 'requirements',
1099 'allowed_hosts'):
maruel@chromium.org3c74bc92011-09-15 19:17:21 +00001100 # First try the native property if it exists.
1101 if hasattr(self, '_' + i):
1102 value = getattr(self, '_' + i, False)
1103 else:
1104 value = getattr(self, i, False)
1105 if value:
1106 out.append('%s: %s' % (i, value))
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001107
1108 for d in self.dependencies:
1109 out.extend([' ' + x for x in str(d).splitlines()])
1110 out.append('')
1111 return '\n'.join(out)
1112
1113 def __repr__(self):
1114 return '%s: %s' % (self.name, self.url)
1115
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001116 def hierarchy(self, include_url=True):
maruel@chromium.orgbc2d2f92010-07-22 21:26:48 +00001117 """Returns a human-readable hierarchical reference to a Dependency."""
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001118 def format_name(d):
1119 if include_url:
1120 return '%s(%s)' % (d.name, d.url)
1121 return d.name
1122 out = format_name(self)
maruel@chromium.orgbffb9042010-07-22 20:59:36 +00001123 i = self.parent
1124 while i and i.name:
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001125 out = '%s -> %s' % (format_name(i), out)
maruel@chromium.orgbffb9042010-07-22 20:59:36 +00001126 i = i.parent
1127 return out
1128
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001129
1130class GClient(Dependency):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001131 """Object that represent a gclient checkout. A tree of Dependency(), one per
1132 solution or DEPS entry."""
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001133
1134 DEPS_OS_CHOICES = {
1135 "win32": "win",
1136 "win": "win",
1137 "cygwin": "win",
1138 "darwin": "mac",
1139 "mac": "mac",
1140 "unix": "unix",
1141 "linux": "unix",
1142 "linux2": "unix",
maruel@chromium.org244e3442011-06-12 15:20:55 +00001143 "linux3": "unix",
szager@chromium.orgf8c95cd2012-06-01 22:26:52 +00001144 "android": "android",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001145 }
1146
1147 DEFAULT_CLIENT_FILE_TEXT = ("""\
1148solutions = [
smutae7ea312016-07-18 11:59:41 -07001149 { "name" : "%(solution_name)s",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001150 "url" : "%(solution_url)s",
nsylvain@google.comefc80932011-05-31 21:27:56 +00001151 "deps_file" : "%(deps_file)s",
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001152 "managed" : %(managed)s,
smutae7ea312016-07-18 11:59:41 -07001153 "custom_deps" : {
1154 },
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001155 },
1156]
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001157cache_dir = %(cache_dir)r
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001158""")
1159
1160 DEFAULT_SNAPSHOT_SOLUTION_TEXT = ("""\
smutae7ea312016-07-18 11:59:41 -07001161 { "name" : "%(solution_name)s",
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001162 "url" : "%(solution_url)s",
nsylvain@google.comefc80932011-05-31 21:27:56 +00001163 "deps_file" : "%(deps_file)s",
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001164 "managed" : %(managed)s,
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001165 "custom_deps" : {
smutae7ea312016-07-18 11:59:41 -07001166%(solution_deps)s },
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001167 },
1168""")
1169
1170 DEFAULT_SNAPSHOT_FILE_TEXT = ("""\
1171# Snapshot generated with gclient revinfo --snapshot
1172solutions = [
maruel@chromium.org73e21142010-07-05 13:32:01 +00001173%(solution_list)s]
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001174""")
1175
1176 def __init__(self, root_dir, options):
maruel@chromium.org0d812442010-08-10 12:41:08 +00001177 # Do not change previous behavior. Only solution level and immediate DEPS
1178 # are processed.
1179 self._recursion_limit = 2
agablea98a6cd2016-11-15 14:30:10 -08001180 Dependency.__init__(self, None, None, None, True, None, None, None,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001181 'unused', True, None, None, True)
maruel@chromium.org0d425922010-06-21 19:22:24 +00001182 self._options = options
maruel@chromium.org271375b2010-06-23 19:17:38 +00001183 if options.deps_os:
1184 enforced_os = options.deps_os.split(',')
1185 else:
1186 enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')]
1187 if 'all' in enforced_os:
1188 enforced_os = self.DEPS_OS_CHOICES.itervalues()
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001189 self._enforced_os = tuple(set(enforced_os))
maruel@chromium.org271375b2010-06-23 19:17:38 +00001190 self._root_dir = root_dir
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001191 self.config_content = None
1192
borenet@google.com88d10082014-03-21 17:24:48 +00001193 def _CheckConfig(self):
1194 """Verify that the config matches the state of the existing checked-out
1195 solutions."""
1196 for dep in self.dependencies:
1197 if dep.managed and dep.url:
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001198 scm = gclient_scm.CreateSCM(
1199 dep.url, self.root_dir, dep.name, self.outbuf)
smut@google.comd33eab32014-07-07 19:35:18 +00001200 actual_url = scm.GetActualRemoteURL(self._options)
borenet@google.com4e9be262014-04-08 19:40:30 +00001201 if actual_url and not scm.DoesRemoteURLMatch(self._options):
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001202 mirror = scm.GetCacheMirror()
1203 if mirror:
1204 mirror_string = '%s (exists=%s)' % (mirror.mirror_path,
1205 mirror.exists())
1206 else:
1207 mirror_string = 'not used'
borenet@google.com0a427372014-04-02 19:12:13 +00001208 raise gclient_utils.Error('''
borenet@google.com88d10082014-03-21 17:24:48 +00001209Your .gclient file seems to be broken. The requested URL is different from what
borenet@google.com0a427372014-04-02 19:12:13 +00001210is actually checked out in %(checkout_path)s.
borenet@google.com88d10082014-03-21 17:24:48 +00001211
borenet@google.com97882362014-04-07 20:06:02 +00001212The .gclient file contains:
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001213URL: %(expected_url)s (%(expected_scm)s)
1214Cache mirror: %(mirror_string)s
borenet@google.com97882362014-04-07 20:06:02 +00001215
1216The local checkout in %(checkout_path)s reports:
1217%(actual_url)s (%(actual_scm)s)
borenet@google.com88d10082014-03-21 17:24:48 +00001218
1219You should ensure that the URL listed in .gclient is correct and either change
agabled437d762016-10-17 09:35:11 -07001220it or fix the checkout.
borenet@google.com88d10082014-03-21 17:24:48 +00001221''' % {'checkout_path': os.path.join(self.root_dir, dep.name),
1222 'expected_url': dep.url,
1223 'expected_scm': gclient_scm.GetScmName(dep.url),
levarum@chromium.org27a6f9a2016-05-28 00:21:49 +00001224 'mirror_string' : mirror_string,
borenet@google.com88d10082014-03-21 17:24:48 +00001225 'actual_url': actual_url,
1226 'actual_scm': gclient_scm.GetScmName(actual_url)})
1227
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001228 def SetConfig(self, content):
maruel@chromium.orgf13a4182011-09-22 00:26:15 +00001229 assert not self.dependencies
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001230 config_dict = {}
1231 self.config_content = content
1232 try:
1233 exec(content, config_dict)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00001234 except SyntaxError as e:
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001235 gclient_utils.SyntaxErrorToError('.gclient', e)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001236
peter@chromium.org1efccc82012-04-27 16:34:38 +00001237 # Append any target OS that is not already being enforced to the tuple.
1238 target_os = config_dict.get('target_os', [])
stuartmorgan@chromium.org18a4f6a2012-11-02 13:37:24 +00001239 if config_dict.get('target_os_only', False):
1240 self._enforced_os = tuple(set(target_os))
1241 else:
1242 self._enforced_os = tuple(set(self._enforced_os).union(target_os))
1243
dyen@chromium.orgd915cca2014-08-07 21:41:37 +00001244 cache_dir = config_dict.get('cache_dir')
1245 if cache_dir:
1246 cache_dir = os.path.join(self.root_dir, cache_dir)
1247 cache_dir = os.path.abspath(cache_dir)
szager@chromium.orgcaf5bef2014-08-24 18:56:32 +00001248 # If running on a bot, force break any stale git cache locks.
dnj@chromium.orgb682b3e2014-08-25 19:17:12 +00001249 if os.path.exists(cache_dir) and os.environ.get('CHROME_HEADLESS'):
szager@chromium.org4848fb62014-08-24 19:16:31 +00001250 subprocess2.check_call(['git', 'cache', 'unlock', '--cache-dir',
1251 cache_dir, '--force', '--all'])
dyen@chromium.orgd915cca2014-08-07 21:41:37 +00001252 gclient_scm.GitWrapper.cache_dir = cache_dir
1253 git_cache.Mirror.SetCachePath(cache_dir)
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001254
stuartmorgan@chromium.org18a4f6a2012-11-02 13:37:24 +00001255 if not target_os and config_dict.get('target_os_only', False):
1256 raise gclient_utils.Error('Can\'t use target_os_only if target_os is '
1257 'not specified')
peter@chromium.org1efccc82012-04-27 16:34:38 +00001258
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001259 deps_to_add = []
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001260 for s in config_dict.get('solutions', []):
maruel@chromium.org81843b82010-06-28 16:49:26 +00001261 try:
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001262 deps_to_add.append(Dependency(
maruel@chromium.org81843b82010-06-28 16:49:26 +00001263 self, s['name'], s['url'],
smutae7ea312016-07-18 11:59:41 -07001264 s.get('managed', True),
maruel@chromium.org81843b82010-06-28 16:49:26 +00001265 s.get('custom_deps', {}),
maruel@chromium.org0d812442010-08-10 12:41:08 +00001266 s.get('custom_vars', {}),
petermayo@chromium.orge79161a2013-07-09 14:40:37 +00001267 s.get('custom_hooks', []),
nsylvain@google.comefc80932011-05-31 21:27:56 +00001268 s.get('deps_file', 'DEPS'),
agabledce6ddc2016-09-08 10:02:16 -07001269 True,
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001270 None,
1271 None,
1272 True))
maruel@chromium.org81843b82010-06-28 16:49:26 +00001273 except KeyError:
1274 raise gclient_utils.Error('Invalid .gclient file. Solution is '
1275 'incomplete: %s' % s)
maruel@chromium.org0bcfd182011-10-10 20:06:09 +00001276 self.add_dependencies_and_close(deps_to_add, config_dict.get('hooks', []))
1277 logging.info('SetConfig() done')
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001278
1279 def SaveConfig(self):
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001280 gclient_utils.FileWrite(os.path.join(self.root_dir,
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001281 self._options.config_filename),
1282 self.config_content)
1283
1284 @staticmethod
1285 def LoadCurrentConfig(options):
1286 """Searches for and loads a .gclient file relative to the current working
1287 dir. Returns a GClient object."""
szager@chromium.orge2e03202012-07-31 18:05:16 +00001288 if options.spec:
1289 client = GClient('.', options)
1290 client.SetConfig(options.spec)
1291 else:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00001292 if options.verbose:
1293 print('Looking for %s starting from %s\n' % (
1294 options.config_filename, os.getcwd()))
szager@chromium.orge2e03202012-07-31 18:05:16 +00001295 path = gclient_utils.FindGclientRoot(os.getcwd(), options.config_filename)
1296 if not path:
1297 return None
1298 client = GClient(path, options)
1299 client.SetConfig(gclient_utils.FileRead(
1300 os.path.join(path, options.config_filename)))
maruel@chromium.org69392e72011-10-13 22:09:00 +00001301
1302 if (options.revisions and
1303 len(client.dependencies) > 1 and
1304 any('@' not in r for r in options.revisions)):
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001305 print(
1306 ('You must specify the full solution name like --revision %s@%s\n'
1307 'when you have multiple solutions setup in your .gclient file.\n'
1308 'Other solutions present are: %s.') % (
maruel@chromium.org69392e72011-10-13 22:09:00 +00001309 client.dependencies[0].name,
1310 options.revisions[0],
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001311 ', '.join(s.name for s in client.dependencies[1:])),
1312 file=sys.stderr)
maruel@chromium.org15804092010-09-02 17:07:37 +00001313 return client
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001314
nsylvain@google.comefc80932011-05-31 21:27:56 +00001315 def SetDefaultConfig(self, solution_name, deps_file, solution_url,
agablea98a6cd2016-11-15 14:30:10 -08001316 managed=True, cache_dir=None):
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001317 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
1318 'solution_name': solution_name,
1319 'solution_url': solution_url,
nsylvain@google.comefc80932011-05-31 21:27:56 +00001320 'deps_file': deps_file,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001321 'managed': managed,
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001322 'cache_dir': cache_dir,
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001323 })
1324
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001325 def _SaveEntries(self):
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001326 """Creates a .gclient_entries file to record the list of unique checkouts.
1327
1328 The .gclient_entries file lives in the same directory as .gclient.
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001329 """
1330 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
1331 # makes testing a bit too fun.
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001332 result = 'entries = {\n'
maruel@chromium.org68988972011-09-20 14:11:42 +00001333 for entry in self.root.subtree(False):
agabled437d762016-10-17 09:35:11 -07001334 result += ' %s: %s,\n' % (pprint.pformat(entry.name),
1335 pprint.pformat(entry.parsed_url))
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001336 result += '}\n'
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001337 file_path = os.path.join(self.root_dir, self._options.entries_filename)
maruel@chromium.org1333cb32011-10-04 23:40:16 +00001338 logging.debug(result)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001339 gclient_utils.FileWrite(file_path, result)
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001340
1341 def _ReadEntries(self):
1342 """Read the .gclient_entries file for the given client.
1343
1344 Returns:
1345 A sequence of solution names, which will be empty if there is the
1346 entries file hasn't been created yet.
1347 """
1348 scope = {}
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001349 filename = os.path.join(self.root_dir, self._options.entries_filename)
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001350 if not os.path.exists(filename):
maruel@chromium.org73e21142010-07-05 13:32:01 +00001351 return {}
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001352 try:
1353 exec(gclient_utils.FileRead(filename), scope)
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00001354 except SyntaxError as e:
maruel@chromium.org5990f9d2010-07-07 18:02:58 +00001355 gclient_utils.SyntaxErrorToError(filename, e)
Aaron Gable3721ee92017-04-03 14:53:14 -07001356 return scope.get('entries', {})
maruel@chromium.org9a66ddf2010-06-16 16:54:16 +00001357
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001358 def _EnforceRevisions(self):
maruel@chromium.org918a9ae2010-05-28 15:50:30 +00001359 """Checks for revision overrides."""
1360 revision_overrides = {}
smutae7ea312016-07-18 11:59:41 -07001361 if self._options.head:
1362 return revision_overrides
joi@chromium.org792ea882010-11-10 02:37:27 +00001363 if not self._options.revisions:
1364 for s in self.dependencies:
smutae7ea312016-07-18 11:59:41 -07001365 if not s.managed:
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001366 self._options.revisions.append('%s@unmanaged' % s.name)
maruel@chromium.org307d1792010-05-31 20:03:13 +00001367 if not self._options.revisions:
1368 return revision_overrides
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001369 solutions_names = [s.name for s in self.dependencies]
smutae7ea312016-07-18 11:59:41 -07001370 index = 0
1371 for revision in self._options.revisions:
1372 if not '@' in revision:
maruel@chromium.org307d1792010-05-31 20:03:13 +00001373 # Support for --revision 123
smutae7ea312016-07-18 11:59:41 -07001374 revision = '%s@%s' % (solutions_names[index], revision)
1375 name, rev = revision.split('@', 1)
szager@chromium.org4ad264b2014-05-20 04:43:47 +00001376 revision_overrides[name] = rev
smutae7ea312016-07-18 11:59:41 -07001377 index += 1
maruel@chromium.org918a9ae2010-05-28 15:50:30 +00001378 return revision_overrides
1379
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001380 def RunOnDeps(self, command, args, ignore_requirements=False, progress=True):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001381 """Runs a command on each dependency in a client and its dependencies.
1382
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001383 Args:
1384 command: The command to use (e.g., 'status' or 'diff')
1385 args: list of str - extra arguments to add to the command line.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001386 """
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001387 if not self.dependencies:
maruel@chromium.org73e21142010-07-05 13:32:01 +00001388 raise gclient_utils.Error('No solution specified')
borenet@google.com0a427372014-04-02 19:12:13 +00001389
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001390 revision_overrides = {}
1391 # It's unnecessary to check for revision overrides for 'recurse'.
1392 # Save a few seconds by not calling _EnforceRevisions() in that case.
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02001393 if command not in ('diff', 'recurse', 'runhooks', 'status', 'revert',
1394 'validate'):
szager@chromium.org5273b8a2014-08-21 15:10:10 +00001395 self._CheckConfig()
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001396 revision_overrides = self._EnforceRevisions()
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001397 pm = None
maruel@chromium.org5b3f8852010-09-10 16:49:54 +00001398 # Disable progress for non-tty stdout.
iannucci@chromium.org596cd5c2016-04-04 21:34:39 +00001399 if (setup_color.IS_TTY and not self._options.verbose and progress):
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001400 if command in ('update', 'revert'):
1401 pm = Progress('Syncing projects', 1)
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02001402 elif command in ('recurse', 'validate'):
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001403 pm = Progress(' '.join(args), 1)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001404 work_queue = gclient_utils.ExecutionQueue(
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001405 self._options.jobs, pm, ignore_requirements=ignore_requirements,
1406 verbose=self._options.verbose)
maruel@chromium.org049bced2010-08-12 13:37:20 +00001407 for s in self.dependencies:
1408 work_queue.enqueue(s)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001409 work_queue.flush(revision_overrides, command, args, options=self._options)
szager@chromium.org4ad264b2014-05-20 04:43:47 +00001410 if revision_overrides:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001411 print('Please fix your script, having invalid --revision flags will soon '
1412 'considered an error.', file=sys.stderr)
piman@chromium.org6f363722010-04-27 00:41:09 +00001413
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001414 # Once all the dependencies have been processed, it's now safe to run the
1415 # hooks.
1416 if not self._options.nohooks:
1417 self.RunHooksRecursively(self._options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001418
1419 if command == 'update':
ajwong@chromium.orgcdcee802009-06-23 15:30:42 +00001420 # Notify the user if there is an orphaned entry in their working copy.
1421 # Only delete the directory if there are no changes in it, and
1422 # delete_unversioned_trees is set to true.
maruel@chromium.org68988972011-09-20 14:11:42 +00001423 entries = [i.name for i in self.root.subtree(False) if i.url]
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001424 full_entries = [os.path.join(self.root_dir, e.replace('/', os.path.sep))
1425 for e in entries]
1426
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001427 for entry, prev_url in self._ReadEntries().iteritems():
maruel@chromium.org04dd7de2010-10-14 13:25:49 +00001428 if not prev_url:
1429 # entry must have been overridden via .gclient custom_deps
1430 continue
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001431 # Fix path separator on Windows.
1432 entry_fixed = entry.replace('/', os.path.sep)
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001433 e_dir = os.path.join(self.root_dir, entry_fixed)
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001434 # Use entry and not entry_fixed there.
jochen@chromium.orga78e5532013-03-11 13:33:03 +00001435 if (entry not in entries and
1436 (not any(path.startswith(entry + '/') for path in entries)) and
jochen@chromium.orgcc475722013-03-11 13:07:40 +00001437 os.path.exists(e_dir)):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001438 # The entry has been removed from DEPS.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001439 scm = gclient_scm.CreateSCM(
1440 prev_url, self.root_dir, entry_fixed, self.outbuf)
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001441
1442 # Check to see if this directory is now part of a higher-up checkout.
borenet@google.com359bb642014-05-13 17:28:19 +00001443 scm_root = None
agabled437d762016-10-17 09:35:11 -07001444 try:
1445 scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(scm.checkout_path)
1446 except subprocess2.CalledProcessError:
1447 pass
1448 if not scm_root:
borenet@google.com359bb642014-05-13 17:28:19 +00001449 logging.warning('Could not find checkout root for %s. Unable to '
1450 'determine whether it is part of a higher-level '
1451 'checkout, so not removing.' % entry)
1452 continue
primiano@chromium.org1c127382015-02-17 11:15:40 +00001453
1454 # This is to handle the case of third_party/WebKit migrating from
1455 # being a DEPS entry to being part of the main project.
1456 # If the subproject is a Git project, we need to remove its .git
1457 # folder. Otherwise git operations on that folder will have different
1458 # effects depending on the current working directory.
agabled437d762016-10-17 09:35:11 -07001459 if os.path.abspath(scm_root) == os.path.abspath(e_dir):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001460 e_par_dir = os.path.join(e_dir, os.pardir)
agabled437d762016-10-17 09:35:11 -07001461 if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir):
1462 par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(e_par_dir)
primiano@chromium.org1c127382015-02-17 11:15:40 +00001463 # rel_e_dir : relative path of entry w.r.t. its parent repo.
1464 rel_e_dir = os.path.relpath(e_dir, par_scm_root)
agabled437d762016-10-17 09:35:11 -07001465 if gclient_scm.scm.GIT.IsDirectoryVersioned(
1466 par_scm_root, rel_e_dir):
primiano@chromium.org1c127382015-02-17 11:15:40 +00001467 save_dir = scm.GetGitBackupDirPath()
1468 # Remove any eventual stale backup dir for the same project.
1469 if os.path.exists(save_dir):
1470 gclient_utils.rmtree(save_dir)
1471 os.rename(os.path.join(e_dir, '.git'), save_dir)
1472 # When switching between the two states (entry/ is a subproject
1473 # -> entry/ is part of the outer project), it is very likely
1474 # that some files are changed in the checkout, unless we are
1475 # jumping *exactly* across the commit which changed just DEPS.
1476 # In such case we want to cleanup any eventual stale files
1477 # (coming from the old subproject) in order to end up with a
1478 # clean checkout.
agabled437d762016-10-17 09:35:11 -07001479 gclient_scm.scm.GIT.CleanupDir(par_scm_root, rel_e_dir)
primiano@chromium.org1c127382015-02-17 11:15:40 +00001480 assert not os.path.exists(os.path.join(e_dir, '.git'))
1481 print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
1482 'level checkout. The git folder containing all the local'
1483 ' branches has been saved to %s.\n'
1484 'If you don\'t care about its state you can safely '
1485 'remove that folder to free up space.') %
1486 (entry, save_dir))
1487 continue
1488
borenet@google.com359bb642014-05-13 17:28:19 +00001489 if scm_root in full_entries:
primiano@chromium.org1c127382015-02-17 11:15:40 +00001490 logging.info('%s is part of a higher level checkout, not removing',
1491 scm.GetCheckoutRoot())
xusydoc@chromium.org885a9602013-05-31 09:54:40 +00001492 continue
1493
1494 file_list = []
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001495 scm.status(self._options, [], file_list)
1496 modified_files = file_list != []
maruel@chromium.org28d14bd2010-11-11 20:37:09 +00001497 if (not self._options.delete_unversioned_trees or
1498 (modified_files and not self._options.force)):
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001499 # There are modified files in this entry. Keep warning until
1500 # removed.
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001501 print(('\nWARNING: \'%s\' is no longer part of this client. '
1502 'It is recommended that you manually remove it.\n') %
maruel@chromium.orgc5e9aec2009-08-03 18:25:56 +00001503 entry_fixed)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001504 else:
1505 # Delete the entry
maruel@chromium.org73e21142010-07-05 13:32:01 +00001506 print('\n________ deleting \'%s\' in \'%s\'' % (
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001507 entry_fixed, self.root_dir))
digit@chromium.orgdc112ac2013-04-24 13:00:19 +00001508 gclient_utils.rmtree(e_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001509 # record the current list of entries for next time
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001510 self._SaveEntries()
maruel@chromium.org17cdf762010-05-28 17:30:52 +00001511 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001512
1513 def PrintRevInfo(self):
maruel@chromium.org54a07a22010-06-14 19:07:39 +00001514 if not self.dependencies:
maruel@chromium.org73e21142010-07-05 13:32:01 +00001515 raise gclient_utils.Error('No solution specified')
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001516 # Load all the settings.
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001517 work_queue = gclient_utils.ExecutionQueue(
1518 self._options.jobs, None, False, verbose=self._options.verbose)
maruel@chromium.org049bced2010-08-12 13:37:20 +00001519 for s in self.dependencies:
1520 work_queue.enqueue(s)
maruel@chromium.org3742c842010-09-09 19:27:14 +00001521 work_queue.flush({}, None, [], options=self._options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001522
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001523 def GetURLAndRev(dep):
1524 """Returns the revision-qualified SCM url for a Dependency."""
1525 if dep.parsed_url is None:
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001526 return None
agabled437d762016-10-17 09:35:11 -07001527 url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url)
szager@chromium.orgfe0d1902014-04-08 20:50:44 +00001528 scm = gclient_scm.CreateSCM(
agabled437d762016-10-17 09:35:11 -07001529 dep.parsed_url, self.root_dir, dep.name, self.outbuf)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001530 if not os.path.isdir(scm.checkout_path):
1531 return None
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001532 return '%s@%s' % (url, scm.revinfo(self._options, [], None))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001533
maruel@chromium.orgbaa578e2010-07-12 17:36:59 +00001534 if self._options.snapshot:
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001535 new_gclient = ''
1536 # First level at .gclient
1537 for d in self.dependencies:
1538 entries = {}
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001539 def GrabDeps(dep):
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001540 """Recursively grab dependencies."""
maruel@chromium.org6da25d02010-08-11 17:32:55 +00001541 for d in dep.dependencies:
1542 entries[d.name] = GetURLAndRev(d)
1543 GrabDeps(d)
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001544 GrabDeps(d)
1545 custom_deps = []
1546 for k in sorted(entries.keys()):
1547 if entries[k]:
1548 # Quotes aren't escaped...
1549 custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k]))
1550 else:
1551 custom_deps.append(' \"%s\": None,\n' % k)
1552 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % {
1553 'solution_name': d.name,
1554 'solution_url': d.url,
nsylvain@google.comefc80932011-05-31 21:27:56 +00001555 'deps_file': d.deps_file,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001556 'managed': d.managed,
maruel@chromium.orgdf2b3152010-07-21 17:35:24 +00001557 'solution_deps': ''.join(custom_deps),
1558 }
1559 # Print the snapshot configuration file
1560 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient})
nasser@codeaurora.orgde8f3522010-03-11 23:47:44 +00001561 else:
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00001562 entries = {}
maruel@chromium.org68988972011-09-20 14:11:42 +00001563 for d in self.root.subtree(False):
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00001564 if self._options.actual:
1565 entries[d.name] = GetURLAndRev(d)
1566 else:
1567 entries[d.name] = d.parsed_url
1568 keys = sorted(entries.keys())
1569 for x in keys:
maruel@chromium.orgce464892010-08-12 17:12:18 +00001570 print('%s: %s' % (x, entries[x]))
maruel@chromium.orgdde32ee2010-08-10 17:44:05 +00001571 logging.info(str(self))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001572
maruel@chromium.orgf50907b2010-08-12 17:05:48 +00001573 def ParseDepsFile(self):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001574 """No DEPS to parse for a .gclient file."""
maruel@chromium.org049bced2010-08-12 13:37:20 +00001575 raise gclient_utils.Error('Internal error')
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001576
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00001577 def PrintLocationAndContents(self):
1578 # Print out the .gclient file. This is longer than if we just printed the
1579 # client dict, but more legible, and it might contain helpful comments.
1580 print('Loaded .gclient config in %s:\n%s' % (
1581 self.root_dir, self.config_content))
1582
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001583 @property
maruel@chromium.org75a59272010-06-11 22:34:03 +00001584 def root_dir(self):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001585 """Root directory of gclient checkout."""
maruel@chromium.org75a59272010-06-11 22:34:03 +00001586 return self._root_dir
1587
maruel@chromium.orgd6db3d52011-09-20 14:00:45 +00001588 @property
maruel@chromium.org271375b2010-06-23 19:17:38 +00001589 def enforced_os(self):
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001590 """What deps_os entries that are to be parsed."""
maruel@chromium.org271375b2010-06-23 19:17:38 +00001591 return self._enforced_os
1592
maruel@chromium.org68988972011-09-20 14:11:42 +00001593 @property
maruel@chromium.orgd36fba82010-06-28 16:50:40 +00001594 def recursion_limit(self):
1595 """How recursive can each dependencies in DEPS file can load DEPS file."""
1596 return self._recursion_limit
1597
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +00001598 @property
cmp@chromium.orgc401ad12014-07-02 23:20:08 +00001599 def try_recursedeps(self):
1600 """Whether to attempt using recursedeps-style recursion processing."""
cmp@chromium.orge84ac912014-06-30 23:14:35 +00001601 return True
1602
1603 @property
sivachandra@chromium.orgd45e73e2012-10-24 23:42:48 +00001604 def target_os(self):
1605 return self._enforced_os
1606
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001607
maruel@chromium.org5ca27692010-05-26 19:32:41 +00001608#### gclient commands.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001609
1610
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001611@subcommand.usage('[command] [args ...]')
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001612def CMDrecurse(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001613 """Operates [command args ...] on all the dependencies.
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001614
1615 Runs a shell command on all entries.
qyearsley12fa6ff2016-08-24 09:18:40 -07001616 Sets GCLIENT_DEP_PATH environment variable as the dep's relative location to
ilevy@chromium.org37116242012-11-28 01:32:48 +00001617 root directory of the checkout.
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001618 """
1619 # Stop parsing at the first non-arg so that these go through to the command
1620 parser.disable_interspersed_args()
1621 parser.add_option('-s', '--scm', action='append', default=[],
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001622 help='Choose scm types to operate upon.')
maruel@chromium.org288054d2012-03-05 00:43:07 +00001623 parser.add_option('-i', '--ignore', action='store_true',
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001624 help='Ignore non-zero return codes from subcommands.')
1625 parser.add_option('--prepend-dir', action='store_true',
1626 help='Prepend relative dir for use with git <cmd> --null.')
1627 parser.add_option('--no-progress', action='store_true',
1628 help='Disable progress bar that shows sub-command updates')
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001629 options, args = parser.parse_args(args)
maruel@chromium.org45e9f2d2010-10-18 13:33:46 +00001630 if not args:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001631 print('Need to supply a command!', file=sys.stderr)
maruel@chromium.org45e9f2d2010-10-18 13:33:46 +00001632 return 1
maruel@chromium.org78cba522010-10-18 13:32:05 +00001633 root_and_entries = gclient_utils.GetGClientRootAndEntries()
1634 if not root_and_entries:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001635 print(
maruel@chromium.org78cba522010-10-18 13:32:05 +00001636 'You need to run gclient sync at least once to use \'recurse\'.\n'
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001637 'This is because .gclient_entries needs to exist and be up to date.',
1638 file=sys.stderr)
maruel@chromium.org78cba522010-10-18 13:32:05 +00001639 return 1
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001640
1641 # Normalize options.scm to a set()
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001642 scm_set = set()
1643 for scm in options.scm:
1644 scm_set.update(scm.split(','))
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001645 options.scm = scm_set
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001646
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001647 options.nohooks = True
1648 client = GClient.LoadCurrentConfig(options)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001649 return client.RunOnDeps('recurse', args, ignore_requirements=True,
1650 progress=not options.no_progress)
piman@chromium.org4b90e3a2010-07-01 20:28:26 +00001651
1652
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001653@subcommand.usage('[args ...]')
davidbarr@chromium.org12f944e2012-03-01 02:18:31 +00001654def CMDfetch(parser, args):
1655 """Fetches upstream commits for all modules.
1656
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001657 Completely git-specific. Simply runs 'git fetch [args ...]' for each module.
1658 """
davidbarr@chromium.org47ca0ee2012-03-02 16:06:11 +00001659 (options, args) = parser.parse_args(args)
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00001660 return CMDrecurse(OptionParser(), [
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001661 '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args)
1662
1663
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001664def CMDflatten(parser, args):
1665 """Flattens the solutions into a single DEPS file."""
1666 parser.add_option('--output-deps', help='Path to the output DEPS file')
1667 parser.add_option(
1668 '--require-pinned-revisions', action='store_true',
1669 help='Fail if any of the dependencies uses unpinned revision.')
1670 options, args = parser.parse_args(args)
1671
1672 options.nohooks = True
1673 client = GClient.LoadCurrentConfig(options)
1674
1675 # Only print progress if we're writing to a file. Otherwise, progress updates
1676 # could obscure intended output.
1677 code = client.RunOnDeps('flatten', args, progress=options.output_deps)
1678 if code != 0:
1679 return code
1680
1681 deps = {}
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001682 deps_os = {}
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001683 hooks = []
1684 pre_deps_hooks = []
1685 unpinned_deps = {}
1686
1687 for solution in client.dependencies:
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001688 _FlattenSolution(
1689 solution, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001690
1691 if options.require_pinned_revisions and unpinned_deps:
1692 sys.stderr.write('The following dependencies are not pinned:\n')
1693 sys.stderr.write('\n'.join(sorted(unpinned_deps)))
1694 return 1
1695
1696 flattened_deps = '\n'.join(
Paweł Hajdan, Jr3c2aa832017-06-07 20:22:16 +02001697 _GNSettingsToLines(
1698 client.dependencies[0]._gn_args_file,
1699 client.dependencies[0]._gn_args) +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001700 _DepsToLines(deps) +
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001701 _DepsOsToLines(deps_os) +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001702 _HooksToLines('hooks', hooks) +
1703 _HooksToLines('pre_deps_hooks', pre_deps_hooks) +
1704 [''] # Ensure newline at end of file.
1705 )
1706
1707 if options.output_deps:
1708 with open(options.output_deps, 'w') as f:
1709 f.write(flattened_deps)
1710 else:
1711 print(flattened_deps)
1712
1713 return 0
1714
1715
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001716def _FlattenSolution(
1717 solution, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps):
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001718 """Visits a solution in order to flatten it (see CMDflatten).
1719
1720 Arguments:
1721 solution (Dependency): one of top-level solutions in .gclient
1722
1723 Out-parameters:
1724 deps (dict of name -> Dependency): will be filled with all Dependency
1725 objects indexed by their name
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001726 deps_os (dict of os name -> dep name -> Dependency): same as above,
1727 for OS-specific deps
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001728 hooks (list of (Dependency, hook)): will be filled with flattened hooks
1729 pre_deps_hooks (list of (Dependency, hook)): will be filled with flattened
1730 pre_deps_hooks
1731 unpinned_deps (dict of name -> Dependency): will be filled with unpinned
1732 deps
1733 """
1734 logging.debug('_FlattenSolution(%r)', solution)
1735
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001736 _FlattenDep(solution, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps)
1737 _FlattenRecurse(solution, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001738
1739
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001740def _FlattenDep(dep, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps):
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001741 """Visits a dependency in order to flatten it (see CMDflatten).
1742
1743 Arguments:
1744 dep (Dependency): dependency to process
1745
1746 Out-parameters:
1747 deps (dict): will be filled with flattened deps
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001748 deps_os (dict): will be filled with flattened deps_os
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001749 hooks (list): will be filled with flattened hooks
1750 pre_deps_hooks (list): will be filled with flattened pre_deps_hooks
1751 unpinned_deps (dict): will be filled with unpinned deps
1752 """
1753 logging.debug('_FlattenDep(%r)', dep)
1754
1755 _AddDep(dep, deps, unpinned_deps)
1756
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001757 for dep_os, os_deps in dep.os_dependencies.iteritems():
1758 for os_dep in os_deps:
1759 deps_os.setdefault(dep_os, {})[os_dep.name] = os_dep
1760
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001761 deps_by_name = dict((d.name, d) for d in dep.dependencies)
1762 for recurse_dep_name in (dep.recursedeps or []):
1763 _FlattenRecurse(
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001764 deps_by_name[recurse_dep_name], deps, deps_os, hooks, pre_deps_hooks,
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001765 unpinned_deps)
1766
1767 # TODO(phajdan.jr): also handle hooks_os.
1768 hooks.extend([(dep, hook) for hook in dep.deps_hooks])
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +02001769 pre_deps_hooks.extend([(dep, hook) for hook in dep.pre_deps_hooks])
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001770
1771
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001772def _FlattenRecurse(dep, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps):
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001773 """Helper for flatten that recurses into |dep|'s dependencies.
1774
1775 Arguments:
1776 dep (Dependency): dependency to process
1777
1778 Out-parameters:
1779 deps (dict): will be filled with flattened deps
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001780 deps_os (dict): will be filled with flattened deps_os
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001781 hooks (list): will be filled with flattened hooks
1782 pre_deps_hooks (list): will be filled with flattened pre_deps_hooks
1783 unpinned_deps (dict): will be filled with unpinned deps
1784 """
1785 logging.debug('_FlattenRecurse(%r)', dep)
1786
1787 # TODO(phajdan.jr): also handle deps_os.
Paweł Hajdan, Jrcd788e32017-06-12 18:42:22 +02001788 for sub_dep in dep.orig_dependencies:
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001789 _FlattenDep(sub_dep, deps, deps_os, hooks, pre_deps_hooks, unpinned_deps)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001790
1791
1792def _AddDep(dep, deps, unpinned_deps):
1793 """Helper to add a dependency to flattened lists.
1794
1795 Arguments:
1796 dep (Dependency): dependency to process
1797
1798 Out-parameters:
1799 deps (dict): will be filled with flattened deps
1800 unpinned_deps (dict): will be filled with unpinned deps
1801 """
1802 logging.debug('_AddDep(%r)', dep)
1803
1804 assert dep.name not in deps
1805 deps[dep.name] = dep
1806
1807 # Detect unpinned deps.
1808 _, revision = gclient_utils.SplitUrlRevision(dep.url)
1809 if not revision or not gclient_utils.IsGitSha(revision):
1810 unpinned_deps[dep.name] = dep
1811
1812
Paweł Hajdan, Jr3c2aa832017-06-07 20:22:16 +02001813def _GNSettingsToLines(gn_args_file, gn_args):
1814 s = []
1815 if gn_args_file:
1816 s.extend([
1817 'gclient_gn_args_file = "%s"' % gn_args_file,
1818 'gclient_gn_args = %r' % gn_args,
1819 ])
1820 return s
1821
1822
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001823def _DepsToLines(deps):
1824 """Converts |deps| dict to list of lines for output."""
1825 s = ['deps = {']
1826 for name, dep in sorted(deps.iteritems()):
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001827 condition_part = ([' "condition": "%s",' % dep.condition]
1828 if dep.condition else [])
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001829 s.extend([
1830 ' # %s' % dep.hierarchy(include_url=False),
Paweł Hajdan, Jrf69860b2017-06-05 20:24:28 +02001831 ' "%s": {' % (name,),
1832 ' "url": "%s",' % (dep.url,),
1833 ] + condition_part + [
1834 ' },',
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001835 '',
1836 ])
1837 s.extend(['}', ''])
1838 return s
1839
1840
Paweł Hajdan, Jrc9603f52017-06-13 22:14:24 +02001841def _DepsOsToLines(deps_os):
1842 """Converts |deps_os| dict to list of lines for output."""
1843 s = ['deps_os = {']
1844 for dep_os, os_deps in sorted(deps_os.iteritems()):
1845 s.append(' "%s": {' % dep_os)
1846 s.extend([' %s' % l for l in _DepsToLines(os_deps)])
1847 s.extend([' },', ''])
1848 s.extend(['}', ''])
1849 return s
1850
1851
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001852def _HooksToLines(name, hooks):
1853 """Converts |hooks| list to list of lines for output."""
1854 s = ['%s = [' % name]
1855 for dep, hook in hooks:
1856 s.extend([
1857 ' # %s' % dep.hierarchy(include_url=False),
1858 ' {',
1859 ])
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02001860 if hook.name is not None:
1861 s.append(' "name": "%s",' % hook.name)
1862 if hook.pattern is not None:
1863 s.append(' "pattern": "%s",' % hook.pattern)
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001864 s.extend(
Paweł Hajdan, Jrc9364392017-06-14 17:11:56 +02001865 # Hooks run in the parent directory of their dep.
1866 [' "cwd": "%s"' % os.path.normpath(os.path.dirname(dep.name))] +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001867 [' "action": ['] +
Paweł Hajdan, Jrc10a4d82017-06-14 14:06:50 +02001868 [' "%s",' % arg for arg in hook.action] +
Paweł Hajdan, Jr064f6f42017-05-18 22:17:55 +02001869 [' ]', ' },', '']
1870 )
1871 s.extend([']', ''])
1872 return s
1873
1874
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001875def CMDgrep(parser, args):
1876 """Greps through git repos managed by gclient.
1877
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001878 Runs 'git grep [args...]' for each module.
1879 """
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001880 # We can't use optparse because it will try to parse arguments sent
1881 # to git grep and throw an error. :-(
1882 if not args or re.match('(-h|--help)$', args[0]):
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001883 print(
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001884 'Usage: gclient grep [-j <N>] git-grep-args...\n\n'
1885 'Example: "gclient grep -j10 -A2 RefCountedBase" runs\n"git grep '
1886 '-A2 RefCountedBase" on each of gclient\'s git\nrepos with up to '
1887 '10 jobs.\n\nBonus: page output by appending "|& less -FRSX" to the'
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00001888 ' end of your query.',
1889 file=sys.stderr)
ilevy@chromium.orgf2ed3fb2012-11-09 23:39:49 +00001890 return 1
1891
1892 jobs_arg = ['--jobs=1']
1893 if re.match(r'(-j|--jobs=)\d+$', args[0]):
1894 jobs_arg, args = args[:1], args[1:]
1895 elif re.match(r'(-j|--jobs)$', args[0]):
1896 jobs_arg, args = args[:2], args[2:]
1897
1898 return CMDrecurse(
1899 parser,
1900 jobs_arg + ['--ignore', '--prepend-dir', '--no-progress', '--scm=git',
1901 'git', 'grep', '--null', '--color=Always'] + args)
davidbarr@chromium.org12f944e2012-03-01 02:18:31 +00001902
1903
stip@chromium.orga735da22015-04-29 23:18:20 +00001904def CMDroot(parser, args):
1905 """Outputs the solution root (or current dir if there isn't one)."""
1906 (options, args) = parser.parse_args(args)
1907 client = GClient.LoadCurrentConfig(options)
1908 if client:
1909 print(os.path.abspath(client.root_dir))
1910 else:
1911 print(os.path.abspath('.'))
1912
1913
agablea98a6cd2016-11-15 14:30:10 -08001914@subcommand.usage('[url]')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00001915def CMDconfig(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001916 """Creates a .gclient file in the current directory.
maruel@chromium.orgddff62d2010-05-17 21:02:36 +00001917
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001918 This specifies the configuration for further commands. After update/sync,
1919 top-level DEPS files in each module are read to determine dependent
1920 modules to operate on as well. If optional [url] parameter is
1921 provided, then configuration is read from a specified Subversion server
1922 URL.
1923 """
szager@chromium.orge2e03202012-07-31 18:05:16 +00001924 # We do a little dance with the --gclientfile option. 'gclient config' is the
1925 # only command where it's acceptable to have both '--gclientfile' and '--spec'
1926 # arguments. So, we temporarily stash any --gclientfile parameter into
1927 # options.output_config_file until after the (gclientfile xor spec) error
1928 # check.
1929 parser.remove_option('--gclientfile')
1930 parser.add_option('--gclientfile', dest='output_config_file',
1931 help='Specify an alternate .gclient file')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00001932 parser.add_option('--name',
1933 help='overrides the default name for the solution')
nsylvain@google.comefc80932011-05-31 21:27:56 +00001934 parser.add_option('--deps-file', default='DEPS',
1935 help='overrides the default name for the DEPS file for the'
1936 'main solutions and all sub-dependencies')
smutae7ea312016-07-18 11:59:41 -07001937 parser.add_option('--unmanaged', action='store_true', default=False,
cmp@chromium.orgeb2756d2011-09-20 20:17:51 +00001938 help='overrides the default behavior to make it possible '
smutae7ea312016-07-18 11:59:41 -07001939 'to have the main solution untouched by gclient '
1940 '(gclient will check out unmanaged dependencies but '
1941 'will never sync them)')
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001942 parser.add_option('--cache-dir',
1943 help='(git only) Cache all git repos into this dir and do '
1944 'shared clones from the cache, instead of cloning '
1945 'directly from the remote. (experimental)')
szager@chromium.orge2e03202012-07-31 18:05:16 +00001946 parser.set_defaults(config_filename=None)
maruel@chromium.org5ca27692010-05-26 19:32:41 +00001947 (options, args) = parser.parse_args(args)
szager@chromium.orge2e03202012-07-31 18:05:16 +00001948 if options.output_config_file:
1949 setattr(options, 'config_filename', getattr(options, 'output_config_file'))
maruel@chromium.org5fc2a332010-05-26 19:37:15 +00001950 if ((options.spec and args) or len(args) > 2 or
1951 (not options.spec and not args)):
1952 parser.error('Inconsistent arguments. Use either --spec or one or 2 args')
1953
maruel@chromium.org2806acc2009-05-15 12:33:34 +00001954 client = GClient('.', options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001955 if options.spec:
1956 client.SetConfig(options.spec)
1957 else:
maruel@chromium.org1ab7ffc2009-06-03 17:21:37 +00001958 base_url = args[0].rstrip('/')
iposva@chromium.org8cf7a392010-04-07 17:20:26 +00001959 if not options.name:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00001960 name = base_url.split('/')[-1]
nsylvain@google.com12649ef2011-06-01 17:11:20 +00001961 if name.endswith('.git'):
1962 name = name[:-4]
iposva@chromium.org8cf7a392010-04-07 17:20:26 +00001963 else:
1964 # specify an alternate relpath for the given URL.
1965 name = options.name
agable@chromium.orgf2214672015-10-27 21:02:48 +00001966 if not os.path.abspath(os.path.join(os.getcwd(), name)).startswith(
1967 os.getcwd()):
1968 parser.error('Do not pass a relative path for --name.')
1969 if any(x in ('..', '.', '/', '\\') for x in name.split(os.sep)):
1970 parser.error('Do not include relative path components in --name.')
1971
nsylvain@google.comefc80932011-05-31 21:27:56 +00001972 deps_file = options.deps_file
agablea98a6cd2016-11-15 14:30:10 -08001973 client.SetDefaultConfig(name, deps_file, base_url,
smutae7ea312016-07-18 11:59:41 -07001974 managed=not options.unmanaged,
iannucci@chromium.org53456aa2013-07-03 19:38:34 +00001975 cache_dir=options.cache_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001976 client.SaveConfig()
maruel@chromium.org79692d62010-05-14 18:57:13 +00001977 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001978
1979
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001980@subcommand.epilog("""Example:
maruel@chromium.org5ca27692010-05-26 19:32:41 +00001981 gclient pack > patch.txt
1982 generate simple patch for configured client and dependences
1983""")
1984def CMDpack(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001985 """Generates a patch which can be applied at the root of the tree.
maruel@chromium.orgddff62d2010-05-17 21:02:36 +00001986
agabled437d762016-10-17 09:35:11 -07001987 Internally, runs 'git diff' on each checked out module and
maruel@chromium.org39c0b222013-08-17 16:57:01 +00001988 dependencies, and performs minimal postprocessing of the output. The
1989 resulting patch is printed to stdout and can be applied to a freshly
1990 checked out tree via 'patch -p0 < patchfile'.
1991 """
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00001992 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1993 help='override deps for the specified (comma-separated) '
1994 'platform(s); \'all\' will process all deps_os '
1995 'references')
haitao.feng@intel.com306080c2012-05-04 13:11:29 +00001996 parser.remove_option('--jobs')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00001997 (options, args) = parser.parse_args(args)
iannucci@chromium.org50395ea2013-04-04 04:47:42 +00001998 # Force jobs to 1 so the stdout is not annotated with the thread ids
haitao.feng@intel.com306080c2012-05-04 13:11:29 +00001999 options.jobs = 1
kbr@google.comab318592009-09-04 00:54:55 +00002000 client = GClient.LoadCurrentConfig(options)
2001 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002002 raise gclient_utils.Error('client not configured; see \'gclient config\'')
kbr@google.comab318592009-09-04 00:54:55 +00002003 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002004 client.PrintLocationAndContents()
kbr@google.comab318592009-09-04 00:54:55 +00002005 return client.RunOnDeps('pack', args)
2006
2007
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002008def CMDstatus(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002009 """Shows modification status for every dependencies."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002010 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2011 help='override deps for the specified (comma-separated) '
2012 'platform(s); \'all\' will process all deps_os '
2013 'references')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002014 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002015 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002016 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002017 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002018 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002019 client.PrintLocationAndContents()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002020 return client.RunOnDeps('status', args)
2021
2022
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002023@subcommand.epilog("""Examples:
maruel@chromium.org79692d62010-05-14 18:57:13 +00002024 gclient sync
2025 update files from SCM according to current configuration,
2026 *for modules which have changed since last update or sync*
2027 gclient sync --force
2028 update files from SCM according to current configuration, for
2029 all modules (useful for recovering files deleted from local copy)
2030 gclient sync --revision src@31000
2031 update src directory to r31000
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002032
2033JSON output format:
2034If the --output-json option is specified, the following document structure will
2035be emitted to the provided file. 'null' entries may occur for subprojects which
2036are present in the gclient solution, but were not processed (due to custom_deps,
2037os_deps, etc.)
2038
2039{
2040 "solutions" : {
2041 "<name>": { # <name> is the posix-normalized path to the solution.
agabled437d762016-10-17 09:35:11 -07002042 "revision": [<git id hex string>|null],
2043 "scm": ["git"|null],
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002044 }
2045 }
2046}
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002047""")
2048def CMDsync(parser, args):
2049 """Checkout/update all modules."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002050 parser.add_option('-f', '--force', action='store_true',
2051 help='force update even for unchanged modules')
2052 parser.add_option('-n', '--nohooks', action='store_true',
2053 help='don\'t run hooks after the update is complete')
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00002054 parser.add_option('-p', '--noprehooks', action='store_true',
2055 help='don\'t run pre-DEPS hooks', default=False)
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002056 parser.add_option('-r', '--revision', action='append',
2057 dest='revisions', metavar='REV', default=[],
2058 help='Enforces revision/hash for the solutions with the '
2059 'format src@rev. The src@ part is optional and can be '
2060 'skipped. -r can be used multiple times when .gclient '
2061 'has multiple solutions configured and will work even '
agablea98a6cd2016-11-15 14:30:10 -08002062 'if the src@ part is skipped.')
maruel@chromium.org794207e2013-03-08 15:29:43 +00002063 parser.add_option('--with_branch_heads', action='store_true',
2064 help='Clone git "branch_heads" refspecs in addition to '
2065 'the default refspecs. This adds about 1/2GB to a '
2066 'full checkout. (git only)')
szager@chromium.org8d3348f2014-08-19 22:49:16 +00002067 parser.add_option('--with_tags', action='store_true',
2068 help='Clone git tags in addition to the default refspecs.')
agable2697cd12016-06-28 10:23:53 -07002069 parser.add_option('-H', '--head', action='store_true',
agablea98a6cd2016-11-15 14:30:10 -08002070 help='DEPRECATED: only made sense with safesync urls.')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002071 parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002072 help='Deletes from the working copy any dependencies that '
2073 'have been removed since the last sync, as long as '
2074 'there are no local modifications. When used with '
2075 '--force, such dependencies are removed even if they '
2076 'have local modifications. When used with --reset, '
2077 'all untracked directories are removed from the '
tnagel@chromium.orga2aaa632014-02-28 21:47:27 +00002078 'working copy, excluding those which are explicitly '
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002079 'ignored in the repository.')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002080 parser.add_option('-R', '--reset', action='store_true',
2081 help='resets any local changes before updating (git only)')
bauerb@chromium.org2aad1b22011-07-22 12:00:41 +00002082 parser.add_option('-M', '--merge', action='store_true',
2083 help='merge upstream changes instead of trying to '
2084 'fast-forward or rebase')
dnj@chromium.org5b23e872015-02-20 21:25:57 +00002085 parser.add_option('-A', '--auto_rebase', action='store_true',
2086 help='Automatically rebase repositories against local '
2087 'checkout during update (git only).')
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002088 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2089 help='override deps for the specified (comma-separated) '
2090 'platform(s); \'all\' will process all deps_os '
2091 'references')
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +00002092 parser.add_option('--upstream', action='store_true',
2093 help='Make repo state match upstream branch.')
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002094 parser.add_option('--output-json',
2095 help='Output a json document to this path containing '
2096 'summary information about the sync.')
primiano@chromium.org5439ea52014-08-06 17:18:18 +00002097 parser.add_option('--no-history', action='store_true',
2098 help='GIT ONLY - Reduces the size/time of the checkout at '
2099 'the cost of no history. Requires Git 1.9+')
hinoka@chromium.org46b87412014-05-15 00:42:05 +00002100 parser.add_option('--shallow', action='store_true',
2101 help='GIT ONLY - Do a shallow clone into the cache dir. '
2102 'Requires Git 1.9+')
e.hakkinen@samsung.come8bc1aa2015-04-08 08:00:37 +00002103 parser.add_option('--no_bootstrap', '--no-bootstrap',
2104 action='store_true',
2105 help='Don\'t bootstrap from Google Storage.')
hinoka@chromium.org8a10f6d2014-06-23 18:38:57 +00002106 parser.add_option('--ignore_locks', action='store_true',
2107 help='GIT ONLY - Ignore cache locks.')
iannucci@chromium.org30a07982016-04-07 21:35:19 +00002108 parser.add_option('--break_repo_locks', action='store_true',
2109 help='GIT ONLY - Forcibly remove repo locks (e.g. '
2110 'index.lock). This should only be used if you know for '
2111 'certain that this invocation of gclient is the only '
2112 'thing operating on the git repos (e.g. on a bot).')
nodir@chromium.org5b48e482016-03-18 20:27:54 +00002113 parser.add_option('--lock_timeout', type='int', default=5000,
szager@chromium.orgdbb6f822016-02-02 22:59:30 +00002114 help='GIT ONLY - Deadline (in seconds) to wait for git '
nodir@chromium.org5b48e482016-03-18 20:27:54 +00002115 'cache lock to become available. Default is %default.')
agabled437d762016-10-17 09:35:11 -07002116 # TODO(agable): Remove these when the oldest CrOS release milestone is M56.
2117 parser.add_option('-t', '--transitive', action='store_true',
2118 help='DEPRECATED: This is a no-op.')
sdefresne69b1be12016-10-18 05:48:02 -07002119 parser.add_option('-m', '--manually_grab_svn_rev', action='store_true',
agabled437d762016-10-17 09:35:11 -07002120 help='DEPRECATED: This is a no-op.')
Paweł Hajdan, Jr7c7b5592017-05-23 15:06:05 +02002121 # TODO(phajdan.jr): Remove validation options once default (crbug/570091).
Paweł Hajdan, Jr694773d2017-05-29 16:06:23 +02002122 parser.add_option('--validate-syntax', action='store_true', default=True,
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02002123 help='Validate the .gclient and DEPS syntax')
Paweł Hajdan, Jr7c7b5592017-05-23 15:06:05 +02002124 parser.add_option('--disable-syntax-validation', action='store_false',
2125 dest='validate_syntax',
2126 help='Disable validation of .gclient and DEPS syntax.')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002127 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002128 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002129
2130 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002131 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002132
smutae7ea312016-07-18 11:59:41 -07002133 if options.revisions and options.head:
2134 # TODO(maruel): Make it a parser.error if it doesn't break any builder.
2135 print('Warning: you cannot use both --head and --revision')
2136
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002137 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002138 client.PrintLocationAndContents()
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002139 ret = client.RunOnDeps('update', args)
2140 if options.output_json:
2141 slns = {}
2142 for d in client.subtree(True):
2143 normed = d.name.replace('\\', '/').rstrip('/') + '/'
2144 slns[normed] = {
2145 'revision': d.got_revision,
2146 'scm': d.used_scm.name if d.used_scm else None,
hinoka@chromium.org17db9052014-05-10 01:11:29 +00002147 'url': str(d.url) if d.url else None,
iannucci@chromium.org2702bcd2013-09-24 19:10:07 +00002148 }
2149 with open(options.output_json, 'wb') as f:
2150 json.dump({'solutions': slns}, f)
2151 return ret
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002152
2153
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002154CMDupdate = CMDsync
2155
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002156
Paweł Hajdan, Jre2f9feec2017-05-09 10:04:02 +02002157def CMDvalidate(parser, args):
2158 """Validates the .gclient and DEPS syntax."""
2159 options, args = parser.parse_args(args)
2160 options.validate_syntax = True
2161 client = GClient.LoadCurrentConfig(options)
2162 rv = client.RunOnDeps('validate', args)
2163 if rv == 0:
2164 print('validate: SUCCESS')
2165 else:
2166 print('validate: FAILURE')
2167 return rv
2168
2169
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002170def CMDdiff(parser, args):
2171 """Displays local diff for every dependencies."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002172 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2173 help='override deps for the specified (comma-separated) '
2174 'platform(s); \'all\' will process all deps_os '
2175 'references')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002176 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002177 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002178 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002179 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002180 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002181 client.PrintLocationAndContents()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002182 return client.RunOnDeps('diff', args)
2183
2184
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002185def CMDrevert(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002186 """Reverts all modifications in every dependencies.
maruel@chromium.org28d14bd2010-11-11 20:37:09 +00002187
2188 That's the nuclear option to get back to a 'clean' state. It removes anything
agabled437d762016-10-17 09:35:11 -07002189 that shows up in git status."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002190 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2191 help='override deps for the specified (comma-separated) '
2192 'platform(s); \'all\' will process all deps_os '
2193 'references')
2194 parser.add_option('-n', '--nohooks', action='store_true',
2195 help='don\'t run hooks after the revert is complete')
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00002196 parser.add_option('-p', '--noprehooks', action='store_true',
2197 help='don\'t run pre-DEPS hooks', default=False)
iannucci@chromium.orgd4fffee2013-06-28 00:35:26 +00002198 parser.add_option('--upstream', action='store_true',
2199 help='Make repo state match upstream branch.')
iannucci@chromium.orgbf525dc2016-04-07 22:00:28 +00002200 parser.add_option('--break_repo_locks', action='store_true',
2201 help='GIT ONLY - Forcibly remove repo locks (e.g. '
2202 'index.lock). This should only be used if you know for '
2203 'certain that this invocation of gclient is the only '
2204 'thing operating on the git repos (e.g. on a bot).')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002205 (options, args) = parser.parse_args(args)
2206 # --force is implied.
2207 options.force = True
steveblock@chromium.org98e69452012-02-16 16:36:43 +00002208 options.reset = False
2209 options.delete_unversioned_trees = False
agablec903d732016-07-26 09:07:24 -07002210 options.merge = False
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002211 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002212 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002213 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002214 return client.RunOnDeps('revert', args)
2215
2216
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002217def CMDrunhooks(parser, args):
2218 """Runs hooks for files that have been modified in the local working copy."""
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002219 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2220 help='override deps for the specified (comma-separated) '
2221 'platform(s); \'all\' will process all deps_os '
2222 'references')
2223 parser.add_option('-f', '--force', action='store_true', default=True,
2224 help='Deprecated. No effect.')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002225 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002226 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002227 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002228 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002229 if options.verbose:
sergiyb@chromium.orgfa2707e2016-03-12 00:40:56 +00002230 client.PrintLocationAndContents()
maruel@chromium.org5df6a462009-08-28 18:52:26 +00002231 options.force = True
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002232 options.nohooks = False
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002233 return client.RunOnDeps('runhooks', args)
2234
2235
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002236def CMDrevinfo(parser, args):
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002237 """Outputs revision info mapping for the client and its dependencies.
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002238
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002239 This allows the capture of an overall 'revision' for the source tree that
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002240 can be used to reproduce the same tree in the future. It is only useful for
agabled437d762016-10-17 09:35:11 -07002241 'unpinned dependencies', i.e. DEPS/deps references without a git hash.
2242 A git branch name isn't 'pinned' since the actual commit can change.
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002243 """
2244 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
2245 help='override deps for the specified (comma-separated) '
2246 'platform(s); \'all\' will process all deps_os '
2247 'references')
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00002248 parser.add_option('-a', '--actual', action='store_true',
2249 help='gets the actual checked out revisions instead of the '
2250 'ones specified in the DEPS and .gclient files')
maruel@chromium.org9eda4112010-06-11 18:56:10 +00002251 parser.add_option('-s', '--snapshot', action='store_true',
2252 help='creates a snapshot .gclient file of the current '
maruel@chromium.orgb1e315f2010-08-11 18:44:50 +00002253 'version of all repositories to reproduce the tree, '
2254 'implies -a')
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002255 (options, args) = parser.parse_args(args)
maruel@chromium.org2806acc2009-05-15 12:33:34 +00002256 client = GClient.LoadCurrentConfig(options)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002257 if not client:
maruel@chromium.org0b6a0842010-06-15 14:34:19 +00002258 raise gclient_utils.Error('client not configured; see \'gclient config\'')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002259 client.PrintRevInfo()
maruel@chromium.org79692d62010-05-14 18:57:13 +00002260 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002261
2262
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00002263def CMDverify(parser, args):
2264 """Verifies the DEPS file deps are only from allowed_hosts."""
2265 (options, args) = parser.parse_args(args)
2266 client = GClient.LoadCurrentConfig(options)
2267 if not client:
2268 raise gclient_utils.Error('client not configured; see \'gclient config\'')
2269 client.RunOnDeps(None, [])
2270 # Look at each first-level dependency of this gclient only.
2271 for dep in client.dependencies:
2272 bad_deps = dep.findDepsFromNotAllowedHosts()
2273 if not bad_deps:
2274 continue
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002275 print("There are deps from not allowed hosts in file %s" % dep.deps_file)
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00002276 for bad_dep in bad_deps:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002277 print("\t%s at %s" % (bad_dep.name, bad_dep.url))
2278 print("allowed_hosts:", ', '.join(dep.allowed_hosts))
tandrii@chromium.orgc137c1a2014-09-23 11:49:52 +00002279 sys.stdout.flush()
2280 raise gclient_utils.Error(
2281 'dependencies from disallowed hosts; check your DEPS file.')
2282 return 0
2283
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002284class OptionParser(optparse.OptionParser):
szager@chromium.orge2e03202012-07-31 18:05:16 +00002285 gclientfile_default = os.environ.get('GCLIENT_FILE', '.gclient')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002286
2287 def __init__(self, **kwargs):
2288 optparse.OptionParser.__init__(
2289 self, version='%prog ' + __version__, **kwargs)
2290
2291 # Some arm boards have issues with parallel sync.
2292 if platform.machine().startswith('arm'):
2293 jobs = 1
2294 else:
2295 jobs = max(8, gclient_utils.NumLocalCpus())
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002296
2297 self.add_option(
2298 '-j', '--jobs', default=jobs, type='int',
2299 help='Specify how many SCM commands can run in parallel; defaults to '
tnagel@chromium.orga2aaa632014-02-28 21:47:27 +00002300 '%default on this machine')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002301 self.add_option(
2302 '-v', '--verbose', action='count', default=0,
2303 help='Produces additional output for diagnostics. Can be used up to '
2304 'three times for more logging info.')
2305 self.add_option(
2306 '--gclientfile', dest='config_filename',
2307 help='Specify an alternate %s file' % self.gclientfile_default)
2308 self.add_option(
2309 '--spec',
2310 help='create a gclient file containing the provided string. Due to '
2311 'Cygwin/Python brokenness, it can\'t contain any newlines.')
2312 self.add_option(
2313 '--no-nag-max', default=False, action='store_true',
scottmg@chromium.orgf547c802013-09-27 17:55:26 +00002314 help='Ignored for backwards compatibility.')
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002315
2316 def parse_args(self, args=None, values=None):
2317 """Integrates standard options processing."""
2318 options, args = optparse.OptionParser.parse_args(self, args, values)
2319 levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
2320 logging.basicConfig(
2321 level=levels[min(options.verbose, len(levels) - 1)],
maruel@chromium.org0895b752011-08-26 20:40:33 +00002322 format='%(module)s(%(lineno)d) %(funcName)s:%(message)s')
szager@chromium.orge2e03202012-07-31 18:05:16 +00002323 if options.config_filename and options.spec:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002324 self.error('Cannot specifiy both --gclientfile and --spec')
rdsmith@chromium.orgd9591f02014-02-05 19:28:20 +00002325 if (options.config_filename and
2326 options.config_filename != os.path.basename(options.config_filename)):
2327 self.error('--gclientfile target must be a filename, not a path')
szager@chromium.orge2e03202012-07-31 18:05:16 +00002328 if not options.config_filename:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002329 options.config_filename = self.gclientfile_default
maruel@chromium.org0895b752011-08-26 20:40:33 +00002330 options.entries_filename = options.config_filename + '_entries'
2331 if options.jobs < 1:
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002332 self.error('--jobs must be 1 or higher')
maruel@chromium.org0895b752011-08-26 20:40:33 +00002333
2334 # These hacks need to die.
2335 if not hasattr(options, 'revisions'):
2336 # GClient.RunOnDeps expects it even if not applicable.
2337 options.revisions = []
smutae7ea312016-07-18 11:59:41 -07002338 if not hasattr(options, 'head'):
2339 options.head = None
maruel@chromium.org0895b752011-08-26 20:40:33 +00002340 if not hasattr(options, 'nohooks'):
2341 options.nohooks = True
borenet@google.com2d1ee9e2013-10-15 08:13:16 +00002342 if not hasattr(options, 'noprehooks'):
2343 options.noprehooks = True
maruel@chromium.org0895b752011-08-26 20:40:33 +00002344 if not hasattr(options, 'deps_os'):
2345 options.deps_os = None
maruel@chromium.org0895b752011-08-26 20:40:33 +00002346 if not hasattr(options, 'force'):
2347 options.force = None
2348 return (options, args)
iannucci@chromium.orgd9c1b202013-07-24 23:52:11 +00002349
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002350
2351def disable_buffering():
2352 # Make stdout auto-flush so buildbot doesn't kill us during lengthy
2353 # operations. Python as a strong tendency to buffer sys.stdout.
2354 sys.stdout = gclient_utils.MakeFileAutoFlush(sys.stdout)
2355 # Make stdout annotated with the thread ids.
2356 sys.stdout = gclient_utils.MakeFileAnnotated(sys.stdout)
maruel@chromium.org0895b752011-08-26 20:40:33 +00002357
2358
sbc@chromium.org013731e2015-02-26 18:28:43 +00002359def main(argv):
maruel@chromium.org5ca27692010-05-26 19:32:41 +00002360 """Doesn't parse the arguments here, just find the right subcommand to
2361 execute."""
maruel@chromium.org82798cb2012-02-23 18:16:12 +00002362 if sys.hexversion < 0x02060000:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002363 print(
maruel@chromium.org82798cb2012-02-23 18:16:12 +00002364 '\nYour python version %s is unsupported, please upgrade.\n' %
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002365 sys.version.split(' ', 1)[0],
2366 file=sys.stderr)
maruel@chromium.org82798cb2012-02-23 18:16:12 +00002367 return 2
bcwhite@chromium.org6683ab42013-02-11 16:13:47 +00002368 if not sys.executable:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002369 print(
2370 '\nPython cannot find the location of it\'s own executable.\n',
2371 file=sys.stderr)
bcwhite@chromium.org6683ab42013-02-11 16:13:47 +00002372 return 2
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002373 fix_encoding.fix_encoding()
2374 disable_buffering()
iannucci@chromium.org596cd5c2016-04-04 21:34:39 +00002375 setup_color.init()
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002376 dispatcher = subcommand.CommandDispatcher(__name__)
maruel@chromium.org6e29d572010-06-04 17:32:20 +00002377 try:
maruel@chromium.org39c0b222013-08-17 16:57:01 +00002378 return dispatcher.execute(OptionParser(), argv)
xusydoc@chromium.org2fd6c3f2013-05-03 21:57:55 +00002379 except KeyboardInterrupt:
2380 gclient_utils.GClientChildren.KillAllRemainingChildren()
2381 raise
vapier@chromium.orga81a56e2015-11-11 07:56:13 +00002382 except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
vapier@chromium.orgbb79bea2015-11-11 07:30:23 +00002383 print('Error: %s' % str(e), file=sys.stderr)
maruel@chromium.org6e29d572010-06-04 17:32:20 +00002384 return 1
borenet@google.com6a9b1682014-03-24 18:35:23 +00002385 finally:
2386 gclient_utils.PrintWarnings()
sbc@chromium.org013731e2015-02-26 18:28:43 +00002387 return 0
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002388
2389
maruel@chromium.orgf0fc9912010-06-11 17:57:33 +00002390if '__main__' == __name__:
sbc@chromium.org013731e2015-02-26 18:28:43 +00002391 try:
2392 sys.exit(main(sys.argv[1:]))
2393 except KeyboardInterrupt:
2394 sys.stderr.write('interrupted\n')
2395 sys.exit(1)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002396
2397# vim: ts=2:sw=2:tw=80:et: