blob: cbbcd77647d1f080cab27ed7cf562a29fffe7071 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A wrapper script to manage a set of client modules in different SCM.
18
19This script is intended to be used to help basic management of client
20program sources residing in one or more Subversion modules, along with
21other modules it depends on, also in Subversion, but possibly on
22multiple respositories, making a wrapper system apparently necessary.
23
24Files
25 .gclient : Current client configuration, written by 'config' command.
26 Format is a Python script defining 'solutions', a list whose
27 entries each are maps binding the strings "name" and "url"
28 to strings specifying the name and location of the client
29 module, as well as "custom_deps" to a map similar to the DEPS
30 file below.
31 .gclient_entries : A cache constructed by 'update' command. Format is a
32 Python script defining 'entries', a list of the names
33 of all modules in the client
34 <module>/DEPS : Python script defining var 'deps' as a map from each requisite
35 submodule name to a URL where it can be found (via one SCM)
36
37Hooks
38 .gclient and DEPS files may optionally contain a list named "hooks" to
39 allow custom actions to be performed based on files that have changed in the
40 working copy as a result of a "sync"/"update" or "revert" operation. Hooks
41 can also be run based on what files have been modified in the working copy
42 with the "runhooks" operation. If any of these operation are run with
43 --force, all known hooks will run regardless of the state of the working
44 copy.
45
46 Each item in a "hooks" list is a dict, containing these two keys:
47 "pattern" The associated value is a string containing a regular
48 expression. When a file whose pathname matches the expression
49 is checked out, updated, or reverted, the hook's "action" will
50 run.
51 "action" A list describing a command to run along with its arguments, if
52 any. An action command will run at most one time per gclient
53 invocation, regardless of how many files matched the pattern.
54 The action is executed in the same directory as the .gclient
55 file. If the first item in the list is the string "python",
56 the current Python interpreter (sys.executable) will be used
57 to run the command.
58
59 Example:
60 hooks = [
61 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
62 "action": ["python", "image_indexer.py", "--all"]},
63 ]
64"""
65
66__author__ = "darinf@gmail.com (Darin Fisher)"
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +000067__version__ = "0.3.2"
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000068
69import errno
70import optparse
71import os
72import re
73import stat
74import subprocess
75import sys
76import time
77import urlparse
78import xml.dom.minidom
79import urllib
80
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000081
82SVN_COMMAND = "svn"
83
84
85# default help text
86DEFAULT_USAGE_TEXT = (
87"""usage: %prog <subcommand> [options] [--] [svn options/args...]
88a wrapper for managing a set of client modules in svn.
89Version """ + __version__ + """
90
91subcommands:
92 cleanup
93 config
94 diff
95 revert
96 status
97 sync
98 update
99 runhooks
100 revinfo
101
102Options and extra arguments can be passed to invoked svn commands by
103appending them to the command line. Note that if the first such
104appended option starts with a dash (-) then the options must be
105preceded by -- to distinguish them from gclient options.
106
107For additional help on a subcommand or examples of usage, try
108 %prog help <subcommand>
109 %prog help files
110""")
111
112GENERIC_UPDATE_USAGE_TEXT = (
113 """Perform a checkout/update of the modules specified by the gclient
114configuration; see 'help config'. Unless --revision is specified,
115then the latest revision of the root solutions is checked out, with
116dependent submodule versions updated according to DEPS files.
117If --revision is specified, then the given revision is used in place
118of the latest, either for a single solution or for all solutions.
119Unless the --force option is provided, solutions and modules whose
120local revision matches the one to update (i.e., they have not changed
121in the repository) are *not* modified.
122This a synonym for 'gclient %(alias)s'
123
124usage: gclient %(cmd)s [options] [--] [svn update options/args]
125
126Valid options:
127 --force : force update even for unchanged modules
128 --revision REV : update/checkout all solutions with specified revision
129 --revision SOLUTION@REV : update given solution to specified revision
130 --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
131 --verbose : output additional diagnostics
132
133Examples:
134 gclient %(cmd)s
135 update files from SVN according to current configuration,
136 *for modules which have changed since last update or sync*
137 gclient %(cmd)s --force
138 update files from SVN according to current configuration, for
139 all modules (useful for recovering files deleted from local copy)
140""")
141
142COMMAND_USAGE_TEXT = {
143 "cleanup":
144 """Clean up all working copies, using 'svn cleanup' for each module.
145Additional options and args may be passed to 'svn cleanup'.
146
147usage: cleanup [options] [--] [svn cleanup args/options]
148
149Valid options:
150 --verbose : output additional diagnostics
151""",
152 "config": """Create a .gclient file in the current directory; this
153specifies the configuration for further commands. After update/sync,
154top-level DEPS files in each module are read to determine dependent
155modules to operate on as well. If optional [url] parameter is
156provided, then configuration is read from a specified Subversion server
157URL. Otherwise, a --spec option must be provided.
158
159usage: config [option | url] [safesync url]
160
161Valid options:
162 --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
163 *Note that due to Cygwin/Python brokenness, it
164 probably can't contain any newlines.*
165
166Examples:
167 gclient config https://gclient.googlecode.com/svn/trunk/gclient
168 configure a new client to check out gclient.py tool sources
169 gclient config --spec='solutions=[{"name":"gclient","""
170 '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
171 '"custom_deps":{}}]',
172 "diff": """Display the differences between two revisions of modules.
173(Does 'svn diff' for each checked out module and dependences.)
174Additional args and options to 'svn diff' can be passed after
175gclient options.
176
177usage: diff [options] [--] [svn args/options]
178
179Valid options:
180 --verbose : output additional diagnostics
181
182Examples:
183 gclient diff
184 simple 'svn diff' for configured client and dependences
185 gclient diff -- -x -b
186 use 'svn diff -x -b' to suppress whitespace-only differences
187 gclient diff -- -r HEAD -x -b
188 diff versus the latest version of each module
189""",
190 "revert":
191 """Revert every file in every managed directory in the client view.
192
193usage: revert
194""",
195 "status":
196 """Show the status of client and dependent modules, using 'svn diff'
197for each module. Additional options and args may be passed to 'svn diff'.
198
199usage: status [options] [--] [svn diff args/options]
200
201Valid options:
202 --verbose : output additional diagnostics
203""",
204 "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
205 "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
206 "help": """Describe the usage of this program or its subcommands.
207
208usage: help [options] [subcommand]
209
210Valid options:
211 --verbose : output additional diagnostics
212""",
213 "runhooks":
214 """Runs hooks for files that have been modified in the local working copy,
215according to 'svn status'.
216
217usage: runhooks [options]
218
219Valid options:
220 --force : runs all known hooks, regardless of the working
221 copy status
222 --verbose : output additional diagnostics
223""",
224 "revinfo":
225 """Outputs source path, server URL and revision information for every
226dependency in all solutions (no local checkout required).
227
228usage: revinfo [options]
229""",
230}
231
232# parameterized by (solution_name, solution_url, safesync_url)
233DEFAULT_CLIENT_FILE_TEXT = (
234 """
235# An element of this array (a \"solution\") describes a repository directory
236# that will be checked out into your working copy. Each solution may
237# optionally define additional dependencies (via its DEPS file) to be
238# checked out alongside the solution's directory. A solution may also
239# specify custom dependencies (via the \"custom_deps\" property) that
240# override or augment the dependencies specified by the DEPS file.
241# If a \"safesync_url\" is specified, it is assumed to reference the location of
242# a text file which contains nothing but the last known good SCM revision to
243# sync against. It is fetched if specified and used unless --head is passed
244solutions = [
245 { \"name\" : \"%s\",
246 \"url\" : \"%s\",
247 \"custom_deps\" : {
248 # To use the trunk of a component instead of what's in DEPS:
249 #\"component\": \"https://svnserver/component/trunk/\",
250 # To exclude a component from your working copy:
251 #\"data/really_large_component\": None,
252 },
253 \"safesync_url\": \"%s\"
254 }
255]
256""")
257
258
259## Generic utils
260
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000261def ParseXML(output):
262 try:
263 return xml.dom.minidom.parseString(output)
264 except xml.parsers.expat.ExpatError:
265 return None
266
267
maruel@chromium.org483b0082009-05-07 02:57:14 +0000268def GetNamedNodeText(node, node_name):
269 child_nodes = node.getElementsByTagName(node_name)
270 if not child_nodes:
271 return None
272 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
273 return child_nodes[0].firstChild.nodeValue
274
275
276def GetNodeNamedAttributeText(node, node_name, attribute_name):
277 child_nodes = node.getElementsByTagName(node_name)
278 if not child_nodes:
279 return None
280 assert len(child_nodes) == 1
281 return child_nodes[0].getAttribute(attribute_name)
282
283
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284class Error(Exception):
285 """gclient exception class."""
286 pass
287
288class PrintableObject(object):
289 def __str__(self):
290 output = ''
291 for i in dir(self):
292 if i.startswith('__'):
293 continue
294 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
295 return output
296
297
298def FileRead(filename):
299 content = None
300 f = open(filename, "rU")
301 try:
302 content = f.read()
303 finally:
304 f.close()
305 return content
306
307
308def FileWrite(filename, content):
309 f = open(filename, "w")
310 try:
311 f.write(content)
312 finally:
313 f.close()
314
315
316def RemoveDirectory(*path):
317 """Recursively removes a directory, even if it's marked read-only.
318
319 Remove the directory located at *path, if it exists.
320
321 shutil.rmtree() doesn't work on Windows if any of the files or directories
322 are read-only, which svn repositories and some .svn files are. We need to
323 be able to force the files to be writable (i.e., deletable) as we traverse
324 the tree.
325
326 Even with all this, Windows still sometimes fails to delete a file, citing
327 a permission error (maybe something to do with antivirus scans or disk
328 indexing). The best suggestion any of the user forums had was to wait a
329 bit and try again, so we do that too. It's hand-waving, but sometimes it
330 works. :/
331
332 On POSIX systems, things are a little bit simpler. The modes of the files
333 to be deleted doesn't matter, only the modes of the directories containing
334 them are significant. As the directory tree is traversed, each directory
335 has its mode set appropriately before descending into it. This should
336 result in the entire tree being removed, with the possible exception of
337 *path itself, because nothing attempts to change the mode of its parent.
338 Doing so would be hazardous, as it's not a directory slated for removal.
339 In the ordinary case, this is not a problem: for our purposes, the user
340 will never lack write permission on *path's parent.
341 """
342 file_path = os.path.join(*path)
343 if not os.path.exists(file_path):
344 return
345
346 if os.path.islink(file_path) or not os.path.isdir(file_path):
347 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
348
349 has_win32api = False
350 if sys.platform == 'win32':
351 has_win32api = True
352 # Some people don't have the APIs installed. In that case we'll do without.
353 try:
354 win32api = __import__('win32api')
355 win32con = __import__('win32con')
356 except ImportError:
357 has_win32api = False
358 else:
359 # On POSIX systems, we need the x-bit set on the directory to access it,
360 # the r-bit to see its contents, and the w-bit to remove files from it.
361 # The actual modes of the files within the directory is irrelevant.
362 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
363 for fn in os.listdir(file_path):
364 fullpath = os.path.join(file_path, fn)
365
366 # If fullpath is a symbolic link that points to a directory, isdir will
367 # be True, but we don't want to descend into that as a directory, we just
368 # want to remove the link. Check islink and treat links as ordinary files
369 # would be treated regardless of what they reference.
370 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
371 if sys.platform == 'win32':
372 os.chmod(fullpath, stat.S_IWRITE)
373 if has_win32api:
374 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
375 try:
376 os.remove(fullpath)
377 except OSError, e:
378 if e.errno != errno.EACCES or sys.platform != 'win32':
379 raise
380 print 'Failed to delete %s: trying again' % fullpath
381 time.sleep(0.1)
382 os.remove(fullpath)
383 else:
384 RemoveDirectory(fullpath)
385
386 if sys.platform == 'win32':
387 os.chmod(file_path, stat.S_IWRITE)
388 if has_win32api:
389 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
390 try:
391 os.rmdir(file_path)
392 except OSError, e:
393 if e.errno != errno.EACCES or sys.platform != 'win32':
394 raise
395 print 'Failed to remove %s: trying again' % file_path
396 time.sleep(0.1)
397 os.rmdir(file_path)
398
399
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000400def SubprocessCall(command, in_directory, fail_status=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000401 """Runs command, a list, in directory in_directory.
402
403 This function wraps SubprocessCallAndCapture, but does not perform the
404 capturing functions. See that function for a more complete usage
405 description.
406 """
407 # Call subprocess and capture nothing:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000408 SubprocessCallAndCapture(command, in_directory, fail_status)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000409
410
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000411def SubprocessCallAndCapture(command, in_directory, fail_status=None,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000412 pattern=None, capture_list=None):
413 """Runs command, a list, in directory in_directory.
414
415 A message indicating what is being done, as well as the command's stdout,
416 is printed to out.
417
418 If a pattern is specified, any line in the output matching pattern will have
419 its first match group appended to capture_list.
420
421 If the command fails, as indicated by a nonzero exit status, gclient will
422 exit with an exit status of fail_status. If fail_status is None (the
423 default), gclient will raise an Error exception.
424 """
425
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000426 print("\n________ running \'%s\' in \'%s\'"
427 % (' '.join(command), in_directory))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000428
429 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
430 # executable, but shell=True makes subprocess on Linux fail when it's called
431 # with a list because it only tries to execute the first item in the list.
432 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
433 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
434
435 if pattern:
436 compiled_pattern = re.compile(pattern)
437
438 # Also, we need to forward stdout to prevent weird re-ordering of output.
439 # This has to be done on a per byte basis to make sure it is not buffered:
440 # normally buffering is done for each line, but if svn requests input, no
441 # end-of-line character is output after the prompt and it would not show up.
442 in_byte = kid.stdout.read(1)
443 in_line = ""
444 while in_byte:
445 if in_byte != "\r":
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000446 sys.stdout.write(in_byte)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000447 in_line += in_byte
448 if in_byte == "\n" and pattern:
449 match = compiled_pattern.search(in_line[:-1])
450 if match:
451 capture_list.append(match.group(1))
452 in_line = ""
453 in_byte = kid.stdout.read(1)
454 rv = kid.wait()
455
456 if rv:
457 msg = "failed to run command: %s" % " ".join(command)
458
459 if fail_status != None:
460 print >>sys.stderr, msg
461 sys.exit(fail_status)
462
463 raise Error(msg)
464
465
466def IsUsingGit(root, paths):
467 """Returns True if we're using git to manage any of our checkouts.
468 |entries| is a list of paths to check."""
469 for path in paths:
470 if os.path.exists(os.path.join(root, path, '.git')):
471 return True
472 return False
473
474# -----------------------------------------------------------------------------
475# SVN utils:
476
477
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000478def RunSVN(args, in_directory):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000479 """Runs svn, sending output to stdout.
480
481 Args:
482 args: A sequence of command line parameters to be passed to svn.
483 in_directory: The directory where svn is to be run.
484
485 Raises:
486 Error: An error occurred while running the svn command.
487 """
488 c = [SVN_COMMAND]
489 c.extend(args)
490
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000491 SubprocessCall(c, in_directory)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000492
493
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000494def CaptureSVN(args, in_directory=None, print_error=True):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000495 """Runs svn, capturing output sent to stdout as a string.
496
497 Args:
498 args: A sequence of command line parameters to be passed to svn.
499 in_directory: The directory where svn is to be run.
500
501 Returns:
502 The output sent to stdout as a string.
503 """
504 c = [SVN_COMMAND]
505 c.extend(args)
506
507 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
508 # the svn.exe executable, but shell=True makes subprocess on Linux fail
509 # when it's called with a list because it only tries to execute the
510 # first string ("svn").
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000511 stderr = None
512 if print_error:
513 stderr = subprocess.PIPE
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000514 return subprocess.Popen(c,
515 cwd=in_directory,
516 shell=(sys.platform == 'win32'),
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000517 stdout=subprocess.PIPE,
518 stderr=stderr).communicate()[0]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000519
520
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000521def RunSVNAndGetFileList(args, in_directory, file_list):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000522 """Runs svn checkout, update, or status, output to stdout.
523
524 The first item in args must be either "checkout", "update", or "status".
525
526 svn's stdout is parsed to collect a list of files checked out or updated.
527 These files are appended to file_list. svn's stdout is also printed to
528 sys.stdout as in RunSVN.
529
530 Args:
531 args: A sequence of command line parameters to be passed to svn.
532 in_directory: The directory where svn is to be run.
533
534 Raises:
535 Error: An error occurred while running the svn command.
536 """
537 command = [SVN_COMMAND]
538 command.extend(args)
539
540 # svn update and svn checkout use the same pattern: the first three columns
541 # are for file status, property status, and lock status. This is followed
542 # by two spaces, and then the path to the file.
543 update_pattern = '^... (.*)$'
544
545 # The first three columns of svn status are the same as for svn update and
546 # svn checkout. The next three columns indicate addition-with-history,
547 # switch, and remote lock status. This is followed by one space, and then
548 # the path to the file.
549 status_pattern = '^...... (.*)$'
550
551 # args[0] must be a supported command. This will blow up if it's something
552 # else, which is good. Note that the patterns are only effective when
553 # these commands are used in their ordinary forms, the patterns are invalid
554 # for "svn status --show-updates", for example.
555 pattern = {
556 'checkout': update_pattern,
557 'status': status_pattern,
558 'update': update_pattern,
559 }[args[0]]
560
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000561 SubprocessCallAndCapture(command,
562 in_directory,
563 pattern=pattern,
564 capture_list=file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000565
566
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000567def CaptureSVNInfo(relpath, in_directory=None, print_error=True):
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000568 """Returns a dictionary from the svn info output for the given file.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000569
570 Args:
571 relpath: The directory where the working copy resides relative to
572 the directory given by in_directory.
573 in_directory: The directory where svn is to be run.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000574 """
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000575 output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000576 dom = ParseXML(output)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000577 result = {}
maruel@chromium.org483b0082009-05-07 02:57:14 +0000578 if dom:
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000579 def C(item, f):
580 if item is not None: return f(item)
maruel@chromium.org483b0082009-05-07 02:57:14 +0000581 # /info/entry/
582 # url
583 # reposityory/(root|uuid)
584 # wc-info/(schedule|depth)
585 # commit/(author|date)
586 # str() the results because they may be returned as Unicode, which
587 # interferes with the higher layers matching up things in the deps
588 # dictionary.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000589 # TODO(maruel): Fix at higher level instead (!)
590 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
591 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
592 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
593 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
594 int)
595 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
596 str)
597 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
598 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
599 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
600 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000601 return result
602
603
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000604def CaptureSVNHeadRevision(url):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000605 """Get the head revision of a SVN repository.
606
607 Returns:
608 Int head revision
609 """
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000610 info = CaptureSVN(["info", "--xml", url], os.getcwd())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000611 dom = xml.dom.minidom.parseString(info)
612 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
613
614
maruel@chromium.org4810a962009-05-12 21:03:34 +0000615def CaptureSVNStatus(files):
616 """Returns the svn 1.5 svn status emulated output.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000617
maruel@chromium.org4810a962009-05-12 21:03:34 +0000618 @files can be a string (one file) or a list of files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000619
maruel@chromium.org4810a962009-05-12 21:03:34 +0000620 Returns an array of (status, file) tuples."""
621 command = ["status", "--xml"]
622 if not files:
623 pass
624 elif isinstance(files, basestring):
625 command.append(files)
626 else:
627 command.extend(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000628
maruel@chromium.org4810a962009-05-12 21:03:34 +0000629 status_letter = {
630 None: ' ',
631 '': ' ',
632 'added': 'A',
633 'conflicted': 'C',
634 'deleted': 'D',
635 'external': 'X',
636 'ignored': 'I',
637 'incomplete': '!',
638 'merged': 'G',
639 'missing': '!',
640 'modified': 'M',
641 'none': ' ',
642 'normal': ' ',
643 'obstructed': '~',
644 'replaced': 'R',
645 'unversioned': '?',
646 }
647 dom = ParseXML(CaptureSVN(command))
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000648 results = []
649 if dom:
650 # /status/target/entry/(wc-status|commit|author|date)
651 for target in dom.getElementsByTagName('target'):
652 base_path = target.getAttribute('path')
653 for entry in target.getElementsByTagName('entry'):
654 file = entry.getAttribute('path')
655 wc_status = entry.getElementsByTagName('wc-status')
656 assert len(wc_status) == 1
657 # Emulate svn 1.5 status ouput...
658 statuses = [' ' for i in range(7)]
659 # Col 0
660 xml_item_status = wc_status[0].getAttribute('item')
maruel@chromium.org4810a962009-05-12 21:03:34 +0000661 if xml_item_status in status_letter:
662 statuses[0] = status_letter[xml_item_status]
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000663 else:
664 raise Exception('Unknown item status "%s"; please implement me!' %
665 xml_item_status)
666 # Col 1
667 xml_props_status = wc_status[0].getAttribute('props')
668 if xml_props_status == 'modified':
669 statuses[1] = 'M'
670 elif xml_props_status == 'conflicted':
671 statuses[1] = 'C'
672 elif (not xml_props_status or xml_props_status == 'none' or
673 xml_props_status == 'normal'):
674 pass
675 else:
676 raise Exception('Unknown props status "%s"; please implement me!' %
677 xml_props_status)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000678 # Col 2
679 if wc_status[0].getAttribute('wc-locked') == 'true':
680 statuses[2] = 'L'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000681 # Col 3
682 if wc_status[0].getAttribute('copied') == 'true':
683 statuses[3] = '+'
maruel@chromium.org4810a962009-05-12 21:03:34 +0000684 item = (''.join(statuses), file)
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000685 results.append(item)
686 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000687
688
689### SCM abstraction layer
690
691
692class SCMWrapper(object):
693 """Add necessary glue between all the supported SCM.
694
695 This is the abstraction layer to bind to different SCM. Since currently only
696 subversion is supported, a lot of subersionism remains. This can be sorted out
697 once another SCM is supported."""
698 def __init__(self, url=None, root_dir=None, relpath=None,
699 scm_name='svn'):
700 # TODO(maruel): Deduce the SCM from the url.
701 self.scm_name = scm_name
702 self.url = url
703 self._root_dir = root_dir
704 if self._root_dir:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000705 self._root_dir = self._root_dir.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000706 self.relpath = relpath
707 if self.relpath:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000708 self.relpath = self.relpath.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000709
710 def FullUrlForRelativeUrl(self, url):
711 # Find the forth '/' and strip from there. A bit hackish.
712 return '/'.join(self.url.split('/')[:4]) + url
713
714 def RunCommand(self, command, options, args, file_list=None):
715 # file_list will have all files that are modified appended to it.
716
717 if file_list == None:
718 file_list = []
719
720 commands = {
721 'cleanup': self.cleanup,
722 'update': self.update,
723 'revert': self.revert,
724 'status': self.status,
725 'diff': self.diff,
726 'runhooks': self.status,
727 }
728
729 if not command in commands:
730 raise Error('Unknown command %s' % command)
731
732 return commands[command](options, args, file_list)
733
734 def cleanup(self, options, args, file_list):
735 """Cleanup working copy."""
736 command = ['cleanup']
737 command.extend(args)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000738 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000739
740 def diff(self, options, args, file_list):
741 # NOTE: This function does not currently modify file_list.
742 command = ['diff']
743 command.extend(args)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000744 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000745
746 def update(self, options, args, file_list):
747 """Runs SCM to update or transparently checkout the working copy.
748
749 All updated files will be appended to file_list.
750
751 Raises:
752 Error: if can't get URL for relative path.
753 """
754 # Only update if git is not controlling the directory.
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000755 checkout_path = os.path.join(self._root_dir, self.relpath)
maruel@chromium.org0329e672009-05-13 18:41:04 +0000756 git_path = os.path.join(self._root_dir, self.relpath, '.git')
757 if os.path.exists(git_path):
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000758 print("________ found .git directory; skipping %s" % self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000759 return
760
761 if args:
762 raise Error("Unsupported argument(s): %s" % ",".join(args))
763
764 url = self.url
765 components = url.split("@")
766 revision = None
767 forced_revision = False
768 if options.revision:
769 # Override the revision number.
770 url = '%s@%s' % (components[0], str(options.revision))
771 revision = int(options.revision)
772 forced_revision = True
773 elif len(components) == 2:
774 revision = int(components[1])
775 forced_revision = True
776
777 rev_str = ""
778 if revision:
779 rev_str = ' at %d' % revision
780
maruel@chromium.org0329e672009-05-13 18:41:04 +0000781 if not os.path.exists(checkout_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000782 # We need to checkout.
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000783 command = ['checkout', url, checkout_path]
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000784 if revision:
785 command.extend(['--revision', str(revision)])
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000786 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000787 return
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000788
789 # Get the existing scm url and the revision number of the current checkout.
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000790 from_info = CaptureSVNInfo(os.path.join(checkout_path, '.'), '.')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000791
792 if options.manually_grab_svn_rev:
793 # Retrieve the current HEAD version because svn is slow at null updates.
794 if not revision:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000795 from_info_live = CaptureSVNInfo(from_info['URL'], '.')
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000796 revision = int(from_info_live['Revision'])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000797 rev_str = ' at %d' % revision
798
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000799 if from_info['URL'] != components[0]:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000800 to_info = CaptureSVNInfo(url, '.')
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000801 can_switch = ((from_info['Repository Root'] != to_info['Repository Root'])
802 and (from_info['UUID'] == to_info['UUID']))
803 if can_switch:
804 print("\n_____ relocating %s to a new checkout" % self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000805 # We have different roots, so check if we can switch --relocate.
806 # Subversion only permits this if the repository UUIDs match.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000807 # Perform the switch --relocate, then rewrite the from_url
808 # to reflect where we "are now." (This is the same way that
809 # Subversion itself handles the metadata when switch --relocate
810 # is used.) This makes the checks below for whether we
811 # can update to a revision or have to switch to a different
812 # branch work as expected.
813 # TODO(maruel): TEST ME !
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000814 command = ["switch", "--relocate",
815 from_info['Repository Root'],
816 to_info['Repository Root'],
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000817 self.relpath]
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000818 RunSVN(command, self._root_dir)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000819 from_info['URL'] = from_info['URL'].replace(
820 from_info['Repository Root'],
821 to_info['Repository Root'])
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000822 else:
823 if CaptureSVNStatus(checkout_path):
824 raise Error("Can't switch the checkout to %s; UUID don't match and "
825 "there is local changes in %s. Delete the directory and "
826 "try again." % (url, checkout_path))
827 # Ok delete it.
828 print("\n_____ switching %s to a new checkout" % self.relpath)
829 RemoveDirectory(checkout_path)
830 # We need to checkout.
831 command = ['checkout', url, checkout_path]
832 if revision:
833 command.extend(['--revision', str(revision)])
834 RunSVNAndGetFileList(command, self._root_dir, file_list)
835 return
836
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000837
838 # If the provided url has a revision number that matches the revision
839 # number of the existing directory, then we don't need to bother updating.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000840 if not options.force and from_info['Revision'] == revision:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000841 if options.verbose or not forced_revision:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000842 print("\n_____ %s%s" % (self.relpath, rev_str))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000843 return
844
maruel@chromium.org8626ff72009-05-13 02:57:02 +0000845 command = ["update", checkout_path]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000846 if revision:
847 command.extend(['--revision', str(revision)])
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000848 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000849
850 def revert(self, options, args, file_list):
851 """Reverts local modifications. Subversion specific.
852
853 All reverted files will be appended to file_list, even if Subversion
854 doesn't know about them.
855 """
856 path = os.path.join(self._root_dir, self.relpath)
857 if not os.path.isdir(path):
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000858 # svn revert won't work if the directory doesn't exist. It needs to
859 # checkout instead.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000860 print("\n_____ %s is missing, synching instead" % self.relpath)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000861 # Don't reuse the args.
862 return self.update(options, [], file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000863
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000864 files = CaptureSVNStatus(path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000865 # Batch the command.
866 files_to_revert = []
867 for file in files:
maruel@chromium.org4810a962009-05-12 21:03:34 +0000868 file_path = os.path.join(path, file[1])
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000869 print(file_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000870 # Unversioned file or unexpected unversioned file.
maruel@chromium.org4810a962009-05-12 21:03:34 +0000871 if file[0][0] in ('?', '~'):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000872 # Remove extraneous file. Also remove unexpected unversioned
873 # directories. svn won't touch them but we want to delete these.
874 file_list.append(file_path)
875 try:
876 os.remove(file_path)
877 except EnvironmentError:
878 RemoveDirectory(file_path)
879
maruel@chromium.org4810a962009-05-12 21:03:34 +0000880 if file[0][0] != '?':
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000881 # For any other status, svn revert will work.
882 file_list.append(file_path)
maruel@chromium.org4810a962009-05-12 21:03:34 +0000883 files_to_revert.append(file[1])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000884
885 # Revert them all at once.
886 if files_to_revert:
887 accumulated_paths = []
888 accumulated_length = 0
889 command = ['revert']
890 for p in files_to_revert:
891 # Some shell have issues with command lines too long.
892 if accumulated_length and accumulated_length + len(p) > 3072:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000893 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000894 os.path.join(self._root_dir, self.relpath))
895 accumulated_paths = []
896 accumulated_length = 0
897 else:
898 accumulated_paths.append(p)
899 accumulated_length += len(p)
900 if accumulated_paths:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000901 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000902 os.path.join(self._root_dir, self.relpath))
903
904 def status(self, options, args, file_list):
905 """Display status information."""
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000906 path = os.path.join(self._root_dir, self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000907 command = ['status']
908 command.extend(args)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000909 if not os.path.isdir(path):
910 # svn status won't work if the directory doesn't exist.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000911 print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
912 "does not exist."
913 % (' '.join(command), path))
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000914 # There's no file list to retrieve.
915 else:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000916 RunSVNAndGetFileList(command, path, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000917
918
919## GClient implementation.
920
921
922class GClient(object):
923 """Object that represent a gclient checkout."""
924
925 supported_commands = [
926 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
927 ]
928
929 def __init__(self, root_dir, options):
930 self._root_dir = root_dir
931 self._options = options
932 self._config_content = None
933 self._config_dict = {}
934 self._deps_hooks = []
935
936 def SetConfig(self, content):
937 self._config_dict = {}
938 self._config_content = content
939 exec(content, self._config_dict)
940
941 def SaveConfig(self):
942 FileWrite(os.path.join(self._root_dir, self._options.config_filename),
943 self._config_content)
944
945 def _LoadConfig(self):
946 client_source = FileRead(os.path.join(self._root_dir,
947 self._options.config_filename))
948 self.SetConfig(client_source)
949
950 def ConfigContent(self):
951 return self._config_content
952
953 def GetVar(self, key, default=None):
954 return self._config_dict.get(key, default)
955
956 @staticmethod
957 def LoadCurrentConfig(options, from_dir=None):
958 """Searches for and loads a .gclient file relative to the current working
959 dir.
960
961 Returns:
962 A dict representing the contents of the .gclient file or an empty dict if
963 the .gclient file doesn't exist.
964 """
965 if not from_dir:
966 from_dir = os.curdir
967 path = os.path.realpath(from_dir)
maruel@chromium.org0329e672009-05-13 18:41:04 +0000968 while not os.path.exists(os.path.join(path, options.config_filename)):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000969 next = os.path.split(path)
970 if not next[1]:
971 return None
972 path = next[0]
973 client = options.gclient(path, options)
974 client._LoadConfig()
975 return client
976
977 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
978 self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
979 solution_name, solution_url, safesync_url
980 ))
981
982 def _SaveEntries(self, entries):
983 """Creates a .gclient_entries file to record the list of unique checkouts.
984
985 The .gclient_entries file lives in the same directory as .gclient.
986
987 Args:
988 entries: A sequence of solution names.
989 """
990 text = "entries = [\n"
991 for entry in entries:
992 text += " \"%s\",\n" % entry
993 text += "]\n"
994 FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
995 text)
996
997 def _ReadEntries(self):
998 """Read the .gclient_entries file for the given client.
999
1000 Args:
1001 client: The client for which the entries file should be read.
1002
1003 Returns:
1004 A sequence of solution names, which will be empty if there is the
1005 entries file hasn't been created yet.
1006 """
1007 scope = {}
1008 filename = os.path.join(self._root_dir, self._options.entries_filename)
maruel@chromium.org0329e672009-05-13 18:41:04 +00001009 if not os.path.exists(filename):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001010 return []
1011 exec(FileRead(filename), scope)
1012 return scope["entries"]
1013
1014 class FromImpl:
1015 """Used to implement the From syntax."""
1016
1017 def __init__(self, module_name):
1018 self.module_name = module_name
1019
1020 def __str__(self):
1021 return 'From("%s")' % self.module_name
1022
1023 class _VarImpl:
1024 def __init__(self, custom_vars, local_scope):
1025 self._custom_vars = custom_vars
1026 self._local_scope = local_scope
1027
1028 def Lookup(self, var_name):
1029 """Implements the Var syntax."""
1030 if var_name in self._custom_vars:
1031 return self._custom_vars[var_name]
1032 elif var_name in self._local_scope.get("vars", {}):
1033 return self._local_scope["vars"][var_name]
1034 raise Error("Var is not defined: %s" % var_name)
1035
1036 def _ParseSolutionDeps(self, solution_name, solution_deps_content,
1037 custom_vars):
1038 """Parses the DEPS file for the specified solution.
1039
1040 Args:
1041 solution_name: The name of the solution to query.
1042 solution_deps_content: Content of the DEPS file for the solution
1043 custom_vars: A dict of vars to override any vars defined in the DEPS file.
1044
1045 Returns:
1046 A dict mapping module names (as relative paths) to URLs or an empty
1047 dict if the solution does not have a DEPS file.
1048 """
1049 # Skip empty
1050 if not solution_deps_content:
1051 return {}
1052 # Eval the content
1053 local_scope = {}
1054 var = self._VarImpl(custom_vars, local_scope)
1055 global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
1056 exec(solution_deps_content, global_scope, local_scope)
1057 deps = local_scope.get("deps", {})
1058
1059 # load os specific dependencies if defined. these dependencies may
1060 # override or extend the values defined by the 'deps' member.
1061 if "deps_os" in local_scope:
1062 deps_os_choices = {
1063 "win32": "win",
1064 "win": "win",
1065 "cygwin": "win",
1066 "darwin": "mac",
1067 "mac": "mac",
1068 "unix": "unix",
1069 "linux": "unix",
1070 "linux2": "unix",
1071 }
1072
1073 if self._options.deps_os is not None:
1074 deps_to_include = self._options.deps_os.split(",")
1075 if "all" in deps_to_include:
1076 deps_to_include = deps_os_choices.values()
1077 else:
1078 deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
1079
1080 deps_to_include = set(deps_to_include)
1081 for deps_os_key in deps_to_include:
1082 os_deps = local_scope["deps_os"].get(deps_os_key, {})
1083 if len(deps_to_include) > 1:
1084 # Ignore any overrides when including deps for more than one
1085 # platform, so we collect the broadest set of dependencies available.
1086 # We may end up with the wrong revision of something for our
1087 # platform, but this is the best we can do.
1088 deps.update([x for x in os_deps.items() if not x[0] in deps])
1089 else:
1090 deps.update(os_deps)
1091
1092 if 'hooks' in local_scope:
1093 self._deps_hooks.extend(local_scope['hooks'])
1094
1095 # If use_relative_paths is set in the DEPS file, regenerate
1096 # the dictionary using paths relative to the directory containing
1097 # the DEPS file.
1098 if local_scope.get('use_relative_paths'):
1099 rel_deps = {}
1100 for d, url in deps.items():
1101 # normpath is required to allow DEPS to use .. in their
1102 # dependency local path.
1103 rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
1104 return rel_deps
1105 else:
1106 return deps
1107
1108 def _ParseAllDeps(self, solution_urls, solution_deps_content):
1109 """Parse the complete list of dependencies for the client.
1110
1111 Args:
1112 solution_urls: A dict mapping module names (as relative paths) to URLs
1113 corresponding to the solutions specified by the client. This parameter
1114 is passed as an optimization.
1115 solution_deps_content: A dict mapping module names to the content
1116 of their DEPS files
1117
1118 Returns:
1119 A dict mapping module names (as relative paths) to URLs corresponding
1120 to the entire set of dependencies to checkout for the given client.
1121
1122 Raises:
1123 Error: If a dependency conflicts with another dependency or of a solution.
1124 """
1125 deps = {}
1126 for solution in self.GetVar("solutions"):
1127 custom_vars = solution.get("custom_vars", {})
1128 solution_deps = self._ParseSolutionDeps(
1129 solution["name"],
1130 solution_deps_content[solution["name"]],
1131 custom_vars)
1132
1133 # If a line is in custom_deps, but not in the solution, we want to append
1134 # this line to the solution.
1135 if "custom_deps" in solution:
1136 for d in solution["custom_deps"]:
1137 if d not in solution_deps:
1138 solution_deps[d] = solution["custom_deps"][d]
1139
1140 for d in solution_deps:
1141 if "custom_deps" in solution and d in solution["custom_deps"]:
1142 # Dependency is overriden.
1143 url = solution["custom_deps"][d]
1144 if url is None:
1145 continue
1146 else:
1147 url = solution_deps[d]
1148 # if we have a From reference dependent on another solution, then
1149 # just skip the From reference. When we pull deps for the solution,
1150 # we will take care of this dependency.
1151 #
1152 # If multiple solutions all have the same From reference, then we
1153 # should only add one to our list of dependencies.
1154 if type(url) != str:
1155 if url.module_name in solution_urls:
1156 # Already parsed.
1157 continue
1158 if d in deps and type(deps[d]) != str:
1159 if url.module_name == deps[d].module_name:
1160 continue
1161 else:
1162 parsed_url = urlparse.urlparse(url)
1163 scheme = parsed_url[0]
1164 if not scheme:
1165 # A relative url. Fetch the real base.
1166 path = parsed_url[2]
1167 if path[0] != "/":
1168 raise Error(
1169 "relative DEPS entry \"%s\" must begin with a slash" % d)
1170 # Create a scm just to query the full url.
1171 scm = self._options.scm_wrapper(solution["url"], self._root_dir,
1172 None)
1173 url = scm.FullUrlForRelativeUrl(url)
1174 if d in deps and deps[d] != url:
1175 raise Error(
1176 "Solutions have conflicting versions of dependency \"%s\"" % d)
1177 if d in solution_urls and solution_urls[d] != url:
1178 raise Error(
1179 "Dependency \"%s\" conflicts with specified solution" % d)
1180 # Grab the dependency.
1181 deps[d] = url
1182 return deps
1183
1184 def _RunHookAction(self, hook_dict):
1185 """Runs the action from a single hook.
1186 """
1187 command = hook_dict['action'][:]
1188 if command[0] == 'python':
1189 # If the hook specified "python" as the first item, the action is a
1190 # Python script. Run it by starting a new copy of the same
1191 # interpreter.
1192 command[0] = sys.executable
1193
1194 # Use a discrete exit status code of 2 to indicate that a hook action
1195 # failed. Users of this script may wish to treat hook action failures
1196 # differently from VC failures.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001197 SubprocessCall(command, self._root_dir, fail_status=2)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001198
1199 def _RunHooks(self, command, file_list, is_using_git):
1200 """Evaluates all hooks, running actions as needed.
1201 """
1202 # Hooks only run for these command types.
1203 if not command in ('update', 'revert', 'runhooks'):
1204 return
1205
1206 # Get any hooks from the .gclient file.
1207 hooks = self.GetVar("hooks", [])
1208 # Add any hooks found in DEPS files.
1209 hooks.extend(self._deps_hooks)
1210
1211 # If "--force" was specified, run all hooks regardless of what files have
1212 # changed. If the user is using git, then we don't know what files have
1213 # changed so we always run all hooks.
1214 if self._options.force or is_using_git:
1215 for hook_dict in hooks:
1216 self._RunHookAction(hook_dict)
1217 return
1218
1219 # Run hooks on the basis of whether the files from the gclient operation
1220 # match each hook's pattern.
1221 for hook_dict in hooks:
1222 pattern = re.compile(hook_dict['pattern'])
1223 for file in file_list:
1224 if not pattern.search(file):
1225 continue
1226
1227 self._RunHookAction(hook_dict)
1228
1229 # The hook's action only runs once. Don't bother looking for any
1230 # more matches.
1231 break
1232
1233 def RunOnDeps(self, command, args):
1234 """Runs a command on each dependency in a client and its dependencies.
1235
1236 The module's dependencies are specified in its top-level DEPS files.
1237
1238 Args:
1239 command: The command to use (e.g., 'status' or 'diff')
1240 args: list of str - extra arguments to add to the command line.
1241
1242 Raises:
1243 Error: If the client has conflicting entries.
1244 """
1245 if not command in self.supported_commands:
1246 raise Error("'%s' is an unsupported command" % command)
1247
1248 # Check for revision overrides.
1249 revision_overrides = {}
1250 for revision in self._options.revisions:
1251 if revision.find("@") == -1:
1252 raise Error(
1253 "Specify the full dependency when specifying a revision number.")
1254 revision_elem = revision.split("@")
1255 # Disallow conflicting revs
1256 if revision_overrides.has_key(revision_elem[0]) and \
1257 revision_overrides[revision_elem[0]] != revision_elem[1]:
1258 raise Error(
1259 "Conflicting revision numbers specified.")
1260 revision_overrides[revision_elem[0]] = revision_elem[1]
1261
1262 solutions = self.GetVar("solutions")
1263 if not solutions:
1264 raise Error("No solution specified")
1265
1266 # When running runhooks --force, there's no need to consult the SCM.
1267 # All known hooks are expected to run unconditionally regardless of working
1268 # copy state, so skip the SCM status check.
1269 run_scm = not (command == 'runhooks' and self._options.force)
1270
1271 entries = {}
1272 entries_deps_content = {}
1273 file_list = []
1274 # Run on the base solutions first.
1275 for solution in solutions:
1276 name = solution["name"]
1277 if name in entries:
1278 raise Error("solution %s specified more than once" % name)
1279 url = solution["url"]
1280 entries[name] = url
1281 if run_scm:
1282 self._options.revision = revision_overrides.get(name)
1283 scm = self._options.scm_wrapper(url, self._root_dir, name)
1284 scm.RunCommand(command, self._options, args, file_list)
1285 self._options.revision = None
1286 try:
1287 deps_content = FileRead(os.path.join(self._root_dir, name,
1288 self._options.deps_file))
1289 except IOError, e:
1290 if e.errno != errno.ENOENT:
1291 raise
1292 deps_content = ""
1293 entries_deps_content[name] = deps_content
1294
1295 # Process the dependencies next (sort alphanumerically to ensure that
1296 # containing directories get populated first and for readability)
1297 deps = self._ParseAllDeps(entries, entries_deps_content)
1298 deps_to_process = deps.keys()
1299 deps_to_process.sort()
1300
1301 # First pass for direct dependencies.
1302 for d in deps_to_process:
1303 if type(deps[d]) == str:
1304 url = deps[d]
1305 entries[d] = url
1306 if run_scm:
1307 self._options.revision = revision_overrides.get(d)
1308 scm = self._options.scm_wrapper(url, self._root_dir, d)
1309 scm.RunCommand(command, self._options, args, file_list)
1310 self._options.revision = None
1311
1312 # Second pass for inherited deps (via the From keyword)
1313 for d in deps_to_process:
1314 if type(deps[d]) != str:
1315 sub_deps = self._ParseSolutionDeps(
1316 deps[d].module_name,
1317 FileRead(os.path.join(self._root_dir,
1318 deps[d].module_name,
1319 self._options.deps_file)),
1320 {})
1321 url = sub_deps[d]
1322 entries[d] = url
1323 if run_scm:
1324 self._options.revision = revision_overrides.get(d)
1325 scm = self._options.scm_wrapper(url, self._root_dir, d)
1326 scm.RunCommand(command, self._options, args, file_list)
1327 self._options.revision = None
1328
1329 is_using_git = IsUsingGit(self._root_dir, entries.keys())
1330 self._RunHooks(command, file_list, is_using_git)
1331
1332 if command == 'update':
1333 # notify the user if there is an orphaned entry in their working copy.
1334 # TODO(darin): we should delete this directory manually if it doesn't
1335 # have any changes in it.
1336 prev_entries = self._ReadEntries()
1337 for entry in prev_entries:
1338 e_dir = os.path.join(self._root_dir, entry)
maruel@chromium.org0329e672009-05-13 18:41:04 +00001339 if entry not in entries and os.path.exists(e_dir):
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001340 if CaptureSVNStatus(e_dir):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001341 # There are modified files in this entry
1342 entries[entry] = None # Keep warning until removed.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001343 print("\nWARNING: \"%s\" is no longer part of this client. "
1344 "It is recommended that you manually remove it.\n") % entry
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001345 else:
1346 # Delete the entry
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001347 print("\n________ deleting \'%s\' " +
1348 "in \'%s\'") % (entry, self._root_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001349 RemoveDirectory(e_dir)
1350 # record the current list of entries for next time
1351 self._SaveEntries(entries)
1352
1353 def PrintRevInfo(self):
1354 """Output revision info mapping for the client and its dependencies. This
1355 allows the capture of a overall "revision" for the source tree that can
1356 be used to reproduce the same tree in the future. The actual output
1357 contains enough information (source paths, svn server urls and revisions)
1358 that it can be used either to generate external svn commands (without
1359 gclient) or as input to gclient's --rev option (with some massaging of
1360 the data).
1361
1362 NOTE: Unlike RunOnDeps this does not require a local checkout and is run
1363 on the Pulse master. It MUST NOT execute hooks.
1364
1365 Raises:
1366 Error: If the client has conflicting entries.
1367 """
1368 # Check for revision overrides.
1369 revision_overrides = {}
1370 for revision in self._options.revisions:
1371 if revision.find("@") < 0:
1372 raise Error(
1373 "Specify the full dependency when specifying a revision number.")
1374 revision_elem = revision.split("@")
1375 # Disallow conflicting revs
1376 if revision_overrides.has_key(revision_elem[0]) and \
1377 revision_overrides[revision_elem[0]] != revision_elem[1]:
1378 raise Error(
1379 "Conflicting revision numbers specified.")
1380 revision_overrides[revision_elem[0]] = revision_elem[1]
1381
1382 solutions = self.GetVar("solutions")
1383 if not solutions:
1384 raise Error("No solution specified")
1385
1386 entries = {}
1387 entries_deps_content = {}
1388
1389 # Inner helper to generate base url and rev tuple (including honoring
1390 # |revision_overrides|)
1391 def GetURLAndRev(name, original_url):
1392 if original_url.find("@") < 0:
1393 if revision_overrides.has_key(name):
1394 return (original_url, int(revision_overrides[name]))
1395 else:
1396 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001397 return (original_url, CaptureSVNHeadRevision(original_url))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001398 else:
1399 url_components = original_url.split("@")
1400 if revision_overrides.has_key(name):
1401 return (url_components[0], int(revision_overrides[name]))
1402 else:
1403 return (url_components[0], int(url_components[1]))
1404
1405 # Run on the base solutions first.
1406 for solution in solutions:
1407 name = solution["name"]
1408 if name in entries:
1409 raise Error("solution %s specified more than once" % name)
1410 (url, rev) = GetURLAndRev(name, solution["url"])
1411 entries[name] = "%s@%d" % (url, rev)
1412 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1413 entries_deps_content[name] = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001414 ["cat",
1415 "%s/%s@%d" % (url,
1416 self._options.deps_file,
1417 rev)],
1418 os.getcwd())
1419
1420 # Process the dependencies next (sort alphanumerically to ensure that
1421 # containing directories get populated first and for readability)
1422 deps = self._ParseAllDeps(entries, entries_deps_content)
1423 deps_to_process = deps.keys()
1424 deps_to_process.sort()
1425
1426 # First pass for direct dependencies.
1427 for d in deps_to_process:
1428 if type(deps[d]) == str:
1429 (url, rev) = GetURLAndRev(d, deps[d])
1430 entries[d] = "%s@%d" % (url, rev)
1431
1432 # Second pass for inherited deps (via the From keyword)
1433 for d in deps_to_process:
1434 if type(deps[d]) != str:
1435 deps_parent_url = entries[deps[d].module_name]
1436 if deps_parent_url.find("@") < 0:
1437 raise Error("From %s missing revisioned url" % deps[d].module_name)
1438 deps_parent_url_components = deps_parent_url.split("@")
1439 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1440 deps_parent_content = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001441 ["cat",
1442 "%s/%s@%s" % (deps_parent_url_components[0],
1443 self._options.deps_file,
1444 deps_parent_url_components[1])],
1445 os.getcwd())
1446 sub_deps = self._ParseSolutionDeps(
1447 deps[d].module_name,
1448 FileRead(os.path.join(self._root_dir,
1449 deps[d].module_name,
1450 self._options.deps_file)),
1451 {})
1452 (url, rev) = GetURLAndRev(d, sub_deps[d])
1453 entries[d] = "%s@%d" % (url, rev)
1454
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001455 print(";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())]))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001456
1457
1458## gclient commands.
1459
1460
1461def DoCleanup(options, args):
1462 """Handle the cleanup subcommand.
1463
1464 Raises:
1465 Error: if client isn't configured properly.
1466 """
1467 client = options.gclient.LoadCurrentConfig(options)
1468 if not client:
1469 raise Error("client not configured; see 'gclient config'")
1470 if options.verbose:
1471 # Print out the .gclient file. This is longer than if we just printed the
1472 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001473 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001474 options.verbose = True
1475 return client.RunOnDeps('cleanup', args)
1476
1477
1478def DoConfig(options, args):
1479 """Handle the config subcommand.
1480
1481 Args:
1482 options: If options.spec set, a string providing contents of config file.
1483 args: The command line args. If spec is not set,
1484 then args[0] is a string URL to get for config file.
1485
1486 Raises:
1487 Error: on usage error
1488 """
1489 if len(args) < 1 and not options.spec:
1490 raise Error("required argument missing; see 'gclient help config'")
maruel@chromium.org0329e672009-05-13 18:41:04 +00001491 if os.path.exists(options.config_filename):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001492 raise Error("%s file already exists in the current directory" %
1493 options.config_filename)
1494 client = options.gclient('.', options)
1495 if options.spec:
1496 client.SetConfig(options.spec)
1497 else:
1498 # TODO(darin): it would be nice to be able to specify an alternate relpath
1499 # for the given URL.
1500 base_url = args[0]
1501 name = args[0].split("/")[-1]
1502 safesync_url = ""
1503 if len(args) > 1:
1504 safesync_url = args[1]
1505 client.SetDefaultConfig(name, base_url, safesync_url)
1506 client.SaveConfig()
1507
1508
1509def DoHelp(options, args):
1510 """Handle the help subcommand giving help for another subcommand.
1511
1512 Raises:
1513 Error: if the command is unknown.
1514 """
1515 if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001516 print(COMMAND_USAGE_TEXT[args[0]])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001517 else:
1518 raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
1519
1520
1521def DoStatus(options, args):
1522 """Handle the status subcommand.
1523
1524 Raises:
1525 Error: if client isn't configured properly.
1526 """
1527 client = options.gclient.LoadCurrentConfig(options)
1528 if not client:
1529 raise Error("client not configured; see 'gclient config'")
1530 if options.verbose:
1531 # Print out the .gclient file. This is longer than if we just printed the
1532 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001533 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001534 options.verbose = True
1535 return client.RunOnDeps('status', args)
1536
1537
1538def DoUpdate(options, args):
1539 """Handle the update and sync subcommands.
1540
1541 Raises:
1542 Error: if client isn't configured properly.
1543 """
1544 client = options.gclient.LoadCurrentConfig(options)
1545
1546 if not client:
1547 raise Error("client not configured; see 'gclient config'")
1548
1549 if not options.head:
1550 solutions = client.GetVar('solutions')
1551 if solutions:
1552 for s in solutions:
1553 if s.get('safesync_url', ''):
1554 # rip through revisions and make sure we're not over-riding
1555 # something that was explicitly passed
1556 has_key = False
1557 for r in options.revisions:
1558 if r.split('@')[0] == s['name']:
1559 has_key = True
1560 break
1561
1562 if not has_key:
1563 handle = urllib.urlopen(s['safesync_url'])
1564 rev = handle.read().strip()
1565 handle.close()
1566 if len(rev):
1567 options.revisions.append(s['name']+'@'+rev)
1568
1569 if options.verbose:
1570 # Print out the .gclient file. This is longer than if we just printed the
1571 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001572 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001573 return client.RunOnDeps('update', args)
1574
1575
1576def DoDiff(options, args):
1577 """Handle the diff subcommand.
1578
1579 Raises:
1580 Error: if client isn't configured properly.
1581 """
1582 client = options.gclient.LoadCurrentConfig(options)
1583 if not client:
1584 raise Error("client not configured; see 'gclient config'")
1585 if options.verbose:
1586 # Print out the .gclient file. This is longer than if we just printed the
1587 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001588 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001589 options.verbose = True
1590 return client.RunOnDeps('diff', args)
1591
1592
1593def DoRevert(options, args):
1594 """Handle the revert subcommand.
1595
1596 Raises:
1597 Error: if client isn't configured properly.
1598 """
1599 client = options.gclient.LoadCurrentConfig(options)
1600 if not client:
1601 raise Error("client not configured; see 'gclient config'")
1602 return client.RunOnDeps('revert', args)
1603
1604
1605def DoRunHooks(options, args):
1606 """Handle the runhooks subcommand.
1607
1608 Raises:
1609 Error: if client isn't configured properly.
1610 """
1611 client = options.gclient.LoadCurrentConfig(options)
1612 if not client:
1613 raise Error("client not configured; see 'gclient config'")
1614 if options.verbose:
1615 # Print out the .gclient file. This is longer than if we just printed the
1616 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001617 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001618 return client.RunOnDeps('runhooks', args)
1619
1620
1621def DoRevInfo(options, args):
1622 """Handle the revinfo subcommand.
1623
1624 Raises:
1625 Error: if client isn't configured properly.
1626 """
1627 client = options.gclient.LoadCurrentConfig(options)
1628 if not client:
1629 raise Error("client not configured; see 'gclient config'")
1630 client.PrintRevInfo()
1631
1632
1633gclient_command_map = {
1634 "cleanup": DoCleanup,
1635 "config": DoConfig,
1636 "diff": DoDiff,
1637 "help": DoHelp,
1638 "status": DoStatus,
1639 "sync": DoUpdate,
1640 "update": DoUpdate,
1641 "revert": DoRevert,
1642 "runhooks": DoRunHooks,
1643 "revinfo" : DoRevInfo,
1644}
1645
1646
1647def DispatchCommand(command, options, args, command_map=None):
1648 """Dispatches the appropriate subcommand based on command line arguments."""
1649 if command_map is None:
1650 command_map = gclient_command_map
1651
1652 if command in command_map:
1653 return command_map[command](options, args)
1654 else:
1655 raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
1656
1657
1658def Main(argv):
1659 """Parse command line arguments and dispatch command."""
1660
1661 option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
1662 version=__version__)
1663 option_parser.disable_interspersed_args()
1664 option_parser.add_option("", "--force", action="store_true", default=False,
1665 help=("(update/sync only) force update even "
1666 "for modules which haven't changed"))
1667 option_parser.add_option("", "--revision", action="append", dest="revisions",
1668 metavar="REV", default=[],
1669 help=("(update/sync only) sync to a specific "
1670 "revision, can be used multiple times for "
1671 "each solution, e.g. --revision=src@123, "
1672 "--revision=internal@32"))
1673 option_parser.add_option("", "--deps", default=None, dest="deps_os",
1674 metavar="OS_LIST",
1675 help=("(update/sync only) sync deps for the "
1676 "specified (comma-separated) platform(s); "
1677 "'all' will sync all platforms"))
1678 option_parser.add_option("", "--spec", default=None,
1679 help=("(config only) create a gclient file "
1680 "containing the provided string"))
1681 option_parser.add_option("", "--verbose", action="store_true", default=False,
1682 help="produce additional output for diagnostics")
1683 option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
1684 default=False,
1685 help="Skip svn up whenever possible by requesting "
1686 "actual HEAD revision from the repository")
1687 option_parser.add_option("", "--head", action="store_true", default=False,
1688 help=("skips any safesync_urls specified in "
1689 "configured solutions"))
1690
1691 if len(argv) < 2:
1692 # Users don't need to be told to use the 'help' command.
1693 option_parser.print_help()
1694 return 1
1695 # Add manual support for --version as first argument.
1696 if argv[1] == '--version':
1697 option_parser.print_version()
1698 return 0
1699
1700 # Add manual support for --help as first argument.
1701 if argv[1] == '--help':
1702 argv[1] = 'help'
1703
1704 command = argv[1]
1705 options, args = option_parser.parse_args(argv[2:])
1706
1707 if len(argv) < 3 and command == "help":
1708 option_parser.print_help()
1709 return 0
1710
1711 # Files used for configuration and state saving.
1712 options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
1713 options.entries_filename = ".gclient_entries"
1714 options.deps_file = "DEPS"
1715
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001716 options.gclient = GClient
1717 options.scm_wrapper = SCMWrapper
1718 options.platform = sys.platform
1719 return DispatchCommand(command, options, args)
1720
1721
1722if "__main__" == __name__:
1723 try:
1724 result = Main(sys.argv)
1725 except Error, e:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001726 print >> sys.stderr, "Error: %s" % str(e)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001727 result = 1
1728 sys.exit(result)
1729
1730# vim: ts=2:sw=2:tw=80:et: