blob: f190b4e1f1d97820c833a5dbf3154744208b1199 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A wrapper script to manage a set of client modules in different SCM.
18
19This script is intended to be used to help basic management of client
20program sources residing in one or more Subversion modules, along with
21other modules it depends on, also in Subversion, but possibly on
22multiple respositories, making a wrapper system apparently necessary.
23
24Files
25 .gclient : Current client configuration, written by 'config' command.
26 Format is a Python script defining 'solutions', a list whose
27 entries each are maps binding the strings "name" and "url"
28 to strings specifying the name and location of the client
29 module, as well as "custom_deps" to a map similar to the DEPS
30 file below.
31 .gclient_entries : A cache constructed by 'update' command. Format is a
32 Python script defining 'entries', a list of the names
33 of all modules in the client
34 <module>/DEPS : Python script defining var 'deps' as a map from each requisite
35 submodule name to a URL where it can be found (via one SCM)
36
37Hooks
38 .gclient and DEPS files may optionally contain a list named "hooks" to
39 allow custom actions to be performed based on files that have changed in the
40 working copy as a result of a "sync"/"update" or "revert" operation. Hooks
41 can also be run based on what files have been modified in the working copy
42 with the "runhooks" operation. If any of these operation are run with
43 --force, all known hooks will run regardless of the state of the working
44 copy.
45
46 Each item in a "hooks" list is a dict, containing these two keys:
47 "pattern" The associated value is a string containing a regular
48 expression. When a file whose pathname matches the expression
49 is checked out, updated, or reverted, the hook's "action" will
50 run.
51 "action" A list describing a command to run along with its arguments, if
52 any. An action command will run at most one time per gclient
53 invocation, regardless of how many files matched the pattern.
54 The action is executed in the same directory as the .gclient
55 file. If the first item in the list is the string "python",
56 the current Python interpreter (sys.executable) will be used
57 to run the command.
58
59 Example:
60 hooks = [
61 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
62 "action": ["python", "image_indexer.py", "--all"]},
63 ]
64"""
65
66__author__ = "darinf@gmail.com (Darin Fisher)"
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +000067__version__ = "0.3.2"
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000068
69import errno
70import optparse
71import os
72import re
73import stat
74import subprocess
75import sys
76import time
77import urlparse
78import xml.dom.minidom
79import urllib
80
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000081
82SVN_COMMAND = "svn"
83
84
85# default help text
86DEFAULT_USAGE_TEXT = (
87"""usage: %prog <subcommand> [options] [--] [svn options/args...]
88a wrapper for managing a set of client modules in svn.
89Version """ + __version__ + """
90
91subcommands:
92 cleanup
93 config
94 diff
95 revert
96 status
97 sync
98 update
99 runhooks
100 revinfo
101
102Options and extra arguments can be passed to invoked svn commands by
103appending them to the command line. Note that if the first such
104appended option starts with a dash (-) then the options must be
105preceded by -- to distinguish them from gclient options.
106
107For additional help on a subcommand or examples of usage, try
108 %prog help <subcommand>
109 %prog help files
110""")
111
112GENERIC_UPDATE_USAGE_TEXT = (
113 """Perform a checkout/update of the modules specified by the gclient
114configuration; see 'help config'. Unless --revision is specified,
115then the latest revision of the root solutions is checked out, with
116dependent submodule versions updated according to DEPS files.
117If --revision is specified, then the given revision is used in place
118of the latest, either for a single solution or for all solutions.
119Unless the --force option is provided, solutions and modules whose
120local revision matches the one to update (i.e., they have not changed
121in the repository) are *not* modified.
122This a synonym for 'gclient %(alias)s'
123
124usage: gclient %(cmd)s [options] [--] [svn update options/args]
125
126Valid options:
127 --force : force update even for unchanged modules
128 --revision REV : update/checkout all solutions with specified revision
129 --revision SOLUTION@REV : update given solution to specified revision
130 --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
131 --verbose : output additional diagnostics
132
133Examples:
134 gclient %(cmd)s
135 update files from SVN according to current configuration,
136 *for modules which have changed since last update or sync*
137 gclient %(cmd)s --force
138 update files from SVN according to current configuration, for
139 all modules (useful for recovering files deleted from local copy)
140""")
141
142COMMAND_USAGE_TEXT = {
143 "cleanup":
144 """Clean up all working copies, using 'svn cleanup' for each module.
145Additional options and args may be passed to 'svn cleanup'.
146
147usage: cleanup [options] [--] [svn cleanup args/options]
148
149Valid options:
150 --verbose : output additional diagnostics
151""",
152 "config": """Create a .gclient file in the current directory; this
153specifies the configuration for further commands. After update/sync,
154top-level DEPS files in each module are read to determine dependent
155modules to operate on as well. If optional [url] parameter is
156provided, then configuration is read from a specified Subversion server
157URL. Otherwise, a --spec option must be provided.
158
159usage: config [option | url] [safesync url]
160
161Valid options:
162 --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
163 *Note that due to Cygwin/Python brokenness, it
164 probably can't contain any newlines.*
165
166Examples:
167 gclient config https://gclient.googlecode.com/svn/trunk/gclient
168 configure a new client to check out gclient.py tool sources
169 gclient config --spec='solutions=[{"name":"gclient","""
170 '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
171 '"custom_deps":{}}]',
172 "diff": """Display the differences between two revisions of modules.
173(Does 'svn diff' for each checked out module and dependences.)
174Additional args and options to 'svn diff' can be passed after
175gclient options.
176
177usage: diff [options] [--] [svn args/options]
178
179Valid options:
180 --verbose : output additional diagnostics
181
182Examples:
183 gclient diff
184 simple 'svn diff' for configured client and dependences
185 gclient diff -- -x -b
186 use 'svn diff -x -b' to suppress whitespace-only differences
187 gclient diff -- -r HEAD -x -b
188 diff versus the latest version of each module
189""",
190 "revert":
191 """Revert every file in every managed directory in the client view.
192
193usage: revert
194""",
195 "status":
196 """Show the status of client and dependent modules, using 'svn diff'
197for each module. Additional options and args may be passed to 'svn diff'.
198
199usage: status [options] [--] [svn diff args/options]
200
201Valid options:
202 --verbose : output additional diagnostics
203""",
204 "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
205 "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
206 "help": """Describe the usage of this program or its subcommands.
207
208usage: help [options] [subcommand]
209
210Valid options:
211 --verbose : output additional diagnostics
212""",
213 "runhooks":
214 """Runs hooks for files that have been modified in the local working copy,
215according to 'svn status'.
216
217usage: runhooks [options]
218
219Valid options:
220 --force : runs all known hooks, regardless of the working
221 copy status
222 --verbose : output additional diagnostics
223""",
224 "revinfo":
225 """Outputs source path, server URL and revision information for every
226dependency in all solutions (no local checkout required).
227
228usage: revinfo [options]
229""",
230}
231
232# parameterized by (solution_name, solution_url, safesync_url)
233DEFAULT_CLIENT_FILE_TEXT = (
234 """
235# An element of this array (a \"solution\") describes a repository directory
236# that will be checked out into your working copy. Each solution may
237# optionally define additional dependencies (via its DEPS file) to be
238# checked out alongside the solution's directory. A solution may also
239# specify custom dependencies (via the \"custom_deps\" property) that
240# override or augment the dependencies specified by the DEPS file.
241# If a \"safesync_url\" is specified, it is assumed to reference the location of
242# a text file which contains nothing but the last known good SCM revision to
243# sync against. It is fetched if specified and used unless --head is passed
244solutions = [
245 { \"name\" : \"%s\",
246 \"url\" : \"%s\",
247 \"custom_deps\" : {
248 # To use the trunk of a component instead of what's in DEPS:
249 #\"component\": \"https://svnserver/component/trunk/\",
250 # To exclude a component from your working copy:
251 #\"data/really_large_component\": None,
252 },
253 \"safesync_url\": \"%s\"
254 }
255]
256""")
257
258
259## Generic utils
260
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000261def ParseXML(output):
262 try:
263 return xml.dom.minidom.parseString(output)
264 except xml.parsers.expat.ExpatError:
265 return None
266
267
maruel@chromium.org483b0082009-05-07 02:57:14 +0000268def GetNamedNodeText(node, node_name):
269 child_nodes = node.getElementsByTagName(node_name)
270 if not child_nodes:
271 return None
272 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
273 return child_nodes[0].firstChild.nodeValue
274
275
276def GetNodeNamedAttributeText(node, node_name, attribute_name):
277 child_nodes = node.getElementsByTagName(node_name)
278 if not child_nodes:
279 return None
280 assert len(child_nodes) == 1
281 return child_nodes[0].getAttribute(attribute_name)
282
283
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284class Error(Exception):
285 """gclient exception class."""
286 pass
287
288class PrintableObject(object):
289 def __str__(self):
290 output = ''
291 for i in dir(self):
292 if i.startswith('__'):
293 continue
294 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
295 return output
296
297
298def FileRead(filename):
299 content = None
300 f = open(filename, "rU")
301 try:
302 content = f.read()
303 finally:
304 f.close()
305 return content
306
307
308def FileWrite(filename, content):
309 f = open(filename, "w")
310 try:
311 f.write(content)
312 finally:
313 f.close()
314
315
316def RemoveDirectory(*path):
317 """Recursively removes a directory, even if it's marked read-only.
318
319 Remove the directory located at *path, if it exists.
320
321 shutil.rmtree() doesn't work on Windows if any of the files or directories
322 are read-only, which svn repositories and some .svn files are. We need to
323 be able to force the files to be writable (i.e., deletable) as we traverse
324 the tree.
325
326 Even with all this, Windows still sometimes fails to delete a file, citing
327 a permission error (maybe something to do with antivirus scans or disk
328 indexing). The best suggestion any of the user forums had was to wait a
329 bit and try again, so we do that too. It's hand-waving, but sometimes it
330 works. :/
331
332 On POSIX systems, things are a little bit simpler. The modes of the files
333 to be deleted doesn't matter, only the modes of the directories containing
334 them are significant. As the directory tree is traversed, each directory
335 has its mode set appropriately before descending into it. This should
336 result in the entire tree being removed, with the possible exception of
337 *path itself, because nothing attempts to change the mode of its parent.
338 Doing so would be hazardous, as it's not a directory slated for removal.
339 In the ordinary case, this is not a problem: for our purposes, the user
340 will never lack write permission on *path's parent.
341 """
342 file_path = os.path.join(*path)
343 if not os.path.exists(file_path):
344 return
345
346 if os.path.islink(file_path) or not os.path.isdir(file_path):
347 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
348
349 has_win32api = False
350 if sys.platform == 'win32':
351 has_win32api = True
352 # Some people don't have the APIs installed. In that case we'll do without.
353 try:
354 win32api = __import__('win32api')
355 win32con = __import__('win32con')
356 except ImportError:
357 has_win32api = False
358 else:
359 # On POSIX systems, we need the x-bit set on the directory to access it,
360 # the r-bit to see its contents, and the w-bit to remove files from it.
361 # The actual modes of the files within the directory is irrelevant.
362 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
363 for fn in os.listdir(file_path):
364 fullpath = os.path.join(file_path, fn)
365
366 # If fullpath is a symbolic link that points to a directory, isdir will
367 # be True, but we don't want to descend into that as a directory, we just
368 # want to remove the link. Check islink and treat links as ordinary files
369 # would be treated regardless of what they reference.
370 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
371 if sys.platform == 'win32':
372 os.chmod(fullpath, stat.S_IWRITE)
373 if has_win32api:
374 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
375 try:
376 os.remove(fullpath)
377 except OSError, e:
378 if e.errno != errno.EACCES or sys.platform != 'win32':
379 raise
380 print 'Failed to delete %s: trying again' % fullpath
381 time.sleep(0.1)
382 os.remove(fullpath)
383 else:
384 RemoveDirectory(fullpath)
385
386 if sys.platform == 'win32':
387 os.chmod(file_path, stat.S_IWRITE)
388 if has_win32api:
389 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
390 try:
391 os.rmdir(file_path)
392 except OSError, e:
393 if e.errno != errno.EACCES or sys.platform != 'win32':
394 raise
395 print 'Failed to remove %s: trying again' % file_path
396 time.sleep(0.1)
397 os.rmdir(file_path)
398
399
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000400def SubprocessCall(command, in_directory, fail_status=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000401 """Runs command, a list, in directory in_directory.
402
403 This function wraps SubprocessCallAndCapture, but does not perform the
404 capturing functions. See that function for a more complete usage
405 description.
406 """
407 # Call subprocess and capture nothing:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000408 SubprocessCallAndCapture(command, in_directory, fail_status)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000409
410
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000411def SubprocessCallAndCapture(command, in_directory, fail_status=None,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000412 pattern=None, capture_list=None):
413 """Runs command, a list, in directory in_directory.
414
415 A message indicating what is being done, as well as the command's stdout,
416 is printed to out.
417
418 If a pattern is specified, any line in the output matching pattern will have
419 its first match group appended to capture_list.
420
421 If the command fails, as indicated by a nonzero exit status, gclient will
422 exit with an exit status of fail_status. If fail_status is None (the
423 default), gclient will raise an Error exception.
424 """
425
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000426 print("\n________ running \'%s\' in \'%s\'"
427 % (' '.join(command), in_directory))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000428
429 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
430 # executable, but shell=True makes subprocess on Linux fail when it's called
431 # with a list because it only tries to execute the first item in the list.
432 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
433 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
434
435 if pattern:
436 compiled_pattern = re.compile(pattern)
437
438 # Also, we need to forward stdout to prevent weird re-ordering of output.
439 # This has to be done on a per byte basis to make sure it is not buffered:
440 # normally buffering is done for each line, but if svn requests input, no
441 # end-of-line character is output after the prompt and it would not show up.
442 in_byte = kid.stdout.read(1)
443 in_line = ""
444 while in_byte:
445 if in_byte != "\r":
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000446 sys.stdout.write(in_byte)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000447 in_line += in_byte
448 if in_byte == "\n" and pattern:
449 match = compiled_pattern.search(in_line[:-1])
450 if match:
451 capture_list.append(match.group(1))
452 in_line = ""
453 in_byte = kid.stdout.read(1)
454 rv = kid.wait()
455
456 if rv:
457 msg = "failed to run command: %s" % " ".join(command)
458
459 if fail_status != None:
460 print >>sys.stderr, msg
461 sys.exit(fail_status)
462
463 raise Error(msg)
464
465
466def IsUsingGit(root, paths):
467 """Returns True if we're using git to manage any of our checkouts.
468 |entries| is a list of paths to check."""
469 for path in paths:
470 if os.path.exists(os.path.join(root, path, '.git')):
471 return True
472 return False
473
474# -----------------------------------------------------------------------------
475# SVN utils:
476
477
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000478def RunSVN(args, in_directory):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000479 """Runs svn, sending output to stdout.
480
481 Args:
482 args: A sequence of command line parameters to be passed to svn.
483 in_directory: The directory where svn is to be run.
484
485 Raises:
486 Error: An error occurred while running the svn command.
487 """
488 c = [SVN_COMMAND]
489 c.extend(args)
490
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000491 SubprocessCall(c, in_directory)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000492
493
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000494def CaptureSVN(args, in_directory=None, print_error=True):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000495 """Runs svn, capturing output sent to stdout as a string.
496
497 Args:
498 args: A sequence of command line parameters to be passed to svn.
499 in_directory: The directory where svn is to be run.
500
501 Returns:
502 The output sent to stdout as a string.
503 """
504 c = [SVN_COMMAND]
505 c.extend(args)
506
507 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
508 # the svn.exe executable, but shell=True makes subprocess on Linux fail
509 # when it's called with a list because it only tries to execute the
510 # first string ("svn").
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000511 stderr = None
512 if print_error:
513 stderr = subprocess.PIPE
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000514 return subprocess.Popen(c,
515 cwd=in_directory,
516 shell=(sys.platform == 'win32'),
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000517 stdout=subprocess.PIPE,
518 stderr=stderr).communicate()[0]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000519
520
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000521def RunSVNAndGetFileList(args, in_directory, file_list):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000522 """Runs svn checkout, update, or status, output to stdout.
523
524 The first item in args must be either "checkout", "update", or "status".
525
526 svn's stdout is parsed to collect a list of files checked out or updated.
527 These files are appended to file_list. svn's stdout is also printed to
528 sys.stdout as in RunSVN.
529
530 Args:
531 args: A sequence of command line parameters to be passed to svn.
532 in_directory: The directory where svn is to be run.
533
534 Raises:
535 Error: An error occurred while running the svn command.
536 """
537 command = [SVN_COMMAND]
538 command.extend(args)
539
540 # svn update and svn checkout use the same pattern: the first three columns
541 # are for file status, property status, and lock status. This is followed
542 # by two spaces, and then the path to the file.
543 update_pattern = '^... (.*)$'
544
545 # The first three columns of svn status are the same as for svn update and
546 # svn checkout. The next three columns indicate addition-with-history,
547 # switch, and remote lock status. This is followed by one space, and then
548 # the path to the file.
549 status_pattern = '^...... (.*)$'
550
551 # args[0] must be a supported command. This will blow up if it's something
552 # else, which is good. Note that the patterns are only effective when
553 # these commands are used in their ordinary forms, the patterns are invalid
554 # for "svn status --show-updates", for example.
555 pattern = {
556 'checkout': update_pattern,
557 'status': status_pattern,
558 'update': update_pattern,
559 }[args[0]]
560
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000561 SubprocessCallAndCapture(command,
562 in_directory,
563 pattern=pattern,
564 capture_list=file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000565
566
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000567def CaptureSVNInfo(relpath, in_directory=None, print_error=True):
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000568 """Returns a dictionary from the svn info output for the given file.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000569
570 Args:
571 relpath: The directory where the working copy resides relative to
572 the directory given by in_directory.
573 in_directory: The directory where svn is to be run.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000574 """
maruel@chromium.org5c3a2ff2009-05-12 19:28:55 +0000575 output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000576 dom = ParseXML(output)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000577 result = {}
maruel@chromium.org483b0082009-05-07 02:57:14 +0000578 if dom:
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000579 def C(item, f):
580 if item is not None: return f(item)
maruel@chromium.org483b0082009-05-07 02:57:14 +0000581 # /info/entry/
582 # url
583 # reposityory/(root|uuid)
584 # wc-info/(schedule|depth)
585 # commit/(author|date)
586 # str() the results because they may be returned as Unicode, which
587 # interferes with the higher layers matching up things in the deps
588 # dictionary.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000589 # TODO(maruel): Fix at higher level instead (!)
590 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
591 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
592 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
593 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
594 int)
595 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
596 str)
597 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
598 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
599 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
600 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000601 return result
602
603
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000604def CaptureSVNHeadRevision(url):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000605 """Get the head revision of a SVN repository.
606
607 Returns:
608 Int head revision
609 """
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000610 info = CaptureSVN(["info", "--xml", url], os.getcwd())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000611 dom = xml.dom.minidom.parseString(info)
612 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
613
614
615class FileStatus:
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000616 def __init__(self, path, text_status, props, lock, history):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000617 self.path = path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000618 self.text_status = text_status
619 self.props = props
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000620 self.lock = lock
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000621 self.history = history
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000622
623 def __str__(self):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000624 # Emulate svn status 1.5 output.
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000625 return (self.text_status + self.props + self.lock + self.history + ' ' +
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626 self.path)
627
628
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000629def CaptureSVNStatus(path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000630 """Runs 'svn status' on an existing path.
631
632 Args:
633 path: The directory to run svn status.
634
635 Returns:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000636 An array of FileStatus corresponding to the emulated output of 'svn status'
637 version 1.5."""
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000638 dom = ParseXML(CaptureSVN(["status", "--xml"], path))
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000639 results = []
640 if dom:
641 # /status/target/entry/(wc-status|commit|author|date)
642 for target in dom.getElementsByTagName('target'):
643 base_path = target.getAttribute('path')
644 for entry in target.getElementsByTagName('entry'):
645 file = entry.getAttribute('path')
646 wc_status = entry.getElementsByTagName('wc-status')
647 assert len(wc_status) == 1
648 # Emulate svn 1.5 status ouput...
649 statuses = [' ' for i in range(7)]
650 # Col 0
651 xml_item_status = wc_status[0].getAttribute('item')
652 if xml_item_status == 'unversioned':
653 statuses[0] = '?'
654 elif xml_item_status == 'modified':
655 statuses[0] = 'M'
656 elif xml_item_status == 'added':
657 statuses[0] = 'A'
658 elif xml_item_status == 'conflicted':
659 statuses[0] = 'C'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000660 elif xml_item_status in ('incomplete', 'missing'):
661 statuses[0] = '!'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000662 elif not xml_item_status:
663 pass
664 else:
665 raise Exception('Unknown item status "%s"; please implement me!' %
666 xml_item_status)
667 # Col 1
668 xml_props_status = wc_status[0].getAttribute('props')
669 if xml_props_status == 'modified':
670 statuses[1] = 'M'
671 elif xml_props_status == 'conflicted':
672 statuses[1] = 'C'
673 elif (not xml_props_status or xml_props_status == 'none' or
674 xml_props_status == 'normal'):
675 pass
676 else:
677 raise Exception('Unknown props status "%s"; please implement me!' %
678 xml_props_status)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000679 # Col 2
680 if wc_status[0].getAttribute('wc-locked') == 'true':
681 statuses[2] = 'L'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000682 # Col 3
683 if wc_status[0].getAttribute('copied') == 'true':
684 statuses[3] = '+'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000685 item = FileStatus(file, statuses[0], statuses[1], statuses[2],
686 statuses[3])
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000687 results.append(item)
688 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000689
690
691### SCM abstraction layer
692
693
694class SCMWrapper(object):
695 """Add necessary glue between all the supported SCM.
696
697 This is the abstraction layer to bind to different SCM. Since currently only
698 subversion is supported, a lot of subersionism remains. This can be sorted out
699 once another SCM is supported."""
700 def __init__(self, url=None, root_dir=None, relpath=None,
701 scm_name='svn'):
702 # TODO(maruel): Deduce the SCM from the url.
703 self.scm_name = scm_name
704 self.url = url
705 self._root_dir = root_dir
706 if self._root_dir:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000707 self._root_dir = self._root_dir.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000708 self.relpath = relpath
709 if self.relpath:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000710 self.relpath = self.relpath.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000711
712 def FullUrlForRelativeUrl(self, url):
713 # Find the forth '/' and strip from there. A bit hackish.
714 return '/'.join(self.url.split('/')[:4]) + url
715
716 def RunCommand(self, command, options, args, file_list=None):
717 # file_list will have all files that are modified appended to it.
718
719 if file_list == None:
720 file_list = []
721
722 commands = {
723 'cleanup': self.cleanup,
724 'update': self.update,
725 'revert': self.revert,
726 'status': self.status,
727 'diff': self.diff,
728 'runhooks': self.status,
729 }
730
731 if not command in commands:
732 raise Error('Unknown command %s' % command)
733
734 return commands[command](options, args, file_list)
735
736 def cleanup(self, options, args, file_list):
737 """Cleanup working copy."""
738 command = ['cleanup']
739 command.extend(args)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000740 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000741
742 def diff(self, options, args, file_list):
743 # NOTE: This function does not currently modify file_list.
744 command = ['diff']
745 command.extend(args)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000746 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000747
748 def update(self, options, args, file_list):
749 """Runs SCM to update or transparently checkout the working copy.
750
751 All updated files will be appended to file_list.
752
753 Raises:
754 Error: if can't get URL for relative path.
755 """
756 # Only update if git is not controlling the directory.
757 git_path = os.path.join(self._root_dir, self.relpath, '.git')
758 if options.path_exists(git_path):
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000759 print("________ found .git directory; skipping %s" % self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000760 return
761
762 if args:
763 raise Error("Unsupported argument(s): %s" % ",".join(args))
764
765 url = self.url
766 components = url.split("@")
767 revision = None
768 forced_revision = False
769 if options.revision:
770 # Override the revision number.
771 url = '%s@%s' % (components[0], str(options.revision))
772 revision = int(options.revision)
773 forced_revision = True
774 elif len(components) == 2:
775 revision = int(components[1])
776 forced_revision = True
777
778 rev_str = ""
779 if revision:
780 rev_str = ' at %d' % revision
781
782 if not options.path_exists(os.path.join(self._root_dir, self.relpath)):
783 # We need to checkout.
784 command = ['checkout', url, os.path.join(self._root_dir, self.relpath)]
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000785 if revision:
786 command.extend(['--revision', str(revision)])
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000787 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000788 return
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000789
790 # Get the existing scm url and the revision number of the current checkout.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000791 from_info = CaptureSVNInfo(os.path.join(self._root_dir, self.relpath, '.'),
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000792 '.')
793
794 if options.manually_grab_svn_rev:
795 # Retrieve the current HEAD version because svn is slow at null updates.
796 if not revision:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000797 from_info_live = CaptureSVNInfo(from_info['URL'], '.')
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000798 revision = int(from_info_live['Revision'])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000799 rev_str = ' at %d' % revision
800
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000801 if from_info['URL'] != components[0]:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000802 to_info = CaptureSVNInfo(url, '.')
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000803 if from_info['Repository Root'] != to_info['Repository Root']:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000804 # We have different roots, so check if we can switch --relocate.
805 # Subversion only permits this if the repository UUIDs match.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000806 if from_info['UUID'] != to_info['UUID']:
807 raise Error("Can't switch the checkout to %s; UUID don't match. That "
808 "simply means in theory, gclient should verify you don't "
809 "have a local change, remove the old checkout and do a "
810 "fresh new checkout of the new repo. Contributions are "
811 "welcome." % url)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000812
813 # Perform the switch --relocate, then rewrite the from_url
814 # to reflect where we "are now." (This is the same way that
815 # Subversion itself handles the metadata when switch --relocate
816 # is used.) This makes the checks below for whether we
817 # can update to a revision or have to switch to a different
818 # branch work as expected.
819 # TODO(maruel): TEST ME !
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000820 command = ["switch", "--relocate",
821 from_info['Repository Root'],
822 to_info['Repository Root'],
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000823 self.relpath]
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000824 RunSVN(command, self._root_dir)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000825 from_info['URL'] = from_info['URL'].replace(
826 from_info['Repository Root'],
827 to_info['Repository Root'])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000828
829 # If the provided url has a revision number that matches the revision
830 # number of the existing directory, then we don't need to bother updating.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000831 if not options.force and from_info['Revision'] == revision:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000832 if options.verbose or not forced_revision:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000833 print("\n_____ %s%s" % (self.relpath, rev_str))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000834 return
835
836 command = ["update", os.path.join(self._root_dir, self.relpath)]
837 if revision:
838 command.extend(['--revision', str(revision)])
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000839 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000840
841 def revert(self, options, args, file_list):
842 """Reverts local modifications. Subversion specific.
843
844 All reverted files will be appended to file_list, even if Subversion
845 doesn't know about them.
846 """
847 path = os.path.join(self._root_dir, self.relpath)
848 if not os.path.isdir(path):
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000849 # svn revert won't work if the directory doesn't exist. It needs to
850 # checkout instead.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000851 print("\n_____ %s is missing, synching instead" % self.relpath)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000852 # Don't reuse the args.
853 return self.update(options, [], file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000854
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000855 files = CaptureSVNStatus(path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000856 # Batch the command.
857 files_to_revert = []
858 for file in files:
859 file_path = os.path.join(path, file.path)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000860 print(file_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000861 # Unversioned file or unexpected unversioned file.
862 if file.text_status in ('?', '~'):
863 # Remove extraneous file. Also remove unexpected unversioned
864 # directories. svn won't touch them but we want to delete these.
865 file_list.append(file_path)
866 try:
867 os.remove(file_path)
868 except EnvironmentError:
869 RemoveDirectory(file_path)
870
871 if file.text_status != '?':
872 # For any other status, svn revert will work.
873 file_list.append(file_path)
874 files_to_revert.append(file.path)
875
876 # Revert them all at once.
877 if files_to_revert:
878 accumulated_paths = []
879 accumulated_length = 0
880 command = ['revert']
881 for p in files_to_revert:
882 # Some shell have issues with command lines too long.
883 if accumulated_length and accumulated_length + len(p) > 3072:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000884 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000885 os.path.join(self._root_dir, self.relpath))
886 accumulated_paths = []
887 accumulated_length = 0
888 else:
889 accumulated_paths.append(p)
890 accumulated_length += len(p)
891 if accumulated_paths:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000892 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000893 os.path.join(self._root_dir, self.relpath))
894
895 def status(self, options, args, file_list):
896 """Display status information."""
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000897 path = os.path.join(self._root_dir, self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000898 command = ['status']
899 command.extend(args)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000900 if not os.path.isdir(path):
901 # svn status won't work if the directory doesn't exist.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000902 print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
903 "does not exist."
904 % (' '.join(command), path))
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000905 # There's no file list to retrieve.
906 else:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +0000907 RunSVNAndGetFileList(command, path, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000908
909
910## GClient implementation.
911
912
913class GClient(object):
914 """Object that represent a gclient checkout."""
915
916 supported_commands = [
917 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
918 ]
919
920 def __init__(self, root_dir, options):
921 self._root_dir = root_dir
922 self._options = options
923 self._config_content = None
924 self._config_dict = {}
925 self._deps_hooks = []
926
927 def SetConfig(self, content):
928 self._config_dict = {}
929 self._config_content = content
930 exec(content, self._config_dict)
931
932 def SaveConfig(self):
933 FileWrite(os.path.join(self._root_dir, self._options.config_filename),
934 self._config_content)
935
936 def _LoadConfig(self):
937 client_source = FileRead(os.path.join(self._root_dir,
938 self._options.config_filename))
939 self.SetConfig(client_source)
940
941 def ConfigContent(self):
942 return self._config_content
943
944 def GetVar(self, key, default=None):
945 return self._config_dict.get(key, default)
946
947 @staticmethod
948 def LoadCurrentConfig(options, from_dir=None):
949 """Searches for and loads a .gclient file relative to the current working
950 dir.
951
952 Returns:
953 A dict representing the contents of the .gclient file or an empty dict if
954 the .gclient file doesn't exist.
955 """
956 if not from_dir:
957 from_dir = os.curdir
958 path = os.path.realpath(from_dir)
959 while not options.path_exists(os.path.join(path, options.config_filename)):
960 next = os.path.split(path)
961 if not next[1]:
962 return None
963 path = next[0]
964 client = options.gclient(path, options)
965 client._LoadConfig()
966 return client
967
968 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
969 self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
970 solution_name, solution_url, safesync_url
971 ))
972
973 def _SaveEntries(self, entries):
974 """Creates a .gclient_entries file to record the list of unique checkouts.
975
976 The .gclient_entries file lives in the same directory as .gclient.
977
978 Args:
979 entries: A sequence of solution names.
980 """
981 text = "entries = [\n"
982 for entry in entries:
983 text += " \"%s\",\n" % entry
984 text += "]\n"
985 FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
986 text)
987
988 def _ReadEntries(self):
989 """Read the .gclient_entries file for the given client.
990
991 Args:
992 client: The client for which the entries file should be read.
993
994 Returns:
995 A sequence of solution names, which will be empty if there is the
996 entries file hasn't been created yet.
997 """
998 scope = {}
999 filename = os.path.join(self._root_dir, self._options.entries_filename)
1000 if not self._options.path_exists(filename):
1001 return []
1002 exec(FileRead(filename), scope)
1003 return scope["entries"]
1004
1005 class FromImpl:
1006 """Used to implement the From syntax."""
1007
1008 def __init__(self, module_name):
1009 self.module_name = module_name
1010
1011 def __str__(self):
1012 return 'From("%s")' % self.module_name
1013
1014 class _VarImpl:
1015 def __init__(self, custom_vars, local_scope):
1016 self._custom_vars = custom_vars
1017 self._local_scope = local_scope
1018
1019 def Lookup(self, var_name):
1020 """Implements the Var syntax."""
1021 if var_name in self._custom_vars:
1022 return self._custom_vars[var_name]
1023 elif var_name in self._local_scope.get("vars", {}):
1024 return self._local_scope["vars"][var_name]
1025 raise Error("Var is not defined: %s" % var_name)
1026
1027 def _ParseSolutionDeps(self, solution_name, solution_deps_content,
1028 custom_vars):
1029 """Parses the DEPS file for the specified solution.
1030
1031 Args:
1032 solution_name: The name of the solution to query.
1033 solution_deps_content: Content of the DEPS file for the solution
1034 custom_vars: A dict of vars to override any vars defined in the DEPS file.
1035
1036 Returns:
1037 A dict mapping module names (as relative paths) to URLs or an empty
1038 dict if the solution does not have a DEPS file.
1039 """
1040 # Skip empty
1041 if not solution_deps_content:
1042 return {}
1043 # Eval the content
1044 local_scope = {}
1045 var = self._VarImpl(custom_vars, local_scope)
1046 global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
1047 exec(solution_deps_content, global_scope, local_scope)
1048 deps = local_scope.get("deps", {})
1049
1050 # load os specific dependencies if defined. these dependencies may
1051 # override or extend the values defined by the 'deps' member.
1052 if "deps_os" in local_scope:
1053 deps_os_choices = {
1054 "win32": "win",
1055 "win": "win",
1056 "cygwin": "win",
1057 "darwin": "mac",
1058 "mac": "mac",
1059 "unix": "unix",
1060 "linux": "unix",
1061 "linux2": "unix",
1062 }
1063
1064 if self._options.deps_os is not None:
1065 deps_to_include = self._options.deps_os.split(",")
1066 if "all" in deps_to_include:
1067 deps_to_include = deps_os_choices.values()
1068 else:
1069 deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
1070
1071 deps_to_include = set(deps_to_include)
1072 for deps_os_key in deps_to_include:
1073 os_deps = local_scope["deps_os"].get(deps_os_key, {})
1074 if len(deps_to_include) > 1:
1075 # Ignore any overrides when including deps for more than one
1076 # platform, so we collect the broadest set of dependencies available.
1077 # We may end up with the wrong revision of something for our
1078 # platform, but this is the best we can do.
1079 deps.update([x for x in os_deps.items() if not x[0] in deps])
1080 else:
1081 deps.update(os_deps)
1082
1083 if 'hooks' in local_scope:
1084 self._deps_hooks.extend(local_scope['hooks'])
1085
1086 # If use_relative_paths is set in the DEPS file, regenerate
1087 # the dictionary using paths relative to the directory containing
1088 # the DEPS file.
1089 if local_scope.get('use_relative_paths'):
1090 rel_deps = {}
1091 for d, url in deps.items():
1092 # normpath is required to allow DEPS to use .. in their
1093 # dependency local path.
1094 rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
1095 return rel_deps
1096 else:
1097 return deps
1098
1099 def _ParseAllDeps(self, solution_urls, solution_deps_content):
1100 """Parse the complete list of dependencies for the client.
1101
1102 Args:
1103 solution_urls: A dict mapping module names (as relative paths) to URLs
1104 corresponding to the solutions specified by the client. This parameter
1105 is passed as an optimization.
1106 solution_deps_content: A dict mapping module names to the content
1107 of their DEPS files
1108
1109 Returns:
1110 A dict mapping module names (as relative paths) to URLs corresponding
1111 to the entire set of dependencies to checkout for the given client.
1112
1113 Raises:
1114 Error: If a dependency conflicts with another dependency or of a solution.
1115 """
1116 deps = {}
1117 for solution in self.GetVar("solutions"):
1118 custom_vars = solution.get("custom_vars", {})
1119 solution_deps = self._ParseSolutionDeps(
1120 solution["name"],
1121 solution_deps_content[solution["name"]],
1122 custom_vars)
1123
1124 # If a line is in custom_deps, but not in the solution, we want to append
1125 # this line to the solution.
1126 if "custom_deps" in solution:
1127 for d in solution["custom_deps"]:
1128 if d not in solution_deps:
1129 solution_deps[d] = solution["custom_deps"][d]
1130
1131 for d in solution_deps:
1132 if "custom_deps" in solution and d in solution["custom_deps"]:
1133 # Dependency is overriden.
1134 url = solution["custom_deps"][d]
1135 if url is None:
1136 continue
1137 else:
1138 url = solution_deps[d]
1139 # if we have a From reference dependent on another solution, then
1140 # just skip the From reference. When we pull deps for the solution,
1141 # we will take care of this dependency.
1142 #
1143 # If multiple solutions all have the same From reference, then we
1144 # should only add one to our list of dependencies.
1145 if type(url) != str:
1146 if url.module_name in solution_urls:
1147 # Already parsed.
1148 continue
1149 if d in deps and type(deps[d]) != str:
1150 if url.module_name == deps[d].module_name:
1151 continue
1152 else:
1153 parsed_url = urlparse.urlparse(url)
1154 scheme = parsed_url[0]
1155 if not scheme:
1156 # A relative url. Fetch the real base.
1157 path = parsed_url[2]
1158 if path[0] != "/":
1159 raise Error(
1160 "relative DEPS entry \"%s\" must begin with a slash" % d)
1161 # Create a scm just to query the full url.
1162 scm = self._options.scm_wrapper(solution["url"], self._root_dir,
1163 None)
1164 url = scm.FullUrlForRelativeUrl(url)
1165 if d in deps and deps[d] != url:
1166 raise Error(
1167 "Solutions have conflicting versions of dependency \"%s\"" % d)
1168 if d in solution_urls and solution_urls[d] != url:
1169 raise Error(
1170 "Dependency \"%s\" conflicts with specified solution" % d)
1171 # Grab the dependency.
1172 deps[d] = url
1173 return deps
1174
1175 def _RunHookAction(self, hook_dict):
1176 """Runs the action from a single hook.
1177 """
1178 command = hook_dict['action'][:]
1179 if command[0] == 'python':
1180 # If the hook specified "python" as the first item, the action is a
1181 # Python script. Run it by starting a new copy of the same
1182 # interpreter.
1183 command[0] = sys.executable
1184
1185 # Use a discrete exit status code of 2 to indicate that a hook action
1186 # failed. Users of this script may wish to treat hook action failures
1187 # differently from VC failures.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001188 SubprocessCall(command, self._root_dir, fail_status=2)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001189
1190 def _RunHooks(self, command, file_list, is_using_git):
1191 """Evaluates all hooks, running actions as needed.
1192 """
1193 # Hooks only run for these command types.
1194 if not command in ('update', 'revert', 'runhooks'):
1195 return
1196
1197 # Get any hooks from the .gclient file.
1198 hooks = self.GetVar("hooks", [])
1199 # Add any hooks found in DEPS files.
1200 hooks.extend(self._deps_hooks)
1201
1202 # If "--force" was specified, run all hooks regardless of what files have
1203 # changed. If the user is using git, then we don't know what files have
1204 # changed so we always run all hooks.
1205 if self._options.force or is_using_git:
1206 for hook_dict in hooks:
1207 self._RunHookAction(hook_dict)
1208 return
1209
1210 # Run hooks on the basis of whether the files from the gclient operation
1211 # match each hook's pattern.
1212 for hook_dict in hooks:
1213 pattern = re.compile(hook_dict['pattern'])
1214 for file in file_list:
1215 if not pattern.search(file):
1216 continue
1217
1218 self._RunHookAction(hook_dict)
1219
1220 # The hook's action only runs once. Don't bother looking for any
1221 # more matches.
1222 break
1223
1224 def RunOnDeps(self, command, args):
1225 """Runs a command on each dependency in a client and its dependencies.
1226
1227 The module's dependencies are specified in its top-level DEPS files.
1228
1229 Args:
1230 command: The command to use (e.g., 'status' or 'diff')
1231 args: list of str - extra arguments to add to the command line.
1232
1233 Raises:
1234 Error: If the client has conflicting entries.
1235 """
1236 if not command in self.supported_commands:
1237 raise Error("'%s' is an unsupported command" % command)
1238
1239 # Check for revision overrides.
1240 revision_overrides = {}
1241 for revision in self._options.revisions:
1242 if revision.find("@") == -1:
1243 raise Error(
1244 "Specify the full dependency when specifying a revision number.")
1245 revision_elem = revision.split("@")
1246 # Disallow conflicting revs
1247 if revision_overrides.has_key(revision_elem[0]) and \
1248 revision_overrides[revision_elem[0]] != revision_elem[1]:
1249 raise Error(
1250 "Conflicting revision numbers specified.")
1251 revision_overrides[revision_elem[0]] = revision_elem[1]
1252
1253 solutions = self.GetVar("solutions")
1254 if not solutions:
1255 raise Error("No solution specified")
1256
1257 # When running runhooks --force, there's no need to consult the SCM.
1258 # All known hooks are expected to run unconditionally regardless of working
1259 # copy state, so skip the SCM status check.
1260 run_scm = not (command == 'runhooks' and self._options.force)
1261
1262 entries = {}
1263 entries_deps_content = {}
1264 file_list = []
1265 # Run on the base solutions first.
1266 for solution in solutions:
1267 name = solution["name"]
1268 if name in entries:
1269 raise Error("solution %s specified more than once" % name)
1270 url = solution["url"]
1271 entries[name] = url
1272 if run_scm:
1273 self._options.revision = revision_overrides.get(name)
1274 scm = self._options.scm_wrapper(url, self._root_dir, name)
1275 scm.RunCommand(command, self._options, args, file_list)
1276 self._options.revision = None
1277 try:
1278 deps_content = FileRead(os.path.join(self._root_dir, name,
1279 self._options.deps_file))
1280 except IOError, e:
1281 if e.errno != errno.ENOENT:
1282 raise
1283 deps_content = ""
1284 entries_deps_content[name] = deps_content
1285
1286 # Process the dependencies next (sort alphanumerically to ensure that
1287 # containing directories get populated first and for readability)
1288 deps = self._ParseAllDeps(entries, entries_deps_content)
1289 deps_to_process = deps.keys()
1290 deps_to_process.sort()
1291
1292 # First pass for direct dependencies.
1293 for d in deps_to_process:
1294 if type(deps[d]) == str:
1295 url = deps[d]
1296 entries[d] = url
1297 if run_scm:
1298 self._options.revision = revision_overrides.get(d)
1299 scm = self._options.scm_wrapper(url, self._root_dir, d)
1300 scm.RunCommand(command, self._options, args, file_list)
1301 self._options.revision = None
1302
1303 # Second pass for inherited deps (via the From keyword)
1304 for d in deps_to_process:
1305 if type(deps[d]) != str:
1306 sub_deps = self._ParseSolutionDeps(
1307 deps[d].module_name,
1308 FileRead(os.path.join(self._root_dir,
1309 deps[d].module_name,
1310 self._options.deps_file)),
1311 {})
1312 url = sub_deps[d]
1313 entries[d] = url
1314 if run_scm:
1315 self._options.revision = revision_overrides.get(d)
1316 scm = self._options.scm_wrapper(url, self._root_dir, d)
1317 scm.RunCommand(command, self._options, args, file_list)
1318 self._options.revision = None
1319
1320 is_using_git = IsUsingGit(self._root_dir, entries.keys())
1321 self._RunHooks(command, file_list, is_using_git)
1322
1323 if command == 'update':
1324 # notify the user if there is an orphaned entry in their working copy.
1325 # TODO(darin): we should delete this directory manually if it doesn't
1326 # have any changes in it.
1327 prev_entries = self._ReadEntries()
1328 for entry in prev_entries:
1329 e_dir = os.path.join(self._root_dir, entry)
1330 if entry not in entries and self._options.path_exists(e_dir):
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001331 if CaptureSVNStatus(e_dir):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001332 # There are modified files in this entry
1333 entries[entry] = None # Keep warning until removed.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001334 print("\nWARNING: \"%s\" is no longer part of this client. "
1335 "It is recommended that you manually remove it.\n") % entry
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001336 else:
1337 # Delete the entry
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001338 print("\n________ deleting \'%s\' " +
1339 "in \'%s\'") % (entry, self._root_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001340 RemoveDirectory(e_dir)
1341 # record the current list of entries for next time
1342 self._SaveEntries(entries)
1343
1344 def PrintRevInfo(self):
1345 """Output revision info mapping for the client and its dependencies. This
1346 allows the capture of a overall "revision" for the source tree that can
1347 be used to reproduce the same tree in the future. The actual output
1348 contains enough information (source paths, svn server urls and revisions)
1349 that it can be used either to generate external svn commands (without
1350 gclient) or as input to gclient's --rev option (with some massaging of
1351 the data).
1352
1353 NOTE: Unlike RunOnDeps this does not require a local checkout and is run
1354 on the Pulse master. It MUST NOT execute hooks.
1355
1356 Raises:
1357 Error: If the client has conflicting entries.
1358 """
1359 # Check for revision overrides.
1360 revision_overrides = {}
1361 for revision in self._options.revisions:
1362 if revision.find("@") < 0:
1363 raise Error(
1364 "Specify the full dependency when specifying a revision number.")
1365 revision_elem = revision.split("@")
1366 # Disallow conflicting revs
1367 if revision_overrides.has_key(revision_elem[0]) and \
1368 revision_overrides[revision_elem[0]] != revision_elem[1]:
1369 raise Error(
1370 "Conflicting revision numbers specified.")
1371 revision_overrides[revision_elem[0]] = revision_elem[1]
1372
1373 solutions = self.GetVar("solutions")
1374 if not solutions:
1375 raise Error("No solution specified")
1376
1377 entries = {}
1378 entries_deps_content = {}
1379
1380 # Inner helper to generate base url and rev tuple (including honoring
1381 # |revision_overrides|)
1382 def GetURLAndRev(name, original_url):
1383 if original_url.find("@") < 0:
1384 if revision_overrides.has_key(name):
1385 return (original_url, int(revision_overrides[name]))
1386 else:
1387 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001388 return (original_url, CaptureSVNHeadRevision(original_url))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001389 else:
1390 url_components = original_url.split("@")
1391 if revision_overrides.has_key(name):
1392 return (url_components[0], int(revision_overrides[name]))
1393 else:
1394 return (url_components[0], int(url_components[1]))
1395
1396 # Run on the base solutions first.
1397 for solution in solutions:
1398 name = solution["name"]
1399 if name in entries:
1400 raise Error("solution %s specified more than once" % name)
1401 (url, rev) = GetURLAndRev(name, solution["url"])
1402 entries[name] = "%s@%d" % (url, rev)
1403 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1404 entries_deps_content[name] = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001405 ["cat",
1406 "%s/%s@%d" % (url,
1407 self._options.deps_file,
1408 rev)],
1409 os.getcwd())
1410
1411 # Process the dependencies next (sort alphanumerically to ensure that
1412 # containing directories get populated first and for readability)
1413 deps = self._ParseAllDeps(entries, entries_deps_content)
1414 deps_to_process = deps.keys()
1415 deps_to_process.sort()
1416
1417 # First pass for direct dependencies.
1418 for d in deps_to_process:
1419 if type(deps[d]) == str:
1420 (url, rev) = GetURLAndRev(d, deps[d])
1421 entries[d] = "%s@%d" % (url, rev)
1422
1423 # Second pass for inherited deps (via the From keyword)
1424 for d in deps_to_process:
1425 if type(deps[d]) != str:
1426 deps_parent_url = entries[deps[d].module_name]
1427 if deps_parent_url.find("@") < 0:
1428 raise Error("From %s missing revisioned url" % deps[d].module_name)
1429 deps_parent_url_components = deps_parent_url.split("@")
1430 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1431 deps_parent_content = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001432 ["cat",
1433 "%s/%s@%s" % (deps_parent_url_components[0],
1434 self._options.deps_file,
1435 deps_parent_url_components[1])],
1436 os.getcwd())
1437 sub_deps = self._ParseSolutionDeps(
1438 deps[d].module_name,
1439 FileRead(os.path.join(self._root_dir,
1440 deps[d].module_name,
1441 self._options.deps_file)),
1442 {})
1443 (url, rev) = GetURLAndRev(d, sub_deps[d])
1444 entries[d] = "%s@%d" % (url, rev)
1445
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001446 print(";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())]))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001447
1448
1449## gclient commands.
1450
1451
1452def DoCleanup(options, args):
1453 """Handle the cleanup subcommand.
1454
1455 Raises:
1456 Error: if client isn't configured properly.
1457 """
1458 client = options.gclient.LoadCurrentConfig(options)
1459 if not client:
1460 raise Error("client not configured; see 'gclient config'")
1461 if options.verbose:
1462 # Print out the .gclient file. This is longer than if we just printed the
1463 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001464 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001465 options.verbose = True
1466 return client.RunOnDeps('cleanup', args)
1467
1468
1469def DoConfig(options, args):
1470 """Handle the config subcommand.
1471
1472 Args:
1473 options: If options.spec set, a string providing contents of config file.
1474 args: The command line args. If spec is not set,
1475 then args[0] is a string URL to get for config file.
1476
1477 Raises:
1478 Error: on usage error
1479 """
1480 if len(args) < 1 and not options.spec:
1481 raise Error("required argument missing; see 'gclient help config'")
1482 if options.path_exists(options.config_filename):
1483 raise Error("%s file already exists in the current directory" %
1484 options.config_filename)
1485 client = options.gclient('.', options)
1486 if options.spec:
1487 client.SetConfig(options.spec)
1488 else:
1489 # TODO(darin): it would be nice to be able to specify an alternate relpath
1490 # for the given URL.
1491 base_url = args[0]
1492 name = args[0].split("/")[-1]
1493 safesync_url = ""
1494 if len(args) > 1:
1495 safesync_url = args[1]
1496 client.SetDefaultConfig(name, base_url, safesync_url)
1497 client.SaveConfig()
1498
1499
1500def DoHelp(options, args):
1501 """Handle the help subcommand giving help for another subcommand.
1502
1503 Raises:
1504 Error: if the command is unknown.
1505 """
1506 if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001507 print(COMMAND_USAGE_TEXT[args[0]])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001508 else:
1509 raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
1510
1511
1512def DoStatus(options, args):
1513 """Handle the status subcommand.
1514
1515 Raises:
1516 Error: if client isn't configured properly.
1517 """
1518 client = options.gclient.LoadCurrentConfig(options)
1519 if not client:
1520 raise Error("client not configured; see 'gclient config'")
1521 if options.verbose:
1522 # Print out the .gclient file. This is longer than if we just printed the
1523 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001524 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001525 options.verbose = True
1526 return client.RunOnDeps('status', args)
1527
1528
1529def DoUpdate(options, args):
1530 """Handle the update and sync subcommands.
1531
1532 Raises:
1533 Error: if client isn't configured properly.
1534 """
1535 client = options.gclient.LoadCurrentConfig(options)
1536
1537 if not client:
1538 raise Error("client not configured; see 'gclient config'")
1539
1540 if not options.head:
1541 solutions = client.GetVar('solutions')
1542 if solutions:
1543 for s in solutions:
1544 if s.get('safesync_url', ''):
1545 # rip through revisions and make sure we're not over-riding
1546 # something that was explicitly passed
1547 has_key = False
1548 for r in options.revisions:
1549 if r.split('@')[0] == s['name']:
1550 has_key = True
1551 break
1552
1553 if not has_key:
1554 handle = urllib.urlopen(s['safesync_url'])
1555 rev = handle.read().strip()
1556 handle.close()
1557 if len(rev):
1558 options.revisions.append(s['name']+'@'+rev)
1559
1560 if options.verbose:
1561 # Print out the .gclient file. This is longer than if we just printed the
1562 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001563 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001564 return client.RunOnDeps('update', args)
1565
1566
1567def DoDiff(options, args):
1568 """Handle the diff subcommand.
1569
1570 Raises:
1571 Error: if client isn't configured properly.
1572 """
1573 client = options.gclient.LoadCurrentConfig(options)
1574 if not client:
1575 raise Error("client not configured; see 'gclient config'")
1576 if options.verbose:
1577 # Print out the .gclient file. This is longer than if we just printed the
1578 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001579 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001580 options.verbose = True
1581 return client.RunOnDeps('diff', args)
1582
1583
1584def DoRevert(options, args):
1585 """Handle the revert subcommand.
1586
1587 Raises:
1588 Error: if client isn't configured properly.
1589 """
1590 client = options.gclient.LoadCurrentConfig(options)
1591 if not client:
1592 raise Error("client not configured; see 'gclient config'")
1593 return client.RunOnDeps('revert', args)
1594
1595
1596def DoRunHooks(options, args):
1597 """Handle the runhooks subcommand.
1598
1599 Raises:
1600 Error: if client isn't configured properly.
1601 """
1602 client = options.gclient.LoadCurrentConfig(options)
1603 if not client:
1604 raise Error("client not configured; see 'gclient config'")
1605 if options.verbose:
1606 # Print out the .gclient file. This is longer than if we just printed the
1607 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001608 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001609 return client.RunOnDeps('runhooks', args)
1610
1611
1612def DoRevInfo(options, args):
1613 """Handle the revinfo subcommand.
1614
1615 Raises:
1616 Error: if client isn't configured properly.
1617 """
1618 client = options.gclient.LoadCurrentConfig(options)
1619 if not client:
1620 raise Error("client not configured; see 'gclient config'")
1621 client.PrintRevInfo()
1622
1623
1624gclient_command_map = {
1625 "cleanup": DoCleanup,
1626 "config": DoConfig,
1627 "diff": DoDiff,
1628 "help": DoHelp,
1629 "status": DoStatus,
1630 "sync": DoUpdate,
1631 "update": DoUpdate,
1632 "revert": DoRevert,
1633 "runhooks": DoRunHooks,
1634 "revinfo" : DoRevInfo,
1635}
1636
1637
1638def DispatchCommand(command, options, args, command_map=None):
1639 """Dispatches the appropriate subcommand based on command line arguments."""
1640 if command_map is None:
1641 command_map = gclient_command_map
1642
1643 if command in command_map:
1644 return command_map[command](options, args)
1645 else:
1646 raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
1647
1648
1649def Main(argv):
1650 """Parse command line arguments and dispatch command."""
1651
1652 option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
1653 version=__version__)
1654 option_parser.disable_interspersed_args()
1655 option_parser.add_option("", "--force", action="store_true", default=False,
1656 help=("(update/sync only) force update even "
1657 "for modules which haven't changed"))
1658 option_parser.add_option("", "--revision", action="append", dest="revisions",
1659 metavar="REV", default=[],
1660 help=("(update/sync only) sync to a specific "
1661 "revision, can be used multiple times for "
1662 "each solution, e.g. --revision=src@123, "
1663 "--revision=internal@32"))
1664 option_parser.add_option("", "--deps", default=None, dest="deps_os",
1665 metavar="OS_LIST",
1666 help=("(update/sync only) sync deps for the "
1667 "specified (comma-separated) platform(s); "
1668 "'all' will sync all platforms"))
1669 option_parser.add_option("", "--spec", default=None,
1670 help=("(config only) create a gclient file "
1671 "containing the provided string"))
1672 option_parser.add_option("", "--verbose", action="store_true", default=False,
1673 help="produce additional output for diagnostics")
1674 option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
1675 default=False,
1676 help="Skip svn up whenever possible by requesting "
1677 "actual HEAD revision from the repository")
1678 option_parser.add_option("", "--head", action="store_true", default=False,
1679 help=("skips any safesync_urls specified in "
1680 "configured solutions"))
1681
1682 if len(argv) < 2:
1683 # Users don't need to be told to use the 'help' command.
1684 option_parser.print_help()
1685 return 1
1686 # Add manual support for --version as first argument.
1687 if argv[1] == '--version':
1688 option_parser.print_version()
1689 return 0
1690
1691 # Add manual support for --help as first argument.
1692 if argv[1] == '--help':
1693 argv[1] = 'help'
1694
1695 command = argv[1]
1696 options, args = option_parser.parse_args(argv[2:])
1697
1698 if len(argv) < 3 and command == "help":
1699 option_parser.print_help()
1700 return 0
1701
1702 # Files used for configuration and state saving.
1703 options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
1704 options.entries_filename = ".gclient_entries"
1705 options.deps_file = "DEPS"
1706
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001707 options.path_exists = os.path.exists
1708 options.gclient = GClient
1709 options.scm_wrapper = SCMWrapper
1710 options.platform = sys.platform
1711 return DispatchCommand(command, options, args)
1712
1713
1714if "__main__" == __name__:
1715 try:
1716 result = Main(sys.argv)
1717 except Error, e:
maruel@chromium.orgdf7a3132009-05-12 17:49:49 +00001718 print >> sys.stderr, "Error: %s" % str(e)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001719 result = 1
1720 sys.exit(result)
1721
1722# vim: ts=2:sw=2:tw=80:et: