blob: e09b14abae73b78b089bb6c38442ebd444c29470 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A wrapper script to manage a set of client modules in different SCM.
18
19This script is intended to be used to help basic management of client
20program sources residing in one or more Subversion modules, along with
21other modules it depends on, also in Subversion, but possibly on
22multiple respositories, making a wrapper system apparently necessary.
23
24Files
25 .gclient : Current client configuration, written by 'config' command.
26 Format is a Python script defining 'solutions', a list whose
27 entries each are maps binding the strings "name" and "url"
28 to strings specifying the name and location of the client
29 module, as well as "custom_deps" to a map similar to the DEPS
30 file below.
31 .gclient_entries : A cache constructed by 'update' command. Format is a
32 Python script defining 'entries', a list of the names
33 of all modules in the client
34 <module>/DEPS : Python script defining var 'deps' as a map from each requisite
35 submodule name to a URL where it can be found (via one SCM)
36
37Hooks
38 .gclient and DEPS files may optionally contain a list named "hooks" to
39 allow custom actions to be performed based on files that have changed in the
40 working copy as a result of a "sync"/"update" or "revert" operation. Hooks
41 can also be run based on what files have been modified in the working copy
42 with the "runhooks" operation. If any of these operation are run with
43 --force, all known hooks will run regardless of the state of the working
44 copy.
45
46 Each item in a "hooks" list is a dict, containing these two keys:
47 "pattern" The associated value is a string containing a regular
48 expression. When a file whose pathname matches the expression
49 is checked out, updated, or reverted, the hook's "action" will
50 run.
51 "action" A list describing a command to run along with its arguments, if
52 any. An action command will run at most one time per gclient
53 invocation, regardless of how many files matched the pattern.
54 The action is executed in the same directory as the .gclient
55 file. If the first item in the list is the string "python",
56 the current Python interpreter (sys.executable) will be used
57 to run the command.
58
59 Example:
60 hooks = [
61 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
62 "action": ["python", "image_indexer.py", "--all"]},
63 ]
64"""
65
66__author__ = "darinf@gmail.com (Darin Fisher)"
67__version__ = "0.3.1"
68
69import errno
70import optparse
71import os
72import re
73import stat
74import subprocess
75import sys
76import time
77import urlparse
78import xml.dom.minidom
79import urllib
80
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000081
82SVN_COMMAND = "svn"
83
84
85# default help text
86DEFAULT_USAGE_TEXT = (
87"""usage: %prog <subcommand> [options] [--] [svn options/args...]
88a wrapper for managing a set of client modules in svn.
89Version """ + __version__ + """
90
91subcommands:
92 cleanup
93 config
94 diff
95 revert
96 status
97 sync
98 update
99 runhooks
100 revinfo
101
102Options and extra arguments can be passed to invoked svn commands by
103appending them to the command line. Note that if the first such
104appended option starts with a dash (-) then the options must be
105preceded by -- to distinguish them from gclient options.
106
107For additional help on a subcommand or examples of usage, try
108 %prog help <subcommand>
109 %prog help files
110""")
111
112GENERIC_UPDATE_USAGE_TEXT = (
113 """Perform a checkout/update of the modules specified by the gclient
114configuration; see 'help config'. Unless --revision is specified,
115then the latest revision of the root solutions is checked out, with
116dependent submodule versions updated according to DEPS files.
117If --revision is specified, then the given revision is used in place
118of the latest, either for a single solution or for all solutions.
119Unless the --force option is provided, solutions and modules whose
120local revision matches the one to update (i.e., they have not changed
121in the repository) are *not* modified.
122This a synonym for 'gclient %(alias)s'
123
124usage: gclient %(cmd)s [options] [--] [svn update options/args]
125
126Valid options:
127 --force : force update even for unchanged modules
128 --revision REV : update/checkout all solutions with specified revision
129 --revision SOLUTION@REV : update given solution to specified revision
130 --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
131 --verbose : output additional diagnostics
132
133Examples:
134 gclient %(cmd)s
135 update files from SVN according to current configuration,
136 *for modules which have changed since last update or sync*
137 gclient %(cmd)s --force
138 update files from SVN according to current configuration, for
139 all modules (useful for recovering files deleted from local copy)
140""")
141
142COMMAND_USAGE_TEXT = {
143 "cleanup":
144 """Clean up all working copies, using 'svn cleanup' for each module.
145Additional options and args may be passed to 'svn cleanup'.
146
147usage: cleanup [options] [--] [svn cleanup args/options]
148
149Valid options:
150 --verbose : output additional diagnostics
151""",
152 "config": """Create a .gclient file in the current directory; this
153specifies the configuration for further commands. After update/sync,
154top-level DEPS files in each module are read to determine dependent
155modules to operate on as well. If optional [url] parameter is
156provided, then configuration is read from a specified Subversion server
157URL. Otherwise, a --spec option must be provided.
158
159usage: config [option | url] [safesync url]
160
161Valid options:
162 --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
163 *Note that due to Cygwin/Python brokenness, it
164 probably can't contain any newlines.*
165
166Examples:
167 gclient config https://gclient.googlecode.com/svn/trunk/gclient
168 configure a new client to check out gclient.py tool sources
169 gclient config --spec='solutions=[{"name":"gclient","""
170 '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
171 '"custom_deps":{}}]',
172 "diff": """Display the differences between two revisions of modules.
173(Does 'svn diff' for each checked out module and dependences.)
174Additional args and options to 'svn diff' can be passed after
175gclient options.
176
177usage: diff [options] [--] [svn args/options]
178
179Valid options:
180 --verbose : output additional diagnostics
181
182Examples:
183 gclient diff
184 simple 'svn diff' for configured client and dependences
185 gclient diff -- -x -b
186 use 'svn diff -x -b' to suppress whitespace-only differences
187 gclient diff -- -r HEAD -x -b
188 diff versus the latest version of each module
189""",
190 "revert":
191 """Revert every file in every managed directory in the client view.
192
193usage: revert
194""",
195 "status":
196 """Show the status of client and dependent modules, using 'svn diff'
197for each module. Additional options and args may be passed to 'svn diff'.
198
199usage: status [options] [--] [svn diff args/options]
200
201Valid options:
202 --verbose : output additional diagnostics
203""",
204 "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
205 "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
206 "help": """Describe the usage of this program or its subcommands.
207
208usage: help [options] [subcommand]
209
210Valid options:
211 --verbose : output additional diagnostics
212""",
213 "runhooks":
214 """Runs hooks for files that have been modified in the local working copy,
215according to 'svn status'.
216
217usage: runhooks [options]
218
219Valid options:
220 --force : runs all known hooks, regardless of the working
221 copy status
222 --verbose : output additional diagnostics
223""",
224 "revinfo":
225 """Outputs source path, server URL and revision information for every
226dependency in all solutions (no local checkout required).
227
228usage: revinfo [options]
229""",
230}
231
232# parameterized by (solution_name, solution_url, safesync_url)
233DEFAULT_CLIENT_FILE_TEXT = (
234 """
235# An element of this array (a \"solution\") describes a repository directory
236# that will be checked out into your working copy. Each solution may
237# optionally define additional dependencies (via its DEPS file) to be
238# checked out alongside the solution's directory. A solution may also
239# specify custom dependencies (via the \"custom_deps\" property) that
240# override or augment the dependencies specified by the DEPS file.
241# If a \"safesync_url\" is specified, it is assumed to reference the location of
242# a text file which contains nothing but the last known good SCM revision to
243# sync against. It is fetched if specified and used unless --head is passed
244solutions = [
245 { \"name\" : \"%s\",
246 \"url\" : \"%s\",
247 \"custom_deps\" : {
248 # To use the trunk of a component instead of what's in DEPS:
249 #\"component\": \"https://svnserver/component/trunk/\",
250 # To exclude a component from your working copy:
251 #\"data/really_large_component\": None,
252 },
253 \"safesync_url\": \"%s\"
254 }
255]
256""")
257
258
259## Generic utils
260
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000261def ParseXML(output):
262 try:
263 return xml.dom.minidom.parseString(output)
264 except xml.parsers.expat.ExpatError:
265 return None
266
267
maruel@chromium.org483b0082009-05-07 02:57:14 +0000268def GetNamedNodeText(node, node_name):
269 child_nodes = node.getElementsByTagName(node_name)
270 if not child_nodes:
271 return None
272 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
273 return child_nodes[0].firstChild.nodeValue
274
275
276def GetNodeNamedAttributeText(node, node_name, attribute_name):
277 child_nodes = node.getElementsByTagName(node_name)
278 if not child_nodes:
279 return None
280 assert len(child_nodes) == 1
281 return child_nodes[0].getAttribute(attribute_name)
282
283
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284class Error(Exception):
285 """gclient exception class."""
286 pass
287
288class PrintableObject(object):
289 def __str__(self):
290 output = ''
291 for i in dir(self):
292 if i.startswith('__'):
293 continue
294 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
295 return output
296
297
298def FileRead(filename):
299 content = None
300 f = open(filename, "rU")
301 try:
302 content = f.read()
303 finally:
304 f.close()
305 return content
306
307
308def FileWrite(filename, content):
309 f = open(filename, "w")
310 try:
311 f.write(content)
312 finally:
313 f.close()
314
315
316def RemoveDirectory(*path):
317 """Recursively removes a directory, even if it's marked read-only.
318
319 Remove the directory located at *path, if it exists.
320
321 shutil.rmtree() doesn't work on Windows if any of the files or directories
322 are read-only, which svn repositories and some .svn files are. We need to
323 be able to force the files to be writable (i.e., deletable) as we traverse
324 the tree.
325
326 Even with all this, Windows still sometimes fails to delete a file, citing
327 a permission error (maybe something to do with antivirus scans or disk
328 indexing). The best suggestion any of the user forums had was to wait a
329 bit and try again, so we do that too. It's hand-waving, but sometimes it
330 works. :/
331
332 On POSIX systems, things are a little bit simpler. The modes of the files
333 to be deleted doesn't matter, only the modes of the directories containing
334 them are significant. As the directory tree is traversed, each directory
335 has its mode set appropriately before descending into it. This should
336 result in the entire tree being removed, with the possible exception of
337 *path itself, because nothing attempts to change the mode of its parent.
338 Doing so would be hazardous, as it's not a directory slated for removal.
339 In the ordinary case, this is not a problem: for our purposes, the user
340 will never lack write permission on *path's parent.
341 """
342 file_path = os.path.join(*path)
343 if not os.path.exists(file_path):
344 return
345
346 if os.path.islink(file_path) or not os.path.isdir(file_path):
347 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
348
349 has_win32api = False
350 if sys.platform == 'win32':
351 has_win32api = True
352 # Some people don't have the APIs installed. In that case we'll do without.
353 try:
354 win32api = __import__('win32api')
355 win32con = __import__('win32con')
356 except ImportError:
357 has_win32api = False
358 else:
359 # On POSIX systems, we need the x-bit set on the directory to access it,
360 # the r-bit to see its contents, and the w-bit to remove files from it.
361 # The actual modes of the files within the directory is irrelevant.
362 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
363 for fn in os.listdir(file_path):
364 fullpath = os.path.join(file_path, fn)
365
366 # If fullpath is a symbolic link that points to a directory, isdir will
367 # be True, but we don't want to descend into that as a directory, we just
368 # want to remove the link. Check islink and treat links as ordinary files
369 # would be treated regardless of what they reference.
370 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
371 if sys.platform == 'win32':
372 os.chmod(fullpath, stat.S_IWRITE)
373 if has_win32api:
374 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
375 try:
376 os.remove(fullpath)
377 except OSError, e:
378 if e.errno != errno.EACCES or sys.platform != 'win32':
379 raise
380 print 'Failed to delete %s: trying again' % fullpath
381 time.sleep(0.1)
382 os.remove(fullpath)
383 else:
384 RemoveDirectory(fullpath)
385
386 if sys.platform == 'win32':
387 os.chmod(file_path, stat.S_IWRITE)
388 if has_win32api:
389 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
390 try:
391 os.rmdir(file_path)
392 except OSError, e:
393 if e.errno != errno.EACCES or sys.platform != 'win32':
394 raise
395 print 'Failed to remove %s: trying again' % file_path
396 time.sleep(0.1)
397 os.rmdir(file_path)
398
399
400def SubprocessCall(command, in_directory, out, fail_status=None):
401 """Runs command, a list, in directory in_directory.
402
403 This function wraps SubprocessCallAndCapture, but does not perform the
404 capturing functions. See that function for a more complete usage
405 description.
406 """
407 # Call subprocess and capture nothing:
408 SubprocessCallAndCapture(command, in_directory, out, fail_status)
409
410
411def SubprocessCallAndCapture(command, in_directory, out, fail_status=None,
412 pattern=None, capture_list=None):
413 """Runs command, a list, in directory in_directory.
414
415 A message indicating what is being done, as well as the command's stdout,
416 is printed to out.
417
418 If a pattern is specified, any line in the output matching pattern will have
419 its first match group appended to capture_list.
420
421 If the command fails, as indicated by a nonzero exit status, gclient will
422 exit with an exit status of fail_status. If fail_status is None (the
423 default), gclient will raise an Error exception.
424 """
425
426 print >> out, ("\n________ running \'%s\' in \'%s\'"
427 % (' '.join(command), in_directory))
428
429 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
430 # executable, but shell=True makes subprocess on Linux fail when it's called
431 # with a list because it only tries to execute the first item in the list.
432 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
433 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
434
435 if pattern:
436 compiled_pattern = re.compile(pattern)
437
438 # Also, we need to forward stdout to prevent weird re-ordering of output.
439 # This has to be done on a per byte basis to make sure it is not buffered:
440 # normally buffering is done for each line, but if svn requests input, no
441 # end-of-line character is output after the prompt and it would not show up.
442 in_byte = kid.stdout.read(1)
443 in_line = ""
444 while in_byte:
445 if in_byte != "\r":
446 out.write(in_byte)
447 in_line += in_byte
448 if in_byte == "\n" and pattern:
449 match = compiled_pattern.search(in_line[:-1])
450 if match:
451 capture_list.append(match.group(1))
452 in_line = ""
453 in_byte = kid.stdout.read(1)
454 rv = kid.wait()
455
456 if rv:
457 msg = "failed to run command: %s" % " ".join(command)
458
459 if fail_status != None:
460 print >>sys.stderr, msg
461 sys.exit(fail_status)
462
463 raise Error(msg)
464
465
466def IsUsingGit(root, paths):
467 """Returns True if we're using git to manage any of our checkouts.
468 |entries| is a list of paths to check."""
469 for path in paths:
470 if os.path.exists(os.path.join(root, path, '.git')):
471 return True
472 return False
473
474# -----------------------------------------------------------------------------
475# SVN utils:
476
477
478def RunSVN(options, args, in_directory):
479 """Runs svn, sending output to stdout.
480
481 Args:
482 args: A sequence of command line parameters to be passed to svn.
483 in_directory: The directory where svn is to be run.
484
485 Raises:
486 Error: An error occurred while running the svn command.
487 """
488 c = [SVN_COMMAND]
489 c.extend(args)
490
491 SubprocessCall(c, in_directory, options.stdout)
492
493
494def CaptureSVN(options, args, in_directory):
495 """Runs svn, capturing output sent to stdout as a string.
496
497 Args:
498 args: A sequence of command line parameters to be passed to svn.
499 in_directory: The directory where svn is to be run.
500
501 Returns:
502 The output sent to stdout as a string.
503 """
504 c = [SVN_COMMAND]
505 c.extend(args)
506
507 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
508 # the svn.exe executable, but shell=True makes subprocess on Linux fail
509 # when it's called with a list because it only tries to execute the
510 # first string ("svn").
511 return subprocess.Popen(c, cwd=in_directory, shell=(sys.platform == 'win32'),
512 stdout=subprocess.PIPE).communicate()[0]
513
514
515def RunSVNAndGetFileList(options, args, in_directory, file_list):
516 """Runs svn checkout, update, or status, output to stdout.
517
518 The first item in args must be either "checkout", "update", or "status".
519
520 svn's stdout is parsed to collect a list of files checked out or updated.
521 These files are appended to file_list. svn's stdout is also printed to
522 sys.stdout as in RunSVN.
523
524 Args:
525 args: A sequence of command line parameters to be passed to svn.
526 in_directory: The directory where svn is to be run.
527
528 Raises:
529 Error: An error occurred while running the svn command.
530 """
531 command = [SVN_COMMAND]
532 command.extend(args)
533
534 # svn update and svn checkout use the same pattern: the first three columns
535 # are for file status, property status, and lock status. This is followed
536 # by two spaces, and then the path to the file.
537 update_pattern = '^... (.*)$'
538
539 # The first three columns of svn status are the same as for svn update and
540 # svn checkout. The next three columns indicate addition-with-history,
541 # switch, and remote lock status. This is followed by one space, and then
542 # the path to the file.
543 status_pattern = '^...... (.*)$'
544
545 # args[0] must be a supported command. This will blow up if it's something
546 # else, which is good. Note that the patterns are only effective when
547 # these commands are used in their ordinary forms, the patterns are invalid
548 # for "svn status --show-updates", for example.
549 pattern = {
550 'checkout': update_pattern,
551 'status': status_pattern,
552 'update': update_pattern,
553 }[args[0]]
554
555 SubprocessCallAndCapture(command, in_directory, options.stdout,
556 pattern=pattern, capture_list=file_list)
557
558
559def CaptureSVNInfo(options, relpath, in_directory):
560 """Runs 'svn info' on an existing path.
561
562 Args:
563 relpath: The directory where the working copy resides relative to
564 the directory given by in_directory.
565 in_directory: The directory where svn is to be run.
566
567 Returns:
568 An object with fields corresponding to the output of 'svn info'
569 """
maruel@chromium.org483b0082009-05-07 02:57:14 +0000570 dom = ParseXML(CaptureSVN(options, ["info", "--xml", relpath], in_directory))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000571 result = PrintableObject()
maruel@chromium.org483b0082009-05-07 02:57:14 +0000572 if dom:
573 # /info/entry/
574 # url
575 # reposityory/(root|uuid)
576 # wc-info/(schedule|depth)
577 # commit/(author|date)
578 # str() the results because they may be returned as Unicode, which
579 # interferes with the higher layers matching up things in the deps
580 # dictionary.
581 result = PrintableObject()
582 result.root = str(GetNamedNodeText(dom, 'root'))
583 result.url = str(GetNamedNodeText(dom, 'url'))
584 result.uuid = str(GetNamedNodeText(dom, 'uuid'))
585 result.revision = int(GetNodeNamedAttributeText(dom, 'entry', 'revision'))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000586 return result
587
588
589def CaptureSVNHeadRevision(options, url):
590 """Get the head revision of a SVN repository.
591
592 Returns:
593 Int head revision
594 """
595 info = CaptureSVN(options, ["info", "--xml", url], os.getcwd())
596 dom = xml.dom.minidom.parseString(info)
597 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
598
599
600class FileStatus:
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000601 def __init__(self, path, text_status, props, lock, history):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000602 self.path = path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000603 self.text_status = text_status
604 self.props = props
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000605 self.lock = lock
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000606 self.history = history
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000607
608 def __str__(self):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000609 # Emulate svn status 1.5 output.
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000610 return (self.text_status + self.props + self.lock + self.history + ' ' +
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000611 self.path)
612
613
614def CaptureSVNStatus(options, path):
615 """Runs 'svn status' on an existing path.
616
617 Args:
618 path: The directory to run svn status.
619
620 Returns:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000621 An array of FileStatus corresponding to the emulated output of 'svn status'
622 version 1.5."""
623 dom = ParseXML(CaptureSVN(options, ["status", "--xml"], path))
624 results = []
625 if dom:
626 # /status/target/entry/(wc-status|commit|author|date)
627 for target in dom.getElementsByTagName('target'):
628 base_path = target.getAttribute('path')
629 for entry in target.getElementsByTagName('entry'):
630 file = entry.getAttribute('path')
631 wc_status = entry.getElementsByTagName('wc-status')
632 assert len(wc_status) == 1
633 # Emulate svn 1.5 status ouput...
634 statuses = [' ' for i in range(7)]
635 # Col 0
636 xml_item_status = wc_status[0].getAttribute('item')
637 if xml_item_status == 'unversioned':
638 statuses[0] = '?'
639 elif xml_item_status == 'modified':
640 statuses[0] = 'M'
641 elif xml_item_status == 'added':
642 statuses[0] = 'A'
643 elif xml_item_status == 'conflicted':
644 statuses[0] = 'C'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000645 elif xml_item_status in ('incomplete', 'missing'):
646 statuses[0] = '!'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000647 elif not xml_item_status:
648 pass
649 else:
650 raise Exception('Unknown item status "%s"; please implement me!' %
651 xml_item_status)
652 # Col 1
653 xml_props_status = wc_status[0].getAttribute('props')
654 if xml_props_status == 'modified':
655 statuses[1] = 'M'
656 elif xml_props_status == 'conflicted':
657 statuses[1] = 'C'
658 elif (not xml_props_status or xml_props_status == 'none' or
659 xml_props_status == 'normal'):
660 pass
661 else:
662 raise Exception('Unknown props status "%s"; please implement me!' %
663 xml_props_status)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000664 # Col 2
665 if wc_status[0].getAttribute('wc-locked') == 'true':
666 statuses[2] = 'L'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000667 # Col 3
668 if wc_status[0].getAttribute('copied') == 'true':
669 statuses[3] = '+'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000670 item = FileStatus(file, statuses[0], statuses[1], statuses[2],
671 statuses[3])
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000672 results.append(item)
673 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000674
675
676### SCM abstraction layer
677
678
679class SCMWrapper(object):
680 """Add necessary glue between all the supported SCM.
681
682 This is the abstraction layer to bind to different SCM. Since currently only
683 subversion is supported, a lot of subersionism remains. This can be sorted out
684 once another SCM is supported."""
685 def __init__(self, url=None, root_dir=None, relpath=None,
686 scm_name='svn'):
687 # TODO(maruel): Deduce the SCM from the url.
688 self.scm_name = scm_name
689 self.url = url
690 self._root_dir = root_dir
691 if self._root_dir:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000692 self._root_dir = self._root_dir.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000693 self.relpath = relpath
694 if self.relpath:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000695 self.relpath = self.relpath.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000696
697 def FullUrlForRelativeUrl(self, url):
698 # Find the forth '/' and strip from there. A bit hackish.
699 return '/'.join(self.url.split('/')[:4]) + url
700
701 def RunCommand(self, command, options, args, file_list=None):
702 # file_list will have all files that are modified appended to it.
703
704 if file_list == None:
705 file_list = []
706
707 commands = {
708 'cleanup': self.cleanup,
709 'update': self.update,
710 'revert': self.revert,
711 'status': self.status,
712 'diff': self.diff,
713 'runhooks': self.status,
714 }
715
716 if not command in commands:
717 raise Error('Unknown command %s' % command)
718
719 return commands[command](options, args, file_list)
720
721 def cleanup(self, options, args, file_list):
722 """Cleanup working copy."""
723 command = ['cleanup']
724 command.extend(args)
725 RunSVN(options, command, os.path.join(self._root_dir, self.relpath))
726
727 def diff(self, options, args, file_list):
728 # NOTE: This function does not currently modify file_list.
729 command = ['diff']
730 command.extend(args)
731 RunSVN(options, command, os.path.join(self._root_dir, self.relpath))
732
733 def update(self, options, args, file_list):
734 """Runs SCM to update or transparently checkout the working copy.
735
736 All updated files will be appended to file_list.
737
738 Raises:
739 Error: if can't get URL for relative path.
740 """
741 # Only update if git is not controlling the directory.
742 git_path = os.path.join(self._root_dir, self.relpath, '.git')
743 if options.path_exists(git_path):
744 print >> options.stdout, (
745 "________ found .git directory; skipping %s" % self.relpath)
746 return
747
748 if args:
749 raise Error("Unsupported argument(s): %s" % ",".join(args))
750
751 url = self.url
752 components = url.split("@")
753 revision = None
754 forced_revision = False
755 if options.revision:
756 # Override the revision number.
757 url = '%s@%s' % (components[0], str(options.revision))
758 revision = int(options.revision)
759 forced_revision = True
760 elif len(components) == 2:
761 revision = int(components[1])
762 forced_revision = True
763
764 rev_str = ""
765 if revision:
766 rev_str = ' at %d' % revision
767
768 if not options.path_exists(os.path.join(self._root_dir, self.relpath)):
769 # We need to checkout.
770 command = ['checkout', url, os.path.join(self._root_dir, self.relpath)]
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000771 if revision:
772 command.extend(['--revision', str(revision)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000773 RunSVNAndGetFileList(options, command, self._root_dir, file_list)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000774 return
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000775
776 # Get the existing scm url and the revision number of the current checkout.
777 from_info = CaptureSVNInfo(options,
778 os.path.join(self._root_dir, self.relpath, '.'),
779 '.')
780
781 if options.manually_grab_svn_rev:
782 # Retrieve the current HEAD version because svn is slow at null updates.
783 if not revision:
784 from_info_live = CaptureSVNInfo(options, from_info.url, '.')
785 revision = int(from_info_live.revision)
786 rev_str = ' at %d' % revision
787
788 if from_info.url != components[0]:
789 to_info = CaptureSVNInfo(options, url, '.')
790 if from_info.root != to_info.root:
791 # We have different roots, so check if we can switch --relocate.
792 # Subversion only permits this if the repository UUIDs match.
793 if from_info.uuid != to_info.uuid:
794 raise Error("Can't switch the checkout to %s; UUID don't match" % url)
795
796 # Perform the switch --relocate, then rewrite the from_url
797 # to reflect where we "are now." (This is the same way that
798 # Subversion itself handles the metadata when switch --relocate
799 # is used.) This makes the checks below for whether we
800 # can update to a revision or have to switch to a different
801 # branch work as expected.
802 # TODO(maruel): TEST ME !
803 command = ["switch", "--relocate", from_info.root, to_info.root,
804 self.relpath]
805 RunSVN(options, command, self._root_dir)
806 from_info.url = from_info.url.replace(from_info.root, to_info.root)
807
808 # If the provided url has a revision number that matches the revision
809 # number of the existing directory, then we don't need to bother updating.
810 if not options.force and from_info.revision == revision:
811 if options.verbose or not forced_revision:
812 print >>options.stdout, ("\n_____ %s%s" % (
813 self.relpath, rev_str))
814 return
815
816 command = ["update", os.path.join(self._root_dir, self.relpath)]
817 if revision:
818 command.extend(['--revision', str(revision)])
819 RunSVNAndGetFileList(options, command, self._root_dir, file_list)
820
821 def revert(self, options, args, file_list):
822 """Reverts local modifications. Subversion specific.
823
824 All reverted files will be appended to file_list, even if Subversion
825 doesn't know about them.
826 """
827 path = os.path.join(self._root_dir, self.relpath)
828 if not os.path.isdir(path):
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000829 # svn revert won't work if the directory doesn't exist. It needs to
830 # checkout instead.
831 print >>options.stdout, ("\n_____ %s is missing, synching instead" %
832 self.relpath)
833 # Don't reuse the args.
834 return self.update(options, [], file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000835
836 files = CaptureSVNStatus(options, path)
837 # Batch the command.
838 files_to_revert = []
839 for file in files:
840 file_path = os.path.join(path, file.path)
841 print >>options.stdout, file_path
842 # Unversioned file or unexpected unversioned file.
843 if file.text_status in ('?', '~'):
844 # Remove extraneous file. Also remove unexpected unversioned
845 # directories. svn won't touch them but we want to delete these.
846 file_list.append(file_path)
847 try:
848 os.remove(file_path)
849 except EnvironmentError:
850 RemoveDirectory(file_path)
851
852 if file.text_status != '?':
853 # For any other status, svn revert will work.
854 file_list.append(file_path)
855 files_to_revert.append(file.path)
856
857 # Revert them all at once.
858 if files_to_revert:
859 accumulated_paths = []
860 accumulated_length = 0
861 command = ['revert']
862 for p in files_to_revert:
863 # Some shell have issues with command lines too long.
864 if accumulated_length and accumulated_length + len(p) > 3072:
865 RunSVN(options, command + accumulated_paths,
866 os.path.join(self._root_dir, self.relpath))
867 accumulated_paths = []
868 accumulated_length = 0
869 else:
870 accumulated_paths.append(p)
871 accumulated_length += len(p)
872 if accumulated_paths:
873 RunSVN(options, command + accumulated_paths,
874 os.path.join(self._root_dir, self.relpath))
875
876 def status(self, options, args, file_list):
877 """Display status information."""
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000878 path = os.path.join(self._root_dir, self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000879 command = ['status']
880 command.extend(args)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000881 if not os.path.isdir(path):
882 # svn status won't work if the directory doesn't exist.
883 print >> options.stdout, (
884 "\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
885 "does not exist."
886 % (' '.join(command), path))
887 # There's no file list to retrieve.
888 else:
889 RunSVNAndGetFileList(options, command, path, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000890
891
892## GClient implementation.
893
894
895class GClient(object):
896 """Object that represent a gclient checkout."""
897
898 supported_commands = [
899 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
900 ]
901
902 def __init__(self, root_dir, options):
903 self._root_dir = root_dir
904 self._options = options
905 self._config_content = None
906 self._config_dict = {}
907 self._deps_hooks = []
908
909 def SetConfig(self, content):
910 self._config_dict = {}
911 self._config_content = content
912 exec(content, self._config_dict)
913
914 def SaveConfig(self):
915 FileWrite(os.path.join(self._root_dir, self._options.config_filename),
916 self._config_content)
917
918 def _LoadConfig(self):
919 client_source = FileRead(os.path.join(self._root_dir,
920 self._options.config_filename))
921 self.SetConfig(client_source)
922
923 def ConfigContent(self):
924 return self._config_content
925
926 def GetVar(self, key, default=None):
927 return self._config_dict.get(key, default)
928
929 @staticmethod
930 def LoadCurrentConfig(options, from_dir=None):
931 """Searches for and loads a .gclient file relative to the current working
932 dir.
933
934 Returns:
935 A dict representing the contents of the .gclient file or an empty dict if
936 the .gclient file doesn't exist.
937 """
938 if not from_dir:
939 from_dir = os.curdir
940 path = os.path.realpath(from_dir)
941 while not options.path_exists(os.path.join(path, options.config_filename)):
942 next = os.path.split(path)
943 if not next[1]:
944 return None
945 path = next[0]
946 client = options.gclient(path, options)
947 client._LoadConfig()
948 return client
949
950 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
951 self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
952 solution_name, solution_url, safesync_url
953 ))
954
955 def _SaveEntries(self, entries):
956 """Creates a .gclient_entries file to record the list of unique checkouts.
957
958 The .gclient_entries file lives in the same directory as .gclient.
959
960 Args:
961 entries: A sequence of solution names.
962 """
963 text = "entries = [\n"
964 for entry in entries:
965 text += " \"%s\",\n" % entry
966 text += "]\n"
967 FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
968 text)
969
970 def _ReadEntries(self):
971 """Read the .gclient_entries file for the given client.
972
973 Args:
974 client: The client for which the entries file should be read.
975
976 Returns:
977 A sequence of solution names, which will be empty if there is the
978 entries file hasn't been created yet.
979 """
980 scope = {}
981 filename = os.path.join(self._root_dir, self._options.entries_filename)
982 if not self._options.path_exists(filename):
983 return []
984 exec(FileRead(filename), scope)
985 return scope["entries"]
986
987 class FromImpl:
988 """Used to implement the From syntax."""
989
990 def __init__(self, module_name):
991 self.module_name = module_name
992
993 def __str__(self):
994 return 'From("%s")' % self.module_name
995
996 class _VarImpl:
997 def __init__(self, custom_vars, local_scope):
998 self._custom_vars = custom_vars
999 self._local_scope = local_scope
1000
1001 def Lookup(self, var_name):
1002 """Implements the Var syntax."""
1003 if var_name in self._custom_vars:
1004 return self._custom_vars[var_name]
1005 elif var_name in self._local_scope.get("vars", {}):
1006 return self._local_scope["vars"][var_name]
1007 raise Error("Var is not defined: %s" % var_name)
1008
1009 def _ParseSolutionDeps(self, solution_name, solution_deps_content,
1010 custom_vars):
1011 """Parses the DEPS file for the specified solution.
1012
1013 Args:
1014 solution_name: The name of the solution to query.
1015 solution_deps_content: Content of the DEPS file for the solution
1016 custom_vars: A dict of vars to override any vars defined in the DEPS file.
1017
1018 Returns:
1019 A dict mapping module names (as relative paths) to URLs or an empty
1020 dict if the solution does not have a DEPS file.
1021 """
1022 # Skip empty
1023 if not solution_deps_content:
1024 return {}
1025 # Eval the content
1026 local_scope = {}
1027 var = self._VarImpl(custom_vars, local_scope)
1028 global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
1029 exec(solution_deps_content, global_scope, local_scope)
1030 deps = local_scope.get("deps", {})
1031
1032 # load os specific dependencies if defined. these dependencies may
1033 # override or extend the values defined by the 'deps' member.
1034 if "deps_os" in local_scope:
1035 deps_os_choices = {
1036 "win32": "win",
1037 "win": "win",
1038 "cygwin": "win",
1039 "darwin": "mac",
1040 "mac": "mac",
1041 "unix": "unix",
1042 "linux": "unix",
1043 "linux2": "unix",
1044 }
1045
1046 if self._options.deps_os is not None:
1047 deps_to_include = self._options.deps_os.split(",")
1048 if "all" in deps_to_include:
1049 deps_to_include = deps_os_choices.values()
1050 else:
1051 deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
1052
1053 deps_to_include = set(deps_to_include)
1054 for deps_os_key in deps_to_include:
1055 os_deps = local_scope["deps_os"].get(deps_os_key, {})
1056 if len(deps_to_include) > 1:
1057 # Ignore any overrides when including deps for more than one
1058 # platform, so we collect the broadest set of dependencies available.
1059 # We may end up with the wrong revision of something for our
1060 # platform, but this is the best we can do.
1061 deps.update([x for x in os_deps.items() if not x[0] in deps])
1062 else:
1063 deps.update(os_deps)
1064
1065 if 'hooks' in local_scope:
1066 self._deps_hooks.extend(local_scope['hooks'])
1067
1068 # If use_relative_paths is set in the DEPS file, regenerate
1069 # the dictionary using paths relative to the directory containing
1070 # the DEPS file.
1071 if local_scope.get('use_relative_paths'):
1072 rel_deps = {}
1073 for d, url in deps.items():
1074 # normpath is required to allow DEPS to use .. in their
1075 # dependency local path.
1076 rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
1077 return rel_deps
1078 else:
1079 return deps
1080
1081 def _ParseAllDeps(self, solution_urls, solution_deps_content):
1082 """Parse the complete list of dependencies for the client.
1083
1084 Args:
1085 solution_urls: A dict mapping module names (as relative paths) to URLs
1086 corresponding to the solutions specified by the client. This parameter
1087 is passed as an optimization.
1088 solution_deps_content: A dict mapping module names to the content
1089 of their DEPS files
1090
1091 Returns:
1092 A dict mapping module names (as relative paths) to URLs corresponding
1093 to the entire set of dependencies to checkout for the given client.
1094
1095 Raises:
1096 Error: If a dependency conflicts with another dependency or of a solution.
1097 """
1098 deps = {}
1099 for solution in self.GetVar("solutions"):
1100 custom_vars = solution.get("custom_vars", {})
1101 solution_deps = self._ParseSolutionDeps(
1102 solution["name"],
1103 solution_deps_content[solution["name"]],
1104 custom_vars)
1105
1106 # If a line is in custom_deps, but not in the solution, we want to append
1107 # this line to the solution.
1108 if "custom_deps" in solution:
1109 for d in solution["custom_deps"]:
1110 if d not in solution_deps:
1111 solution_deps[d] = solution["custom_deps"][d]
1112
1113 for d in solution_deps:
1114 if "custom_deps" in solution and d in solution["custom_deps"]:
1115 # Dependency is overriden.
1116 url = solution["custom_deps"][d]
1117 if url is None:
1118 continue
1119 else:
1120 url = solution_deps[d]
1121 # if we have a From reference dependent on another solution, then
1122 # just skip the From reference. When we pull deps for the solution,
1123 # we will take care of this dependency.
1124 #
1125 # If multiple solutions all have the same From reference, then we
1126 # should only add one to our list of dependencies.
1127 if type(url) != str:
1128 if url.module_name in solution_urls:
1129 # Already parsed.
1130 continue
1131 if d in deps and type(deps[d]) != str:
1132 if url.module_name == deps[d].module_name:
1133 continue
1134 else:
1135 parsed_url = urlparse.urlparse(url)
1136 scheme = parsed_url[0]
1137 if not scheme:
1138 # A relative url. Fetch the real base.
1139 path = parsed_url[2]
1140 if path[0] != "/":
1141 raise Error(
1142 "relative DEPS entry \"%s\" must begin with a slash" % d)
1143 # Create a scm just to query the full url.
1144 scm = self._options.scm_wrapper(solution["url"], self._root_dir,
1145 None)
1146 url = scm.FullUrlForRelativeUrl(url)
1147 if d in deps and deps[d] != url:
1148 raise Error(
1149 "Solutions have conflicting versions of dependency \"%s\"" % d)
1150 if d in solution_urls and solution_urls[d] != url:
1151 raise Error(
1152 "Dependency \"%s\" conflicts with specified solution" % d)
1153 # Grab the dependency.
1154 deps[d] = url
1155 return deps
1156
1157 def _RunHookAction(self, hook_dict):
1158 """Runs the action from a single hook.
1159 """
1160 command = hook_dict['action'][:]
1161 if command[0] == 'python':
1162 # If the hook specified "python" as the first item, the action is a
1163 # Python script. Run it by starting a new copy of the same
1164 # interpreter.
1165 command[0] = sys.executable
1166
1167 # Use a discrete exit status code of 2 to indicate that a hook action
1168 # failed. Users of this script may wish to treat hook action failures
1169 # differently from VC failures.
1170 SubprocessCall(command, self._root_dir, self._options.stdout,
1171 fail_status=2)
1172
1173 def _RunHooks(self, command, file_list, is_using_git):
1174 """Evaluates all hooks, running actions as needed.
1175 """
1176 # Hooks only run for these command types.
1177 if not command in ('update', 'revert', 'runhooks'):
1178 return
1179
1180 # Get any hooks from the .gclient file.
1181 hooks = self.GetVar("hooks", [])
1182 # Add any hooks found in DEPS files.
1183 hooks.extend(self._deps_hooks)
1184
1185 # If "--force" was specified, run all hooks regardless of what files have
1186 # changed. If the user is using git, then we don't know what files have
1187 # changed so we always run all hooks.
1188 if self._options.force or is_using_git:
1189 for hook_dict in hooks:
1190 self._RunHookAction(hook_dict)
1191 return
1192
1193 # Run hooks on the basis of whether the files from the gclient operation
1194 # match each hook's pattern.
1195 for hook_dict in hooks:
1196 pattern = re.compile(hook_dict['pattern'])
1197 for file in file_list:
1198 if not pattern.search(file):
1199 continue
1200
1201 self._RunHookAction(hook_dict)
1202
1203 # The hook's action only runs once. Don't bother looking for any
1204 # more matches.
1205 break
1206
1207 def RunOnDeps(self, command, args):
1208 """Runs a command on each dependency in a client and its dependencies.
1209
1210 The module's dependencies are specified in its top-level DEPS files.
1211
1212 Args:
1213 command: The command to use (e.g., 'status' or 'diff')
1214 args: list of str - extra arguments to add to the command line.
1215
1216 Raises:
1217 Error: If the client has conflicting entries.
1218 """
1219 if not command in self.supported_commands:
1220 raise Error("'%s' is an unsupported command" % command)
1221
1222 # Check for revision overrides.
1223 revision_overrides = {}
1224 for revision in self._options.revisions:
1225 if revision.find("@") == -1:
1226 raise Error(
1227 "Specify the full dependency when specifying a revision number.")
1228 revision_elem = revision.split("@")
1229 # Disallow conflicting revs
1230 if revision_overrides.has_key(revision_elem[0]) and \
1231 revision_overrides[revision_elem[0]] != revision_elem[1]:
1232 raise Error(
1233 "Conflicting revision numbers specified.")
1234 revision_overrides[revision_elem[0]] = revision_elem[1]
1235
1236 solutions = self.GetVar("solutions")
1237 if not solutions:
1238 raise Error("No solution specified")
1239
1240 # When running runhooks --force, there's no need to consult the SCM.
1241 # All known hooks are expected to run unconditionally regardless of working
1242 # copy state, so skip the SCM status check.
1243 run_scm = not (command == 'runhooks' and self._options.force)
1244
1245 entries = {}
1246 entries_deps_content = {}
1247 file_list = []
1248 # Run on the base solutions first.
1249 for solution in solutions:
1250 name = solution["name"]
1251 if name in entries:
1252 raise Error("solution %s specified more than once" % name)
1253 url = solution["url"]
1254 entries[name] = url
1255 if run_scm:
1256 self._options.revision = revision_overrides.get(name)
1257 scm = self._options.scm_wrapper(url, self._root_dir, name)
1258 scm.RunCommand(command, self._options, args, file_list)
1259 self._options.revision = None
1260 try:
1261 deps_content = FileRead(os.path.join(self._root_dir, name,
1262 self._options.deps_file))
1263 except IOError, e:
1264 if e.errno != errno.ENOENT:
1265 raise
1266 deps_content = ""
1267 entries_deps_content[name] = deps_content
1268
1269 # Process the dependencies next (sort alphanumerically to ensure that
1270 # containing directories get populated first and for readability)
1271 deps = self._ParseAllDeps(entries, entries_deps_content)
1272 deps_to_process = deps.keys()
1273 deps_to_process.sort()
1274
1275 # First pass for direct dependencies.
1276 for d in deps_to_process:
1277 if type(deps[d]) == str:
1278 url = deps[d]
1279 entries[d] = url
1280 if run_scm:
1281 self._options.revision = revision_overrides.get(d)
1282 scm = self._options.scm_wrapper(url, self._root_dir, d)
1283 scm.RunCommand(command, self._options, args, file_list)
1284 self._options.revision = None
1285
1286 # Second pass for inherited deps (via the From keyword)
1287 for d in deps_to_process:
1288 if type(deps[d]) != str:
1289 sub_deps = self._ParseSolutionDeps(
1290 deps[d].module_name,
1291 FileRead(os.path.join(self._root_dir,
1292 deps[d].module_name,
1293 self._options.deps_file)),
1294 {})
1295 url = sub_deps[d]
1296 entries[d] = url
1297 if run_scm:
1298 self._options.revision = revision_overrides.get(d)
1299 scm = self._options.scm_wrapper(url, self._root_dir, d)
1300 scm.RunCommand(command, self._options, args, file_list)
1301 self._options.revision = None
1302
1303 is_using_git = IsUsingGit(self._root_dir, entries.keys())
1304 self._RunHooks(command, file_list, is_using_git)
1305
1306 if command == 'update':
1307 # notify the user if there is an orphaned entry in their working copy.
1308 # TODO(darin): we should delete this directory manually if it doesn't
1309 # have any changes in it.
1310 prev_entries = self._ReadEntries()
1311 for entry in prev_entries:
1312 e_dir = os.path.join(self._root_dir, entry)
1313 if entry not in entries and self._options.path_exists(e_dir):
1314 if CaptureSVNStatus(self._options, e_dir):
1315 # There are modified files in this entry
1316 entries[entry] = None # Keep warning until removed.
1317 print >> self._options.stdout, (
1318 "\nWARNING: \"%s\" is no longer part of this client. "
1319 "It is recommended that you manually remove it.\n") % entry
1320 else:
1321 # Delete the entry
1322 print >> self._options.stdout, ("\n________ deleting \'%s\' " +
1323 "in \'%s\'") % (entry, self._root_dir)
1324 RemoveDirectory(e_dir)
1325 # record the current list of entries for next time
1326 self._SaveEntries(entries)
1327
1328 def PrintRevInfo(self):
1329 """Output revision info mapping for the client and its dependencies. This
1330 allows the capture of a overall "revision" for the source tree that can
1331 be used to reproduce the same tree in the future. The actual output
1332 contains enough information (source paths, svn server urls and revisions)
1333 that it can be used either to generate external svn commands (without
1334 gclient) or as input to gclient's --rev option (with some massaging of
1335 the data).
1336
1337 NOTE: Unlike RunOnDeps this does not require a local checkout and is run
1338 on the Pulse master. It MUST NOT execute hooks.
1339
1340 Raises:
1341 Error: If the client has conflicting entries.
1342 """
1343 # Check for revision overrides.
1344 revision_overrides = {}
1345 for revision in self._options.revisions:
1346 if revision.find("@") < 0:
1347 raise Error(
1348 "Specify the full dependency when specifying a revision number.")
1349 revision_elem = revision.split("@")
1350 # Disallow conflicting revs
1351 if revision_overrides.has_key(revision_elem[0]) and \
1352 revision_overrides[revision_elem[0]] != revision_elem[1]:
1353 raise Error(
1354 "Conflicting revision numbers specified.")
1355 revision_overrides[revision_elem[0]] = revision_elem[1]
1356
1357 solutions = self.GetVar("solutions")
1358 if not solutions:
1359 raise Error("No solution specified")
1360
1361 entries = {}
1362 entries_deps_content = {}
1363
1364 # Inner helper to generate base url and rev tuple (including honoring
1365 # |revision_overrides|)
1366 def GetURLAndRev(name, original_url):
1367 if original_url.find("@") < 0:
1368 if revision_overrides.has_key(name):
1369 return (original_url, int(revision_overrides[name]))
1370 else:
1371 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1372 return (original_url, CaptureSVNHeadRevision(self._options,
1373 original_url))
1374 else:
1375 url_components = original_url.split("@")
1376 if revision_overrides.has_key(name):
1377 return (url_components[0], int(revision_overrides[name]))
1378 else:
1379 return (url_components[0], int(url_components[1]))
1380
1381 # Run on the base solutions first.
1382 for solution in solutions:
1383 name = solution["name"]
1384 if name in entries:
1385 raise Error("solution %s specified more than once" % name)
1386 (url, rev) = GetURLAndRev(name, solution["url"])
1387 entries[name] = "%s@%d" % (url, rev)
1388 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1389 entries_deps_content[name] = CaptureSVN(
1390 self._options,
1391 ["cat",
1392 "%s/%s@%d" % (url,
1393 self._options.deps_file,
1394 rev)],
1395 os.getcwd())
1396
1397 # Process the dependencies next (sort alphanumerically to ensure that
1398 # containing directories get populated first and for readability)
1399 deps = self._ParseAllDeps(entries, entries_deps_content)
1400 deps_to_process = deps.keys()
1401 deps_to_process.sort()
1402
1403 # First pass for direct dependencies.
1404 for d in deps_to_process:
1405 if type(deps[d]) == str:
1406 (url, rev) = GetURLAndRev(d, deps[d])
1407 entries[d] = "%s@%d" % (url, rev)
1408
1409 # Second pass for inherited deps (via the From keyword)
1410 for d in deps_to_process:
1411 if type(deps[d]) != str:
1412 deps_parent_url = entries[deps[d].module_name]
1413 if deps_parent_url.find("@") < 0:
1414 raise Error("From %s missing revisioned url" % deps[d].module_name)
1415 deps_parent_url_components = deps_parent_url.split("@")
1416 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1417 deps_parent_content = CaptureSVN(
1418 self._options,
1419 ["cat",
1420 "%s/%s@%s" % (deps_parent_url_components[0],
1421 self._options.deps_file,
1422 deps_parent_url_components[1])],
1423 os.getcwd())
1424 sub_deps = self._ParseSolutionDeps(
1425 deps[d].module_name,
1426 FileRead(os.path.join(self._root_dir,
1427 deps[d].module_name,
1428 self._options.deps_file)),
1429 {})
1430 (url, rev) = GetURLAndRev(d, sub_deps[d])
1431 entries[d] = "%s@%d" % (url, rev)
1432
1433 print ";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())])
1434
1435
1436## gclient commands.
1437
1438
1439def DoCleanup(options, args):
1440 """Handle the cleanup subcommand.
1441
1442 Raises:
1443 Error: if client isn't configured properly.
1444 """
1445 client = options.gclient.LoadCurrentConfig(options)
1446 if not client:
1447 raise Error("client not configured; see 'gclient config'")
1448 if options.verbose:
1449 # Print out the .gclient file. This is longer than if we just printed the
1450 # client dict, but more legible, and it might contain helpful comments.
1451 print >>options.stdout, client.ConfigContent()
1452 options.verbose = True
1453 return client.RunOnDeps('cleanup', args)
1454
1455
1456def DoConfig(options, args):
1457 """Handle the config subcommand.
1458
1459 Args:
1460 options: If options.spec set, a string providing contents of config file.
1461 args: The command line args. If spec is not set,
1462 then args[0] is a string URL to get for config file.
1463
1464 Raises:
1465 Error: on usage error
1466 """
1467 if len(args) < 1 and not options.spec:
1468 raise Error("required argument missing; see 'gclient help config'")
1469 if options.path_exists(options.config_filename):
1470 raise Error("%s file already exists in the current directory" %
1471 options.config_filename)
1472 client = options.gclient('.', options)
1473 if options.spec:
1474 client.SetConfig(options.spec)
1475 else:
1476 # TODO(darin): it would be nice to be able to specify an alternate relpath
1477 # for the given URL.
1478 base_url = args[0]
1479 name = args[0].split("/")[-1]
1480 safesync_url = ""
1481 if len(args) > 1:
1482 safesync_url = args[1]
1483 client.SetDefaultConfig(name, base_url, safesync_url)
1484 client.SaveConfig()
1485
1486
1487def DoHelp(options, args):
1488 """Handle the help subcommand giving help for another subcommand.
1489
1490 Raises:
1491 Error: if the command is unknown.
1492 """
1493 if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
1494 print >>options.stdout, COMMAND_USAGE_TEXT[args[0]]
1495 else:
1496 raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
1497
1498
1499def DoStatus(options, args):
1500 """Handle the status subcommand.
1501
1502 Raises:
1503 Error: if client isn't configured properly.
1504 """
1505 client = options.gclient.LoadCurrentConfig(options)
1506 if not client:
1507 raise Error("client not configured; see 'gclient config'")
1508 if options.verbose:
1509 # Print out the .gclient file. This is longer than if we just printed the
1510 # client dict, but more legible, and it might contain helpful comments.
1511 print >>options.stdout, client.ConfigContent()
1512 options.verbose = True
1513 return client.RunOnDeps('status', args)
1514
1515
1516def DoUpdate(options, args):
1517 """Handle the update and sync subcommands.
1518
1519 Raises:
1520 Error: if client isn't configured properly.
1521 """
1522 client = options.gclient.LoadCurrentConfig(options)
1523
1524 if not client:
1525 raise Error("client not configured; see 'gclient config'")
1526
1527 if not options.head:
1528 solutions = client.GetVar('solutions')
1529 if solutions:
1530 for s in solutions:
1531 if s.get('safesync_url', ''):
1532 # rip through revisions and make sure we're not over-riding
1533 # something that was explicitly passed
1534 has_key = False
1535 for r in options.revisions:
1536 if r.split('@')[0] == s['name']:
1537 has_key = True
1538 break
1539
1540 if not has_key:
1541 handle = urllib.urlopen(s['safesync_url'])
1542 rev = handle.read().strip()
1543 handle.close()
1544 if len(rev):
1545 options.revisions.append(s['name']+'@'+rev)
1546
1547 if options.verbose:
1548 # Print out the .gclient file. This is longer than if we just printed the
1549 # client dict, but more legible, and it might contain helpful comments.
1550 print >>options.stdout, client.ConfigContent()
1551 return client.RunOnDeps('update', args)
1552
1553
1554def DoDiff(options, args):
1555 """Handle the diff subcommand.
1556
1557 Raises:
1558 Error: if client isn't configured properly.
1559 """
1560 client = options.gclient.LoadCurrentConfig(options)
1561 if not client:
1562 raise Error("client not configured; see 'gclient config'")
1563 if options.verbose:
1564 # Print out the .gclient file. This is longer than if we just printed the
1565 # client dict, but more legible, and it might contain helpful comments.
1566 print >>options.stdout, client.ConfigContent()
1567 options.verbose = True
1568 return client.RunOnDeps('diff', args)
1569
1570
1571def DoRevert(options, args):
1572 """Handle the revert subcommand.
1573
1574 Raises:
1575 Error: if client isn't configured properly.
1576 """
1577 client = options.gclient.LoadCurrentConfig(options)
1578 if not client:
1579 raise Error("client not configured; see 'gclient config'")
1580 return client.RunOnDeps('revert', args)
1581
1582
1583def DoRunHooks(options, args):
1584 """Handle the runhooks subcommand.
1585
1586 Raises:
1587 Error: if client isn't configured properly.
1588 """
1589 client = options.gclient.LoadCurrentConfig(options)
1590 if not client:
1591 raise Error("client not configured; see 'gclient config'")
1592 if options.verbose:
1593 # Print out the .gclient file. This is longer than if we just printed the
1594 # client dict, but more legible, and it might contain helpful comments.
1595 print >>options.stdout, client.ConfigContent()
1596 return client.RunOnDeps('runhooks', args)
1597
1598
1599def DoRevInfo(options, args):
1600 """Handle the revinfo subcommand.
1601
1602 Raises:
1603 Error: if client isn't configured properly.
1604 """
1605 client = options.gclient.LoadCurrentConfig(options)
1606 if not client:
1607 raise Error("client not configured; see 'gclient config'")
1608 client.PrintRevInfo()
1609
1610
1611gclient_command_map = {
1612 "cleanup": DoCleanup,
1613 "config": DoConfig,
1614 "diff": DoDiff,
1615 "help": DoHelp,
1616 "status": DoStatus,
1617 "sync": DoUpdate,
1618 "update": DoUpdate,
1619 "revert": DoRevert,
1620 "runhooks": DoRunHooks,
1621 "revinfo" : DoRevInfo,
1622}
1623
1624
1625def DispatchCommand(command, options, args, command_map=None):
1626 """Dispatches the appropriate subcommand based on command line arguments."""
1627 if command_map is None:
1628 command_map = gclient_command_map
1629
1630 if command in command_map:
1631 return command_map[command](options, args)
1632 else:
1633 raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
1634
1635
1636def Main(argv):
1637 """Parse command line arguments and dispatch command."""
1638
1639 option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
1640 version=__version__)
1641 option_parser.disable_interspersed_args()
1642 option_parser.add_option("", "--force", action="store_true", default=False,
1643 help=("(update/sync only) force update even "
1644 "for modules which haven't changed"))
1645 option_parser.add_option("", "--revision", action="append", dest="revisions",
1646 metavar="REV", default=[],
1647 help=("(update/sync only) sync to a specific "
1648 "revision, can be used multiple times for "
1649 "each solution, e.g. --revision=src@123, "
1650 "--revision=internal@32"))
1651 option_parser.add_option("", "--deps", default=None, dest="deps_os",
1652 metavar="OS_LIST",
1653 help=("(update/sync only) sync deps for the "
1654 "specified (comma-separated) platform(s); "
1655 "'all' will sync all platforms"))
1656 option_parser.add_option("", "--spec", default=None,
1657 help=("(config only) create a gclient file "
1658 "containing the provided string"))
1659 option_parser.add_option("", "--verbose", action="store_true", default=False,
1660 help="produce additional output for diagnostics")
1661 option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
1662 default=False,
1663 help="Skip svn up whenever possible by requesting "
1664 "actual HEAD revision from the repository")
1665 option_parser.add_option("", "--head", action="store_true", default=False,
1666 help=("skips any safesync_urls specified in "
1667 "configured solutions"))
1668
1669 if len(argv) < 2:
1670 # Users don't need to be told to use the 'help' command.
1671 option_parser.print_help()
1672 return 1
1673 # Add manual support for --version as first argument.
1674 if argv[1] == '--version':
1675 option_parser.print_version()
1676 return 0
1677
1678 # Add manual support for --help as first argument.
1679 if argv[1] == '--help':
1680 argv[1] = 'help'
1681
1682 command = argv[1]
1683 options, args = option_parser.parse_args(argv[2:])
1684
1685 if len(argv) < 3 and command == "help":
1686 option_parser.print_help()
1687 return 0
1688
1689 # Files used for configuration and state saving.
1690 options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
1691 options.entries_filename = ".gclient_entries"
1692 options.deps_file = "DEPS"
1693
1694 # These are overridded when testing. They are not externally visible.
1695 options.stdout = sys.stdout
1696 options.path_exists = os.path.exists
1697 options.gclient = GClient
1698 options.scm_wrapper = SCMWrapper
1699 options.platform = sys.platform
1700 return DispatchCommand(command, options, args)
1701
1702
1703if "__main__" == __name__:
1704 try:
1705 result = Main(sys.argv)
1706 except Error, e:
1707 print "Error: %s" % str(e)
1708 result = 1
1709 sys.exit(result)
1710
1711# vim: ts=2:sw=2:tw=80:et: