blob: 71cb00804b00d93604f5c2b327ccf7caea413d65 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2#
3# Copyright 2008 Google Inc. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""A wrapper script to manage a set of client modules in different SCM.
18
19This script is intended to be used to help basic management of client
20program sources residing in one or more Subversion modules, along with
21other modules it depends on, also in Subversion, but possibly on
22multiple respositories, making a wrapper system apparently necessary.
23
24Files
25 .gclient : Current client configuration, written by 'config' command.
26 Format is a Python script defining 'solutions', a list whose
27 entries each are maps binding the strings "name" and "url"
28 to strings specifying the name and location of the client
29 module, as well as "custom_deps" to a map similar to the DEPS
30 file below.
31 .gclient_entries : A cache constructed by 'update' command. Format is a
32 Python script defining 'entries', a list of the names
33 of all modules in the client
34 <module>/DEPS : Python script defining var 'deps' as a map from each requisite
35 submodule name to a URL where it can be found (via one SCM)
36
37Hooks
38 .gclient and DEPS files may optionally contain a list named "hooks" to
39 allow custom actions to be performed based on files that have changed in the
40 working copy as a result of a "sync"/"update" or "revert" operation. Hooks
41 can also be run based on what files have been modified in the working copy
42 with the "runhooks" operation. If any of these operation are run with
43 --force, all known hooks will run regardless of the state of the working
44 copy.
45
46 Each item in a "hooks" list is a dict, containing these two keys:
47 "pattern" The associated value is a string containing a regular
48 expression. When a file whose pathname matches the expression
49 is checked out, updated, or reverted, the hook's "action" will
50 run.
51 "action" A list describing a command to run along with its arguments, if
52 any. An action command will run at most one time per gclient
53 invocation, regardless of how many files matched the pattern.
54 The action is executed in the same directory as the .gclient
55 file. If the first item in the list is the string "python",
56 the current Python interpreter (sys.executable) will be used
57 to run the command.
58
59 Example:
60 hooks = [
61 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
62 "action": ["python", "image_indexer.py", "--all"]},
63 ]
64"""
65
66__author__ = "darinf@gmail.com (Darin Fisher)"
maruel@chromium.orge77173e2009-05-12 17:03:33 +000067__version__ = "0.3.2"
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000068
69import errno
70import optparse
71import os
72import re
73import stat
74import subprocess
75import sys
76import time
77import urlparse
78import xml.dom.minidom
79import urllib
80
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000081
82SVN_COMMAND = "svn"
83
84
85# default help text
86DEFAULT_USAGE_TEXT = (
87"""usage: %prog <subcommand> [options] [--] [svn options/args...]
88a wrapper for managing a set of client modules in svn.
89Version """ + __version__ + """
90
91subcommands:
92 cleanup
93 config
94 diff
95 revert
96 status
97 sync
98 update
99 runhooks
100 revinfo
101
102Options and extra arguments can be passed to invoked svn commands by
103appending them to the command line. Note that if the first such
104appended option starts with a dash (-) then the options must be
105preceded by -- to distinguish them from gclient options.
106
107For additional help on a subcommand or examples of usage, try
108 %prog help <subcommand>
109 %prog help files
110""")
111
112GENERIC_UPDATE_USAGE_TEXT = (
113 """Perform a checkout/update of the modules specified by the gclient
114configuration; see 'help config'. Unless --revision is specified,
115then the latest revision of the root solutions is checked out, with
116dependent submodule versions updated according to DEPS files.
117If --revision is specified, then the given revision is used in place
118of the latest, either for a single solution or for all solutions.
119Unless the --force option is provided, solutions and modules whose
120local revision matches the one to update (i.e., they have not changed
121in the repository) are *not* modified.
122This a synonym for 'gclient %(alias)s'
123
124usage: gclient %(cmd)s [options] [--] [svn update options/args]
125
126Valid options:
127 --force : force update even for unchanged modules
128 --revision REV : update/checkout all solutions with specified revision
129 --revision SOLUTION@REV : update given solution to specified revision
130 --deps PLATFORM(S) : sync deps for the given platform(s), or 'all'
131 --verbose : output additional diagnostics
132
133Examples:
134 gclient %(cmd)s
135 update files from SVN according to current configuration,
136 *for modules which have changed since last update or sync*
137 gclient %(cmd)s --force
138 update files from SVN according to current configuration, for
139 all modules (useful for recovering files deleted from local copy)
140""")
141
142COMMAND_USAGE_TEXT = {
143 "cleanup":
144 """Clean up all working copies, using 'svn cleanup' for each module.
145Additional options and args may be passed to 'svn cleanup'.
146
147usage: cleanup [options] [--] [svn cleanup args/options]
148
149Valid options:
150 --verbose : output additional diagnostics
151""",
152 "config": """Create a .gclient file in the current directory; this
153specifies the configuration for further commands. After update/sync,
154top-level DEPS files in each module are read to determine dependent
155modules to operate on as well. If optional [url] parameter is
156provided, then configuration is read from a specified Subversion server
157URL. Otherwise, a --spec option must be provided.
158
159usage: config [option | url] [safesync url]
160
161Valid options:
162 --spec=GCLIENT_SPEC : contents of .gclient are read from string parameter.
163 *Note that due to Cygwin/Python brokenness, it
164 probably can't contain any newlines.*
165
166Examples:
167 gclient config https://gclient.googlecode.com/svn/trunk/gclient
168 configure a new client to check out gclient.py tool sources
169 gclient config --spec='solutions=[{"name":"gclient","""
170 '"url":"https://gclient.googlecode.com/svn/trunk/gclient",'
171 '"custom_deps":{}}]',
172 "diff": """Display the differences between two revisions of modules.
173(Does 'svn diff' for each checked out module and dependences.)
174Additional args and options to 'svn diff' can be passed after
175gclient options.
176
177usage: diff [options] [--] [svn args/options]
178
179Valid options:
180 --verbose : output additional diagnostics
181
182Examples:
183 gclient diff
184 simple 'svn diff' for configured client and dependences
185 gclient diff -- -x -b
186 use 'svn diff -x -b' to suppress whitespace-only differences
187 gclient diff -- -r HEAD -x -b
188 diff versus the latest version of each module
189""",
190 "revert":
191 """Revert every file in every managed directory in the client view.
192
193usage: revert
194""",
195 "status":
196 """Show the status of client and dependent modules, using 'svn diff'
197for each module. Additional options and args may be passed to 'svn diff'.
198
199usage: status [options] [--] [svn diff args/options]
200
201Valid options:
202 --verbose : output additional diagnostics
203""",
204 "sync": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "sync", "alias": "update"},
205 "update": GENERIC_UPDATE_USAGE_TEXT % {"cmd": "update", "alias": "sync"},
206 "help": """Describe the usage of this program or its subcommands.
207
208usage: help [options] [subcommand]
209
210Valid options:
211 --verbose : output additional diagnostics
212""",
213 "runhooks":
214 """Runs hooks for files that have been modified in the local working copy,
215according to 'svn status'.
216
217usage: runhooks [options]
218
219Valid options:
220 --force : runs all known hooks, regardless of the working
221 copy status
222 --verbose : output additional diagnostics
223""",
224 "revinfo":
225 """Outputs source path, server URL and revision information for every
226dependency in all solutions (no local checkout required).
227
228usage: revinfo [options]
229""",
230}
231
232# parameterized by (solution_name, solution_url, safesync_url)
233DEFAULT_CLIENT_FILE_TEXT = (
234 """
235# An element of this array (a \"solution\") describes a repository directory
236# that will be checked out into your working copy. Each solution may
237# optionally define additional dependencies (via its DEPS file) to be
238# checked out alongside the solution's directory. A solution may also
239# specify custom dependencies (via the \"custom_deps\" property) that
240# override or augment the dependencies specified by the DEPS file.
241# If a \"safesync_url\" is specified, it is assumed to reference the location of
242# a text file which contains nothing but the last known good SCM revision to
243# sync against. It is fetched if specified and used unless --head is passed
244solutions = [
245 { \"name\" : \"%s\",
246 \"url\" : \"%s\",
247 \"custom_deps\" : {
248 # To use the trunk of a component instead of what's in DEPS:
249 #\"component\": \"https://svnserver/component/trunk/\",
250 # To exclude a component from your working copy:
251 #\"data/really_large_component\": None,
252 },
253 \"safesync_url\": \"%s\"
254 }
255]
256""")
257
258
259## Generic utils
260
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000261def ParseXML(output):
262 try:
263 return xml.dom.minidom.parseString(output)
264 except xml.parsers.expat.ExpatError:
265 return None
266
267
maruel@chromium.org483b0082009-05-07 02:57:14 +0000268def GetNamedNodeText(node, node_name):
269 child_nodes = node.getElementsByTagName(node_name)
270 if not child_nodes:
271 return None
272 assert len(child_nodes) == 1 and child_nodes[0].childNodes.length == 1
273 return child_nodes[0].firstChild.nodeValue
274
275
276def GetNodeNamedAttributeText(node, node_name, attribute_name):
277 child_nodes = node.getElementsByTagName(node_name)
278 if not child_nodes:
279 return None
280 assert len(child_nodes) == 1
281 return child_nodes[0].getAttribute(attribute_name)
282
283
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284class Error(Exception):
285 """gclient exception class."""
286 pass
287
288class PrintableObject(object):
289 def __str__(self):
290 output = ''
291 for i in dir(self):
292 if i.startswith('__'):
293 continue
294 output += '%s = %s\n' % (i, str(getattr(self, i, '')))
295 return output
296
297
298def FileRead(filename):
299 content = None
300 f = open(filename, "rU")
301 try:
302 content = f.read()
303 finally:
304 f.close()
305 return content
306
307
308def FileWrite(filename, content):
309 f = open(filename, "w")
310 try:
311 f.write(content)
312 finally:
313 f.close()
314
315
316def RemoveDirectory(*path):
317 """Recursively removes a directory, even if it's marked read-only.
318
319 Remove the directory located at *path, if it exists.
320
321 shutil.rmtree() doesn't work on Windows if any of the files or directories
322 are read-only, which svn repositories and some .svn files are. We need to
323 be able to force the files to be writable (i.e., deletable) as we traverse
324 the tree.
325
326 Even with all this, Windows still sometimes fails to delete a file, citing
327 a permission error (maybe something to do with antivirus scans or disk
328 indexing). The best suggestion any of the user forums had was to wait a
329 bit and try again, so we do that too. It's hand-waving, but sometimes it
330 works. :/
331
332 On POSIX systems, things are a little bit simpler. The modes of the files
333 to be deleted doesn't matter, only the modes of the directories containing
334 them are significant. As the directory tree is traversed, each directory
335 has its mode set appropriately before descending into it. This should
336 result in the entire tree being removed, with the possible exception of
337 *path itself, because nothing attempts to change the mode of its parent.
338 Doing so would be hazardous, as it's not a directory slated for removal.
339 In the ordinary case, this is not a problem: for our purposes, the user
340 will never lack write permission on *path's parent.
341 """
342 file_path = os.path.join(*path)
343 if not os.path.exists(file_path):
344 return
345
346 if os.path.islink(file_path) or not os.path.isdir(file_path):
347 raise Error("RemoveDirectory asked to remove non-directory %s" % file_path)
348
349 has_win32api = False
350 if sys.platform == 'win32':
351 has_win32api = True
352 # Some people don't have the APIs installed. In that case we'll do without.
353 try:
354 win32api = __import__('win32api')
355 win32con = __import__('win32con')
356 except ImportError:
357 has_win32api = False
358 else:
359 # On POSIX systems, we need the x-bit set on the directory to access it,
360 # the r-bit to see its contents, and the w-bit to remove files from it.
361 # The actual modes of the files within the directory is irrelevant.
362 os.chmod(file_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
363 for fn in os.listdir(file_path):
364 fullpath = os.path.join(file_path, fn)
365
366 # If fullpath is a symbolic link that points to a directory, isdir will
367 # be True, but we don't want to descend into that as a directory, we just
368 # want to remove the link. Check islink and treat links as ordinary files
369 # would be treated regardless of what they reference.
370 if os.path.islink(fullpath) or not os.path.isdir(fullpath):
371 if sys.platform == 'win32':
372 os.chmod(fullpath, stat.S_IWRITE)
373 if has_win32api:
374 win32api.SetFileAttributes(fullpath, win32con.FILE_ATTRIBUTE_NORMAL)
375 try:
376 os.remove(fullpath)
377 except OSError, e:
378 if e.errno != errno.EACCES or sys.platform != 'win32':
379 raise
380 print 'Failed to delete %s: trying again' % fullpath
381 time.sleep(0.1)
382 os.remove(fullpath)
383 else:
384 RemoveDirectory(fullpath)
385
386 if sys.platform == 'win32':
387 os.chmod(file_path, stat.S_IWRITE)
388 if has_win32api:
389 win32api.SetFileAttributes(file_path, win32con.FILE_ATTRIBUTE_NORMAL)
390 try:
391 os.rmdir(file_path)
392 except OSError, e:
393 if e.errno != errno.EACCES or sys.platform != 'win32':
394 raise
395 print 'Failed to remove %s: trying again' % file_path
396 time.sleep(0.1)
397 os.rmdir(file_path)
398
399
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000400def SubprocessCall(command, in_directory, fail_status=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000401 """Runs command, a list, in directory in_directory.
402
403 This function wraps SubprocessCallAndCapture, but does not perform the
404 capturing functions. See that function for a more complete usage
405 description.
406 """
407 # Call subprocess and capture nothing:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000408 SubprocessCallAndCapture(command, in_directory, fail_status)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000409
410
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000411def SubprocessCallAndCapture(command, in_directory, fail_status=None,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000412 pattern=None, capture_list=None):
413 """Runs command, a list, in directory in_directory.
414
415 A message indicating what is being done, as well as the command's stdout,
416 is printed to out.
417
418 If a pattern is specified, any line in the output matching pattern will have
419 its first match group appended to capture_list.
420
421 If the command fails, as indicated by a nonzero exit status, gclient will
422 exit with an exit status of fail_status. If fail_status is None (the
423 default), gclient will raise an Error exception.
424 """
425
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000426 print("\n________ running \'%s\' in \'%s\'"
427 % (' '.join(command), in_directory))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000428
429 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for the
430 # executable, but shell=True makes subprocess on Linux fail when it's called
431 # with a list because it only tries to execute the first item in the list.
432 kid = subprocess.Popen(command, bufsize=0, cwd=in_directory,
433 shell=(sys.platform == 'win32'), stdout=subprocess.PIPE)
434
435 if pattern:
436 compiled_pattern = re.compile(pattern)
437
438 # Also, we need to forward stdout to prevent weird re-ordering of output.
439 # This has to be done on a per byte basis to make sure it is not buffered:
440 # normally buffering is done for each line, but if svn requests input, no
441 # end-of-line character is output after the prompt and it would not show up.
442 in_byte = kid.stdout.read(1)
443 in_line = ""
444 while in_byte:
445 if in_byte != "\r":
446 out.write(in_byte)
447 in_line += in_byte
448 if in_byte == "\n" and pattern:
449 match = compiled_pattern.search(in_line[:-1])
450 if match:
451 capture_list.append(match.group(1))
452 in_line = ""
453 in_byte = kid.stdout.read(1)
454 rv = kid.wait()
455
456 if rv:
457 msg = "failed to run command: %s" % " ".join(command)
458
459 if fail_status != None:
460 print >>sys.stderr, msg
461 sys.exit(fail_status)
462
463 raise Error(msg)
464
465
466def IsUsingGit(root, paths):
467 """Returns True if we're using git to manage any of our checkouts.
468 |entries| is a list of paths to check."""
469 for path in paths:
470 if os.path.exists(os.path.join(root, path, '.git')):
471 return True
472 return False
473
474# -----------------------------------------------------------------------------
475# SVN utils:
476
477
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000478def RunSVN(args, in_directory):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000479 """Runs svn, sending output to stdout.
480
481 Args:
482 args: A sequence of command line parameters to be passed to svn.
483 in_directory: The directory where svn is to be run.
484
485 Raises:
486 Error: An error occurred while running the svn command.
487 """
488 c = [SVN_COMMAND]
489 c.extend(args)
490
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000491 SubprocessCall(c, in_directory)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000492
493
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000494def CaptureSVN(args, in_directory):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000495 """Runs svn, capturing output sent to stdout as a string.
496
497 Args:
498 args: A sequence of command line parameters to be passed to svn.
499 in_directory: The directory where svn is to be run.
500
501 Returns:
502 The output sent to stdout as a string.
503 """
504 c = [SVN_COMMAND]
505 c.extend(args)
506
507 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
508 # the svn.exe executable, but shell=True makes subprocess on Linux fail
509 # when it's called with a list because it only tries to execute the
510 # first string ("svn").
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000511 return subprocess.Popen(c,
512 cwd=in_directory,
513 shell=(sys.platform == 'win32'),
maruel@chromium.org11e0fd62009-05-12 00:47:31 +0000514 stdout=subprocess.PIPE).communicate()[0]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000515
516
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000517def RunSVNAndGetFileList(args, in_directory, file_list):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000518 """Runs svn checkout, update, or status, output to stdout.
519
520 The first item in args must be either "checkout", "update", or "status".
521
522 svn's stdout is parsed to collect a list of files checked out or updated.
523 These files are appended to file_list. svn's stdout is also printed to
524 sys.stdout as in RunSVN.
525
526 Args:
527 args: A sequence of command line parameters to be passed to svn.
528 in_directory: The directory where svn is to be run.
529
530 Raises:
531 Error: An error occurred while running the svn command.
532 """
533 command = [SVN_COMMAND]
534 command.extend(args)
535
536 # svn update and svn checkout use the same pattern: the first three columns
537 # are for file status, property status, and lock status. This is followed
538 # by two spaces, and then the path to the file.
539 update_pattern = '^... (.*)$'
540
541 # The first three columns of svn status are the same as for svn update and
542 # svn checkout. The next three columns indicate addition-with-history,
543 # switch, and remote lock status. This is followed by one space, and then
544 # the path to the file.
545 status_pattern = '^...... (.*)$'
546
547 # args[0] must be a supported command. This will blow up if it's something
548 # else, which is good. Note that the patterns are only effective when
549 # these commands are used in their ordinary forms, the patterns are invalid
550 # for "svn status --show-updates", for example.
551 pattern = {
552 'checkout': update_pattern,
553 'status': status_pattern,
554 'update': update_pattern,
555 }[args[0]]
556
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000557 SubprocessCallAndCapture(command,
558 in_directory,
559 pattern=pattern,
560 capture_list=file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000561
562
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000563def CaptureSVNInfo(relpath, in_directory=None):
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000564 """Returns a dictionary from the svn info output for the given file.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000565
566 Args:
567 relpath: The directory where the working copy resides relative to
568 the directory given by in_directory.
569 in_directory: The directory where svn is to be run.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000570 """
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000571 output = CaptureSVN(["info", "--xml", relpath], in_directory)
572 dom = ParseXML(output)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000573 result = {}
maruel@chromium.org483b0082009-05-07 02:57:14 +0000574 if dom:
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000575 def C(item, f):
576 if item is not None: return f(item)
maruel@chromium.org483b0082009-05-07 02:57:14 +0000577 # /info/entry/
578 # url
579 # reposityory/(root|uuid)
580 # wc-info/(schedule|depth)
581 # commit/(author|date)
582 # str() the results because they may be returned as Unicode, which
583 # interferes with the higher layers matching up things in the deps
584 # dictionary.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000585 # TODO(maruel): Fix at higher level instead (!)
586 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
587 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
588 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
589 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
590 int)
591 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
592 str)
593 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
594 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
595 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
596 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000597 return result
598
599
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000600def CaptureSVNHeadRevision(url):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000601 """Get the head revision of a SVN repository.
602
603 Returns:
604 Int head revision
605 """
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000606 info = CaptureSVN(["info", "--xml", url], os.getcwd())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000607 dom = xml.dom.minidom.parseString(info)
608 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
609
610
611class FileStatus:
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000612 def __init__(self, path, text_status, props, lock, history):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000613 self.path = path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000614 self.text_status = text_status
615 self.props = props
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000616 self.lock = lock
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000617 self.history = history
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000618
619 def __str__(self):
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000620 # Emulate svn status 1.5 output.
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000621 return (self.text_status + self.props + self.lock + self.history + ' ' +
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000622 self.path)
623
624
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000625def CaptureSVNStatus(path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626 """Runs 'svn status' on an existing path.
627
628 Args:
629 path: The directory to run svn status.
630
631 Returns:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000632 An array of FileStatus corresponding to the emulated output of 'svn status'
633 version 1.5."""
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000634 dom = ParseXML(CaptureSVN(["status", "--xml"], path))
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000635 results = []
636 if dom:
637 # /status/target/entry/(wc-status|commit|author|date)
638 for target in dom.getElementsByTagName('target'):
639 base_path = target.getAttribute('path')
640 for entry in target.getElementsByTagName('entry'):
641 file = entry.getAttribute('path')
642 wc_status = entry.getElementsByTagName('wc-status')
643 assert len(wc_status) == 1
644 # Emulate svn 1.5 status ouput...
645 statuses = [' ' for i in range(7)]
646 # Col 0
647 xml_item_status = wc_status[0].getAttribute('item')
648 if xml_item_status == 'unversioned':
649 statuses[0] = '?'
650 elif xml_item_status == 'modified':
651 statuses[0] = 'M'
652 elif xml_item_status == 'added':
653 statuses[0] = 'A'
654 elif xml_item_status == 'conflicted':
655 statuses[0] = 'C'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000656 elif xml_item_status in ('incomplete', 'missing'):
657 statuses[0] = '!'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000658 elif not xml_item_status:
659 pass
660 else:
661 raise Exception('Unknown item status "%s"; please implement me!' %
662 xml_item_status)
663 # Col 1
664 xml_props_status = wc_status[0].getAttribute('props')
665 if xml_props_status == 'modified':
666 statuses[1] = 'M'
667 elif xml_props_status == 'conflicted':
668 statuses[1] = 'C'
669 elif (not xml_props_status or xml_props_status == 'none' or
670 xml_props_status == 'normal'):
671 pass
672 else:
673 raise Exception('Unknown props status "%s"; please implement me!' %
674 xml_props_status)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000675 # Col 2
676 if wc_status[0].getAttribute('wc-locked') == 'true':
677 statuses[2] = 'L'
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000678 # Col 3
679 if wc_status[0].getAttribute('copied') == 'true':
680 statuses[3] = '+'
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000681 item = FileStatus(file, statuses[0], statuses[1], statuses[2],
682 statuses[3])
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000683 results.append(item)
684 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000685
686
687### SCM abstraction layer
688
689
690class SCMWrapper(object):
691 """Add necessary glue between all the supported SCM.
692
693 This is the abstraction layer to bind to different SCM. Since currently only
694 subversion is supported, a lot of subersionism remains. This can be sorted out
695 once another SCM is supported."""
696 def __init__(self, url=None, root_dir=None, relpath=None,
697 scm_name='svn'):
698 # TODO(maruel): Deduce the SCM from the url.
699 self.scm_name = scm_name
700 self.url = url
701 self._root_dir = root_dir
702 if self._root_dir:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000703 self._root_dir = self._root_dir.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000704 self.relpath = relpath
705 if self.relpath:
maruel@chromium.orge105d8d2009-04-30 17:58:25 +0000706 self.relpath = self.relpath.replace('/', os.sep)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000707
708 def FullUrlForRelativeUrl(self, url):
709 # Find the forth '/' and strip from there. A bit hackish.
710 return '/'.join(self.url.split('/')[:4]) + url
711
712 def RunCommand(self, command, options, args, file_list=None):
713 # file_list will have all files that are modified appended to it.
714
715 if file_list == None:
716 file_list = []
717
718 commands = {
719 'cleanup': self.cleanup,
720 'update': self.update,
721 'revert': self.revert,
722 'status': self.status,
723 'diff': self.diff,
724 'runhooks': self.status,
725 }
726
727 if not command in commands:
728 raise Error('Unknown command %s' % command)
729
730 return commands[command](options, args, file_list)
731
732 def cleanup(self, options, args, file_list):
733 """Cleanup working copy."""
734 command = ['cleanup']
735 command.extend(args)
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000736 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000737
738 def diff(self, options, args, file_list):
739 # NOTE: This function does not currently modify file_list.
740 command = ['diff']
741 command.extend(args)
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000742 RunSVN(command, os.path.join(self._root_dir, self.relpath))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000743
744 def update(self, options, args, file_list):
745 """Runs SCM to update or transparently checkout the working copy.
746
747 All updated files will be appended to file_list.
748
749 Raises:
750 Error: if can't get URL for relative path.
751 """
752 # Only update if git is not controlling the directory.
753 git_path = os.path.join(self._root_dir, self.relpath, '.git')
754 if options.path_exists(git_path):
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000755 print("________ found .git directory; skipping %s" % self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000756 return
757
758 if args:
759 raise Error("Unsupported argument(s): %s" % ",".join(args))
760
761 url = self.url
762 components = url.split("@")
763 revision = None
764 forced_revision = False
765 if options.revision:
766 # Override the revision number.
767 url = '%s@%s' % (components[0], str(options.revision))
768 revision = int(options.revision)
769 forced_revision = True
770 elif len(components) == 2:
771 revision = int(components[1])
772 forced_revision = True
773
774 rev_str = ""
775 if revision:
776 rev_str = ' at %d' % revision
777
778 if not options.path_exists(os.path.join(self._root_dir, self.relpath)):
779 # We need to checkout.
780 command = ['checkout', url, os.path.join(self._root_dir, self.relpath)]
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000781 if revision:
782 command.extend(['--revision', str(revision)])
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000783 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000784 return
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000785
786 # Get the existing scm url and the revision number of the current checkout.
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000787 from_info = CaptureSVNInfo(os.path.join(self._root_dir, self.relpath, '.'),
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000788 '.')
789
790 if options.manually_grab_svn_rev:
791 # Retrieve the current HEAD version because svn is slow at null updates.
792 if not revision:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000793 from_info_live = CaptureSVNInfo(from_info['URL'], '.')
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000794 revision = int(from_info_live['Revision'])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000795 rev_str = ' at %d' % revision
796
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000797 if from_info['URL'] != components[0]:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000798 to_info = CaptureSVNInfo(url, '.')
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000799 if from_info['Repository Root'] != to_info['Repository Root']:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000800 # We have different roots, so check if we can switch --relocate.
801 # Subversion only permits this if the repository UUIDs match.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000802 if from_info['UUID'] != to_info['UUID']:
803 raise Error("Can't switch the checkout to %s; UUID don't match. That "
804 "simply means in theory, gclient should verify you don't "
805 "have a local change, remove the old checkout and do a "
806 "fresh new checkout of the new repo. Contributions are "
807 "welcome." % url)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000808
809 # Perform the switch --relocate, then rewrite the from_url
810 # to reflect where we "are now." (This is the same way that
811 # Subversion itself handles the metadata when switch --relocate
812 # is used.) This makes the checks below for whether we
813 # can update to a revision or have to switch to a different
814 # branch work as expected.
815 # TODO(maruel): TEST ME !
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000816 command = ["switch", "--relocate",
817 from_info['Repository Root'],
818 to_info['Repository Root'],
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000819 self.relpath]
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000820 RunSVN(command, self._root_dir)
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000821 from_info['URL'] = from_info['URL'].replace(
822 from_info['Repository Root'],
823 to_info['Repository Root'])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000824
825 # If the provided url has a revision number that matches the revision
826 # number of the existing directory, then we don't need to bother updating.
maruel@chromium.org2dc8a4d2009-05-11 12:41:20 +0000827 if not options.force and from_info['Revision'] == revision:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000828 if options.verbose or not forced_revision:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000829 print("\n_____ %s%s" % (self.relpath, rev_str))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000830 return
831
832 command = ["update", os.path.join(self._root_dir, self.relpath)]
833 if revision:
834 command.extend(['--revision', str(revision)])
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000835 RunSVNAndGetFileList(command, self._root_dir, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000836
837 def revert(self, options, args, file_list):
838 """Reverts local modifications. Subversion specific.
839
840 All reverted files will be appended to file_list, even if Subversion
841 doesn't know about them.
842 """
843 path = os.path.join(self._root_dir, self.relpath)
844 if not os.path.isdir(path):
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000845 # svn revert won't work if the directory doesn't exist. It needs to
846 # checkout instead.
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000847 print("\n_____ %s is missing, synching instead" % self.relpath)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000848 # Don't reuse the args.
849 return self.update(options, [], file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000850
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000851 files = CaptureSVNStatus(path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000852 # Batch the command.
853 files_to_revert = []
854 for file in files:
855 file_path = os.path.join(path, file.path)
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000856 print(file_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000857 # Unversioned file or unexpected unversioned file.
858 if file.text_status in ('?', '~'):
859 # Remove extraneous file. Also remove unexpected unversioned
860 # directories. svn won't touch them but we want to delete these.
861 file_list.append(file_path)
862 try:
863 os.remove(file_path)
864 except EnvironmentError:
865 RemoveDirectory(file_path)
866
867 if file.text_status != '?':
868 # For any other status, svn revert will work.
869 file_list.append(file_path)
870 files_to_revert.append(file.path)
871
872 # Revert them all at once.
873 if files_to_revert:
874 accumulated_paths = []
875 accumulated_length = 0
876 command = ['revert']
877 for p in files_to_revert:
878 # Some shell have issues with command lines too long.
879 if accumulated_length and accumulated_length + len(p) > 3072:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000880 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000881 os.path.join(self._root_dir, self.relpath))
882 accumulated_paths = []
883 accumulated_length = 0
884 else:
885 accumulated_paths.append(p)
886 accumulated_length += len(p)
887 if accumulated_paths:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000888 RunSVN(command + accumulated_paths,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000889 os.path.join(self._root_dir, self.relpath))
890
891 def status(self, options, args, file_list):
892 """Display status information."""
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000893 path = os.path.join(self._root_dir, self.relpath)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000894 command = ['status']
895 command.extend(args)
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000896 if not os.path.isdir(path):
897 # svn status won't work if the directory doesn't exist.
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000898 print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
899 "does not exist."
900 % (' '.join(command), path))
maruel@chromium.orgedd27d12009-05-01 17:46:56 +0000901 # There's no file list to retrieve.
902 else:
maruel@chromium.orge77173e2009-05-12 17:03:33 +0000903 RunSVNAndGetFileList(command, path, file_list)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000904
905
906## GClient implementation.
907
908
909class GClient(object):
910 """Object that represent a gclient checkout."""
911
912 supported_commands = [
913 'cleanup', 'diff', 'revert', 'status', 'update', 'runhooks'
914 ]
915
916 def __init__(self, root_dir, options):
917 self._root_dir = root_dir
918 self._options = options
919 self._config_content = None
920 self._config_dict = {}
921 self._deps_hooks = []
922
923 def SetConfig(self, content):
924 self._config_dict = {}
925 self._config_content = content
926 exec(content, self._config_dict)
927
928 def SaveConfig(self):
929 FileWrite(os.path.join(self._root_dir, self._options.config_filename),
930 self._config_content)
931
932 def _LoadConfig(self):
933 client_source = FileRead(os.path.join(self._root_dir,
934 self._options.config_filename))
935 self.SetConfig(client_source)
936
937 def ConfigContent(self):
938 return self._config_content
939
940 def GetVar(self, key, default=None):
941 return self._config_dict.get(key, default)
942
943 @staticmethod
944 def LoadCurrentConfig(options, from_dir=None):
945 """Searches for and loads a .gclient file relative to the current working
946 dir.
947
948 Returns:
949 A dict representing the contents of the .gclient file or an empty dict if
950 the .gclient file doesn't exist.
951 """
952 if not from_dir:
953 from_dir = os.curdir
954 path = os.path.realpath(from_dir)
955 while not options.path_exists(os.path.join(path, options.config_filename)):
956 next = os.path.split(path)
957 if not next[1]:
958 return None
959 path = next[0]
960 client = options.gclient(path, options)
961 client._LoadConfig()
962 return client
963
964 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
965 self.SetConfig(DEFAULT_CLIENT_FILE_TEXT % (
966 solution_name, solution_url, safesync_url
967 ))
968
969 def _SaveEntries(self, entries):
970 """Creates a .gclient_entries file to record the list of unique checkouts.
971
972 The .gclient_entries file lives in the same directory as .gclient.
973
974 Args:
975 entries: A sequence of solution names.
976 """
977 text = "entries = [\n"
978 for entry in entries:
979 text += " \"%s\",\n" % entry
980 text += "]\n"
981 FileWrite(os.path.join(self._root_dir, self._options.entries_filename),
982 text)
983
984 def _ReadEntries(self):
985 """Read the .gclient_entries file for the given client.
986
987 Args:
988 client: The client for which the entries file should be read.
989
990 Returns:
991 A sequence of solution names, which will be empty if there is the
992 entries file hasn't been created yet.
993 """
994 scope = {}
995 filename = os.path.join(self._root_dir, self._options.entries_filename)
996 if not self._options.path_exists(filename):
997 return []
998 exec(FileRead(filename), scope)
999 return scope["entries"]
1000
1001 class FromImpl:
1002 """Used to implement the From syntax."""
1003
1004 def __init__(self, module_name):
1005 self.module_name = module_name
1006
1007 def __str__(self):
1008 return 'From("%s")' % self.module_name
1009
1010 class _VarImpl:
1011 def __init__(self, custom_vars, local_scope):
1012 self._custom_vars = custom_vars
1013 self._local_scope = local_scope
1014
1015 def Lookup(self, var_name):
1016 """Implements the Var syntax."""
1017 if var_name in self._custom_vars:
1018 return self._custom_vars[var_name]
1019 elif var_name in self._local_scope.get("vars", {}):
1020 return self._local_scope["vars"][var_name]
1021 raise Error("Var is not defined: %s" % var_name)
1022
1023 def _ParseSolutionDeps(self, solution_name, solution_deps_content,
1024 custom_vars):
1025 """Parses the DEPS file for the specified solution.
1026
1027 Args:
1028 solution_name: The name of the solution to query.
1029 solution_deps_content: Content of the DEPS file for the solution
1030 custom_vars: A dict of vars to override any vars defined in the DEPS file.
1031
1032 Returns:
1033 A dict mapping module names (as relative paths) to URLs or an empty
1034 dict if the solution does not have a DEPS file.
1035 """
1036 # Skip empty
1037 if not solution_deps_content:
1038 return {}
1039 # Eval the content
1040 local_scope = {}
1041 var = self._VarImpl(custom_vars, local_scope)
1042 global_scope = {"From": self.FromImpl, "Var": var.Lookup, "deps_os": {}}
1043 exec(solution_deps_content, global_scope, local_scope)
1044 deps = local_scope.get("deps", {})
1045
1046 # load os specific dependencies if defined. these dependencies may
1047 # override or extend the values defined by the 'deps' member.
1048 if "deps_os" in local_scope:
1049 deps_os_choices = {
1050 "win32": "win",
1051 "win": "win",
1052 "cygwin": "win",
1053 "darwin": "mac",
1054 "mac": "mac",
1055 "unix": "unix",
1056 "linux": "unix",
1057 "linux2": "unix",
1058 }
1059
1060 if self._options.deps_os is not None:
1061 deps_to_include = self._options.deps_os.split(",")
1062 if "all" in deps_to_include:
1063 deps_to_include = deps_os_choices.values()
1064 else:
1065 deps_to_include = [deps_os_choices.get(self._options.platform, "unix")]
1066
1067 deps_to_include = set(deps_to_include)
1068 for deps_os_key in deps_to_include:
1069 os_deps = local_scope["deps_os"].get(deps_os_key, {})
1070 if len(deps_to_include) > 1:
1071 # Ignore any overrides when including deps for more than one
1072 # platform, so we collect the broadest set of dependencies available.
1073 # We may end up with the wrong revision of something for our
1074 # platform, but this is the best we can do.
1075 deps.update([x for x in os_deps.items() if not x[0] in deps])
1076 else:
1077 deps.update(os_deps)
1078
1079 if 'hooks' in local_scope:
1080 self._deps_hooks.extend(local_scope['hooks'])
1081
1082 # If use_relative_paths is set in the DEPS file, regenerate
1083 # the dictionary using paths relative to the directory containing
1084 # the DEPS file.
1085 if local_scope.get('use_relative_paths'):
1086 rel_deps = {}
1087 for d, url in deps.items():
1088 # normpath is required to allow DEPS to use .. in their
1089 # dependency local path.
1090 rel_deps[os.path.normpath(os.path.join(solution_name, d))] = url
1091 return rel_deps
1092 else:
1093 return deps
1094
1095 def _ParseAllDeps(self, solution_urls, solution_deps_content):
1096 """Parse the complete list of dependencies for the client.
1097
1098 Args:
1099 solution_urls: A dict mapping module names (as relative paths) to URLs
1100 corresponding to the solutions specified by the client. This parameter
1101 is passed as an optimization.
1102 solution_deps_content: A dict mapping module names to the content
1103 of their DEPS files
1104
1105 Returns:
1106 A dict mapping module names (as relative paths) to URLs corresponding
1107 to the entire set of dependencies to checkout for the given client.
1108
1109 Raises:
1110 Error: If a dependency conflicts with another dependency or of a solution.
1111 """
1112 deps = {}
1113 for solution in self.GetVar("solutions"):
1114 custom_vars = solution.get("custom_vars", {})
1115 solution_deps = self._ParseSolutionDeps(
1116 solution["name"],
1117 solution_deps_content[solution["name"]],
1118 custom_vars)
1119
1120 # If a line is in custom_deps, but not in the solution, we want to append
1121 # this line to the solution.
1122 if "custom_deps" in solution:
1123 for d in solution["custom_deps"]:
1124 if d not in solution_deps:
1125 solution_deps[d] = solution["custom_deps"][d]
1126
1127 for d in solution_deps:
1128 if "custom_deps" in solution and d in solution["custom_deps"]:
1129 # Dependency is overriden.
1130 url = solution["custom_deps"][d]
1131 if url is None:
1132 continue
1133 else:
1134 url = solution_deps[d]
1135 # if we have a From reference dependent on another solution, then
1136 # just skip the From reference. When we pull deps for the solution,
1137 # we will take care of this dependency.
1138 #
1139 # If multiple solutions all have the same From reference, then we
1140 # should only add one to our list of dependencies.
1141 if type(url) != str:
1142 if url.module_name in solution_urls:
1143 # Already parsed.
1144 continue
1145 if d in deps and type(deps[d]) != str:
1146 if url.module_name == deps[d].module_name:
1147 continue
1148 else:
1149 parsed_url = urlparse.urlparse(url)
1150 scheme = parsed_url[0]
1151 if not scheme:
1152 # A relative url. Fetch the real base.
1153 path = parsed_url[2]
1154 if path[0] != "/":
1155 raise Error(
1156 "relative DEPS entry \"%s\" must begin with a slash" % d)
1157 # Create a scm just to query the full url.
1158 scm = self._options.scm_wrapper(solution["url"], self._root_dir,
1159 None)
1160 url = scm.FullUrlForRelativeUrl(url)
1161 if d in deps and deps[d] != url:
1162 raise Error(
1163 "Solutions have conflicting versions of dependency \"%s\"" % d)
1164 if d in solution_urls and solution_urls[d] != url:
1165 raise Error(
1166 "Dependency \"%s\" conflicts with specified solution" % d)
1167 # Grab the dependency.
1168 deps[d] = url
1169 return deps
1170
1171 def _RunHookAction(self, hook_dict):
1172 """Runs the action from a single hook.
1173 """
1174 command = hook_dict['action'][:]
1175 if command[0] == 'python':
1176 # If the hook specified "python" as the first item, the action is a
1177 # Python script. Run it by starting a new copy of the same
1178 # interpreter.
1179 command[0] = sys.executable
1180
1181 # Use a discrete exit status code of 2 to indicate that a hook action
1182 # failed. Users of this script may wish to treat hook action failures
1183 # differently from VC failures.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001184 SubprocessCall(command, self._root_dir, fail_status=2)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001185
1186 def _RunHooks(self, command, file_list, is_using_git):
1187 """Evaluates all hooks, running actions as needed.
1188 """
1189 # Hooks only run for these command types.
1190 if not command in ('update', 'revert', 'runhooks'):
1191 return
1192
1193 # Get any hooks from the .gclient file.
1194 hooks = self.GetVar("hooks", [])
1195 # Add any hooks found in DEPS files.
1196 hooks.extend(self._deps_hooks)
1197
1198 # If "--force" was specified, run all hooks regardless of what files have
1199 # changed. If the user is using git, then we don't know what files have
1200 # changed so we always run all hooks.
1201 if self._options.force or is_using_git:
1202 for hook_dict in hooks:
1203 self._RunHookAction(hook_dict)
1204 return
1205
1206 # Run hooks on the basis of whether the files from the gclient operation
1207 # match each hook's pattern.
1208 for hook_dict in hooks:
1209 pattern = re.compile(hook_dict['pattern'])
1210 for file in file_list:
1211 if not pattern.search(file):
1212 continue
1213
1214 self._RunHookAction(hook_dict)
1215
1216 # The hook's action only runs once. Don't bother looking for any
1217 # more matches.
1218 break
1219
1220 def RunOnDeps(self, command, args):
1221 """Runs a command on each dependency in a client and its dependencies.
1222
1223 The module's dependencies are specified in its top-level DEPS files.
1224
1225 Args:
1226 command: The command to use (e.g., 'status' or 'diff')
1227 args: list of str - extra arguments to add to the command line.
1228
1229 Raises:
1230 Error: If the client has conflicting entries.
1231 """
1232 if not command in self.supported_commands:
1233 raise Error("'%s' is an unsupported command" % command)
1234
1235 # Check for revision overrides.
1236 revision_overrides = {}
1237 for revision in self._options.revisions:
1238 if revision.find("@") == -1:
1239 raise Error(
1240 "Specify the full dependency when specifying a revision number.")
1241 revision_elem = revision.split("@")
1242 # Disallow conflicting revs
1243 if revision_overrides.has_key(revision_elem[0]) and \
1244 revision_overrides[revision_elem[0]] != revision_elem[1]:
1245 raise Error(
1246 "Conflicting revision numbers specified.")
1247 revision_overrides[revision_elem[0]] = revision_elem[1]
1248
1249 solutions = self.GetVar("solutions")
1250 if not solutions:
1251 raise Error("No solution specified")
1252
1253 # When running runhooks --force, there's no need to consult the SCM.
1254 # All known hooks are expected to run unconditionally regardless of working
1255 # copy state, so skip the SCM status check.
1256 run_scm = not (command == 'runhooks' and self._options.force)
1257
1258 entries = {}
1259 entries_deps_content = {}
1260 file_list = []
1261 # Run on the base solutions first.
1262 for solution in solutions:
1263 name = solution["name"]
1264 if name in entries:
1265 raise Error("solution %s specified more than once" % name)
1266 url = solution["url"]
1267 entries[name] = url
1268 if run_scm:
1269 self._options.revision = revision_overrides.get(name)
1270 scm = self._options.scm_wrapper(url, self._root_dir, name)
1271 scm.RunCommand(command, self._options, args, file_list)
1272 self._options.revision = None
1273 try:
1274 deps_content = FileRead(os.path.join(self._root_dir, name,
1275 self._options.deps_file))
1276 except IOError, e:
1277 if e.errno != errno.ENOENT:
1278 raise
1279 deps_content = ""
1280 entries_deps_content[name] = deps_content
1281
1282 # Process the dependencies next (sort alphanumerically to ensure that
1283 # containing directories get populated first and for readability)
1284 deps = self._ParseAllDeps(entries, entries_deps_content)
1285 deps_to_process = deps.keys()
1286 deps_to_process.sort()
1287
1288 # First pass for direct dependencies.
1289 for d in deps_to_process:
1290 if type(deps[d]) == str:
1291 url = deps[d]
1292 entries[d] = url
1293 if run_scm:
1294 self._options.revision = revision_overrides.get(d)
1295 scm = self._options.scm_wrapper(url, self._root_dir, d)
1296 scm.RunCommand(command, self._options, args, file_list)
1297 self._options.revision = None
1298
1299 # Second pass for inherited deps (via the From keyword)
1300 for d in deps_to_process:
1301 if type(deps[d]) != str:
1302 sub_deps = self._ParseSolutionDeps(
1303 deps[d].module_name,
1304 FileRead(os.path.join(self._root_dir,
1305 deps[d].module_name,
1306 self._options.deps_file)),
1307 {})
1308 url = sub_deps[d]
1309 entries[d] = url
1310 if run_scm:
1311 self._options.revision = revision_overrides.get(d)
1312 scm = self._options.scm_wrapper(url, self._root_dir, d)
1313 scm.RunCommand(command, self._options, args, file_list)
1314 self._options.revision = None
1315
1316 is_using_git = IsUsingGit(self._root_dir, entries.keys())
1317 self._RunHooks(command, file_list, is_using_git)
1318
1319 if command == 'update':
1320 # notify the user if there is an orphaned entry in their working copy.
1321 # TODO(darin): we should delete this directory manually if it doesn't
1322 # have any changes in it.
1323 prev_entries = self._ReadEntries()
1324 for entry in prev_entries:
1325 e_dir = os.path.join(self._root_dir, entry)
1326 if entry not in entries and self._options.path_exists(e_dir):
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001327 if CaptureSVNStatus(e_dir):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001328 # There are modified files in this entry
1329 entries[entry] = None # Keep warning until removed.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001330 print("\nWARNING: \"%s\" is no longer part of this client. "
1331 "It is recommended that you manually remove it.\n") % entry
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001332 else:
1333 # Delete the entry
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001334 print("\n________ deleting \'%s\' " +
1335 "in \'%s\'") % (entry, self._root_dir)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001336 RemoveDirectory(e_dir)
1337 # record the current list of entries for next time
1338 self._SaveEntries(entries)
1339
1340 def PrintRevInfo(self):
1341 """Output revision info mapping for the client and its dependencies. This
1342 allows the capture of a overall "revision" for the source tree that can
1343 be used to reproduce the same tree in the future. The actual output
1344 contains enough information (source paths, svn server urls and revisions)
1345 that it can be used either to generate external svn commands (without
1346 gclient) or as input to gclient's --rev option (with some massaging of
1347 the data).
1348
1349 NOTE: Unlike RunOnDeps this does not require a local checkout and is run
1350 on the Pulse master. It MUST NOT execute hooks.
1351
1352 Raises:
1353 Error: If the client has conflicting entries.
1354 """
1355 # Check for revision overrides.
1356 revision_overrides = {}
1357 for revision in self._options.revisions:
1358 if revision.find("@") < 0:
1359 raise Error(
1360 "Specify the full dependency when specifying a revision number.")
1361 revision_elem = revision.split("@")
1362 # Disallow conflicting revs
1363 if revision_overrides.has_key(revision_elem[0]) and \
1364 revision_overrides[revision_elem[0]] != revision_elem[1]:
1365 raise Error(
1366 "Conflicting revision numbers specified.")
1367 revision_overrides[revision_elem[0]] = revision_elem[1]
1368
1369 solutions = self.GetVar("solutions")
1370 if not solutions:
1371 raise Error("No solution specified")
1372
1373 entries = {}
1374 entries_deps_content = {}
1375
1376 # Inner helper to generate base url and rev tuple (including honoring
1377 # |revision_overrides|)
1378 def GetURLAndRev(name, original_url):
1379 if original_url.find("@") < 0:
1380 if revision_overrides.has_key(name):
1381 return (original_url, int(revision_overrides[name]))
1382 else:
1383 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001384 return (original_url, CaptureSVNHeadRevision(original_url))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001385 else:
1386 url_components = original_url.split("@")
1387 if revision_overrides.has_key(name):
1388 return (url_components[0], int(revision_overrides[name]))
1389 else:
1390 return (url_components[0], int(url_components[1]))
1391
1392 # Run on the base solutions first.
1393 for solution in solutions:
1394 name = solution["name"]
1395 if name in entries:
1396 raise Error("solution %s specified more than once" % name)
1397 (url, rev) = GetURLAndRev(name, solution["url"])
1398 entries[name] = "%s@%d" % (url, rev)
1399 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1400 entries_deps_content[name] = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001401 ["cat",
1402 "%s/%s@%d" % (url,
1403 self._options.deps_file,
1404 rev)],
1405 os.getcwd())
1406
1407 # Process the dependencies next (sort alphanumerically to ensure that
1408 # containing directories get populated first and for readability)
1409 deps = self._ParseAllDeps(entries, entries_deps_content)
1410 deps_to_process = deps.keys()
1411 deps_to_process.sort()
1412
1413 # First pass for direct dependencies.
1414 for d in deps_to_process:
1415 if type(deps[d]) == str:
1416 (url, rev) = GetURLAndRev(d, deps[d])
1417 entries[d] = "%s@%d" % (url, rev)
1418
1419 # Second pass for inherited deps (via the From keyword)
1420 for d in deps_to_process:
1421 if type(deps[d]) != str:
1422 deps_parent_url = entries[deps[d].module_name]
1423 if deps_parent_url.find("@") < 0:
1424 raise Error("From %s missing revisioned url" % deps[d].module_name)
1425 deps_parent_url_components = deps_parent_url.split("@")
1426 # TODO(aharper): SVN/SCMWrapper cleanup (non-local commandset)
1427 deps_parent_content = CaptureSVN(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001428 ["cat",
1429 "%s/%s@%s" % (deps_parent_url_components[0],
1430 self._options.deps_file,
1431 deps_parent_url_components[1])],
1432 os.getcwd())
1433 sub_deps = self._ParseSolutionDeps(
1434 deps[d].module_name,
1435 FileRead(os.path.join(self._root_dir,
1436 deps[d].module_name,
1437 self._options.deps_file)),
1438 {})
1439 (url, rev) = GetURLAndRev(d, sub_deps[d])
1440 entries[d] = "%s@%d" % (url, rev)
1441
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001442 print(";".join(["%s,%s" % (x, entries[x]) for x in sorted(entries.keys())]))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001443
1444
1445## gclient commands.
1446
1447
1448def DoCleanup(options, args):
1449 """Handle the cleanup subcommand.
1450
1451 Raises:
1452 Error: if client isn't configured properly.
1453 """
1454 client = options.gclient.LoadCurrentConfig(options)
1455 if not client:
1456 raise Error("client not configured; see 'gclient config'")
1457 if options.verbose:
1458 # Print out the .gclient file. This is longer than if we just printed the
1459 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001460 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001461 options.verbose = True
1462 return client.RunOnDeps('cleanup', args)
1463
1464
1465def DoConfig(options, args):
1466 """Handle the config subcommand.
1467
1468 Args:
1469 options: If options.spec set, a string providing contents of config file.
1470 args: The command line args. If spec is not set,
1471 then args[0] is a string URL to get for config file.
1472
1473 Raises:
1474 Error: on usage error
1475 """
1476 if len(args) < 1 and not options.spec:
1477 raise Error("required argument missing; see 'gclient help config'")
1478 if options.path_exists(options.config_filename):
1479 raise Error("%s file already exists in the current directory" %
1480 options.config_filename)
1481 client = options.gclient('.', options)
1482 if options.spec:
1483 client.SetConfig(options.spec)
1484 else:
1485 # TODO(darin): it would be nice to be able to specify an alternate relpath
1486 # for the given URL.
1487 base_url = args[0]
1488 name = args[0].split("/")[-1]
1489 safesync_url = ""
1490 if len(args) > 1:
1491 safesync_url = args[1]
1492 client.SetDefaultConfig(name, base_url, safesync_url)
1493 client.SaveConfig()
1494
1495
1496def DoHelp(options, args):
1497 """Handle the help subcommand giving help for another subcommand.
1498
1499 Raises:
1500 Error: if the command is unknown.
1501 """
1502 if len(args) == 1 and args[0] in COMMAND_USAGE_TEXT:
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001503 print(COMMAND_USAGE_TEXT[args[0]])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001504 else:
1505 raise Error("unknown subcommand '%s'; see 'gclient help'" % args[0])
1506
1507
1508def DoStatus(options, args):
1509 """Handle the status subcommand.
1510
1511 Raises:
1512 Error: if client isn't configured properly.
1513 """
1514 client = options.gclient.LoadCurrentConfig(options)
1515 if not client:
1516 raise Error("client not configured; see 'gclient config'")
1517 if options.verbose:
1518 # Print out the .gclient file. This is longer than if we just printed the
1519 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001520 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001521 options.verbose = True
1522 return client.RunOnDeps('status', args)
1523
1524
1525def DoUpdate(options, args):
1526 """Handle the update and sync subcommands.
1527
1528 Raises:
1529 Error: if client isn't configured properly.
1530 """
1531 client = options.gclient.LoadCurrentConfig(options)
1532
1533 if not client:
1534 raise Error("client not configured; see 'gclient config'")
1535
1536 if not options.head:
1537 solutions = client.GetVar('solutions')
1538 if solutions:
1539 for s in solutions:
1540 if s.get('safesync_url', ''):
1541 # rip through revisions and make sure we're not over-riding
1542 # something that was explicitly passed
1543 has_key = False
1544 for r in options.revisions:
1545 if r.split('@')[0] == s['name']:
1546 has_key = True
1547 break
1548
1549 if not has_key:
1550 handle = urllib.urlopen(s['safesync_url'])
1551 rev = handle.read().strip()
1552 handle.close()
1553 if len(rev):
1554 options.revisions.append(s['name']+'@'+rev)
1555
1556 if options.verbose:
1557 # Print out the .gclient file. This is longer than if we just printed the
1558 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001559 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001560 return client.RunOnDeps('update', args)
1561
1562
1563def DoDiff(options, args):
1564 """Handle the diff subcommand.
1565
1566 Raises:
1567 Error: if client isn't configured properly.
1568 """
1569 client = options.gclient.LoadCurrentConfig(options)
1570 if not client:
1571 raise Error("client not configured; see 'gclient config'")
1572 if options.verbose:
1573 # Print out the .gclient file. This is longer than if we just printed the
1574 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001575 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001576 options.verbose = True
1577 return client.RunOnDeps('diff', args)
1578
1579
1580def DoRevert(options, args):
1581 """Handle the revert subcommand.
1582
1583 Raises:
1584 Error: if client isn't configured properly.
1585 """
1586 client = options.gclient.LoadCurrentConfig(options)
1587 if not client:
1588 raise Error("client not configured; see 'gclient config'")
1589 return client.RunOnDeps('revert', args)
1590
1591
1592def DoRunHooks(options, args):
1593 """Handle the runhooks subcommand.
1594
1595 Raises:
1596 Error: if client isn't configured properly.
1597 """
1598 client = options.gclient.LoadCurrentConfig(options)
1599 if not client:
1600 raise Error("client not configured; see 'gclient config'")
1601 if options.verbose:
1602 # Print out the .gclient file. This is longer than if we just printed the
1603 # client dict, but more legible, and it might contain helpful comments.
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001604 print(client.ConfigContent())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001605 return client.RunOnDeps('runhooks', args)
1606
1607
1608def DoRevInfo(options, args):
1609 """Handle the revinfo subcommand.
1610
1611 Raises:
1612 Error: if client isn't configured properly.
1613 """
1614 client = options.gclient.LoadCurrentConfig(options)
1615 if not client:
1616 raise Error("client not configured; see 'gclient config'")
1617 client.PrintRevInfo()
1618
1619
1620gclient_command_map = {
1621 "cleanup": DoCleanup,
1622 "config": DoConfig,
1623 "diff": DoDiff,
1624 "help": DoHelp,
1625 "status": DoStatus,
1626 "sync": DoUpdate,
1627 "update": DoUpdate,
1628 "revert": DoRevert,
1629 "runhooks": DoRunHooks,
1630 "revinfo" : DoRevInfo,
1631}
1632
1633
1634def DispatchCommand(command, options, args, command_map=None):
1635 """Dispatches the appropriate subcommand based on command line arguments."""
1636 if command_map is None:
1637 command_map = gclient_command_map
1638
1639 if command in command_map:
1640 return command_map[command](options, args)
1641 else:
1642 raise Error("unknown subcommand '%s'; see 'gclient help'" % command)
1643
1644
1645def Main(argv):
1646 """Parse command line arguments and dispatch command."""
1647
1648 option_parser = optparse.OptionParser(usage=DEFAULT_USAGE_TEXT,
1649 version=__version__)
1650 option_parser.disable_interspersed_args()
1651 option_parser.add_option("", "--force", action="store_true", default=False,
1652 help=("(update/sync only) force update even "
1653 "for modules which haven't changed"))
1654 option_parser.add_option("", "--revision", action="append", dest="revisions",
1655 metavar="REV", default=[],
1656 help=("(update/sync only) sync to a specific "
1657 "revision, can be used multiple times for "
1658 "each solution, e.g. --revision=src@123, "
1659 "--revision=internal@32"))
1660 option_parser.add_option("", "--deps", default=None, dest="deps_os",
1661 metavar="OS_LIST",
1662 help=("(update/sync only) sync deps for the "
1663 "specified (comma-separated) platform(s); "
1664 "'all' will sync all platforms"))
1665 option_parser.add_option("", "--spec", default=None,
1666 help=("(config only) create a gclient file "
1667 "containing the provided string"))
1668 option_parser.add_option("", "--verbose", action="store_true", default=False,
1669 help="produce additional output for diagnostics")
1670 option_parser.add_option("", "--manually_grab_svn_rev", action="store_true",
1671 default=False,
1672 help="Skip svn up whenever possible by requesting "
1673 "actual HEAD revision from the repository")
1674 option_parser.add_option("", "--head", action="store_true", default=False,
1675 help=("skips any safesync_urls specified in "
1676 "configured solutions"))
1677
1678 if len(argv) < 2:
1679 # Users don't need to be told to use the 'help' command.
1680 option_parser.print_help()
1681 return 1
1682 # Add manual support for --version as first argument.
1683 if argv[1] == '--version':
1684 option_parser.print_version()
1685 return 0
1686
1687 # Add manual support for --help as first argument.
1688 if argv[1] == '--help':
1689 argv[1] = 'help'
1690
1691 command = argv[1]
1692 options, args = option_parser.parse_args(argv[2:])
1693
1694 if len(argv) < 3 and command == "help":
1695 option_parser.print_help()
1696 return 0
1697
1698 # Files used for configuration and state saving.
1699 options.config_filename = os.environ.get("GCLIENT_FILE", ".gclient")
1700 options.entries_filename = ".gclient_entries"
1701 options.deps_file = "DEPS"
1702
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001703 options.path_exists = os.path.exists
1704 options.gclient = GClient
1705 options.scm_wrapper = SCMWrapper
1706 options.platform = sys.platform
1707 return DispatchCommand(command, options, args)
1708
1709
1710if "__main__" == __name__:
1711 try:
1712 result = Main(sys.argv)
1713 except Error, e:
maruel@chromium.orge77173e2009-05-12 17:03:33 +00001714 print >> sys.stderr, "Error: %s" % str(e)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001715 result = 1
1716 sys.exit(result)
1717
1718# vim: ts=2:sw=2:tw=80:et: