blob: 99adc1f1a9d891a133ec5dd2ec7f37aa9a05d166 [file] [log] [blame]
maruel@chromium.org5f3eee32009-09-17 00:34:30 +00001# Copyright 2009 Google Inc. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15
maruel@chromium.org754960e2009-09-21 12:31:05 +000016import logging
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000017import os
18import re
19import subprocess
20import sys
21import xml.dom.minidom
22
23import gclient_utils
24
25SVN_COMMAND = "svn"
26
27
28### SCM abstraction layer
29
30
31class SCMWrapper(object):
32 """Add necessary glue between all the supported SCM.
33
34 This is the abstraction layer to bind to different SCM. Since currently only
35 subversion is supported, a lot of subersionism remains. This can be sorted out
36 once another SCM is supported."""
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000037 def __init__(self, url=None, root_dir=None, relpath=None,
38 scm_name='svn'):
39 # TODO(maruel): Deduce the SCM from the url.
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000040 self.scm_name = scm_name
41 self.url = url
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000042 self._root_dir = root_dir
43 if self._root_dir:
44 self._root_dir = self._root_dir.replace('/', os.sep)
45 self.relpath = relpath
46 if self.relpath:
47 self.relpath = self.relpath.replace('/', os.sep)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000048
49 def FullUrlForRelativeUrl(self, url):
50 # Find the forth '/' and strip from there. A bit hackish.
51 return '/'.join(self.url.split('/')[:4]) + url
52
53 def RunCommand(self, command, options, args, file_list=None):
54 # file_list will have all files that are modified appended to it.
maruel@chromium.orgde754ac2009-09-17 18:04:50 +000055 if file_list is None:
56 file_list = []
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000057
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000058 commands = {
59 'cleanup': self.cleanup,
60 'export': self.export,
61 'update': self.update,
62 'revert': self.revert,
63 'status': self.status,
64 'diff': self.diff,
65 'pack': self.pack,
66 'runhooks': self.status,
67 }
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000068
69 if not command in commands:
70 raise gclient_utils.Error('Unknown command %s' % command)
71
maruel@chromium.org5e73b0c2009-09-18 19:47:48 +000072 return commands[command](options, args, file_list)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +000073
74 def cleanup(self, options, args, file_list):
75 """Cleanup working copy."""
76 command = ['cleanup']
77 command.extend(args)
78 RunSVN(command, os.path.join(self._root_dir, self.relpath))
79
80 def diff(self, options, args, file_list):
81 # NOTE: This function does not currently modify file_list.
82 command = ['diff']
83 command.extend(args)
84 RunSVN(command, os.path.join(self._root_dir, self.relpath))
85
86 def export(self, options, args, file_list):
87 assert len(args) == 1
88 export_path = os.path.abspath(os.path.join(args[0], self.relpath))
89 try:
90 os.makedirs(export_path)
91 except OSError:
92 pass
93 assert os.path.exists(export_path)
94 command = ['export', '--force', '.']
95 command.append(export_path)
96 RunSVN(command, os.path.join(self._root_dir, self.relpath))
97
98 def update(self, options, args, file_list):
99 """Runs SCM to update or transparently checkout the working copy.
100
101 All updated files will be appended to file_list.
102
103 Raises:
104 Error: if can't get URL for relative path.
105 """
106 # Only update if git is not controlling the directory.
107 checkout_path = os.path.join(self._root_dir, self.relpath)
108 git_path = os.path.join(self._root_dir, self.relpath, '.git')
109 if os.path.exists(git_path):
110 print("________ found .git directory; skipping %s" % self.relpath)
111 return
112
113 if args:
114 raise gclient_utils.Error("Unsupported argument(s): %s" % ",".join(args))
115
116 url = self.url
117 components = url.split("@")
118 revision = None
119 forced_revision = False
120 if options.revision:
121 # Override the revision number.
122 url = '%s@%s' % (components[0], str(options.revision))
123 revision = int(options.revision)
124 forced_revision = True
125 elif len(components) == 2:
126 revision = int(components[1])
127 forced_revision = True
128
129 rev_str = ""
130 if revision:
131 rev_str = ' at %d' % revision
132
133 if not os.path.exists(checkout_path):
134 # We need to checkout.
135 command = ['checkout', url, checkout_path]
136 if revision:
137 command.extend(['--revision', str(revision)])
138 RunSVNAndGetFileList(command, self._root_dir, file_list)
139 return
140
141 # Get the existing scm url and the revision number of the current checkout.
142 from_info = CaptureSVNInfo(os.path.join(checkout_path, '.'), '.')
143 if not from_info:
144 raise gclient_utils.Error("Can't update/checkout %r if an unversioned "
145 "directory is present. Delete the directory "
146 "and try again." %
147 checkout_path)
148
149 if options.manually_grab_svn_rev:
150 # Retrieve the current HEAD version because svn is slow at null updates.
151 if not revision:
152 from_info_live = CaptureSVNInfo(from_info['URL'], '.')
153 revision = int(from_info_live['Revision'])
154 rev_str = ' at %d' % revision
155
156 if from_info['URL'] != components[0]:
157 to_info = CaptureSVNInfo(url, '.')
158 can_switch = ((from_info['Repository Root'] != to_info['Repository Root'])
159 and (from_info['UUID'] == to_info['UUID']))
160 if can_switch:
161 print("\n_____ relocating %s to a new checkout" % self.relpath)
162 # We have different roots, so check if we can switch --relocate.
163 # Subversion only permits this if the repository UUIDs match.
164 # Perform the switch --relocate, then rewrite the from_url
165 # to reflect where we "are now." (This is the same way that
166 # Subversion itself handles the metadata when switch --relocate
167 # is used.) This makes the checks below for whether we
168 # can update to a revision or have to switch to a different
169 # branch work as expected.
170 # TODO(maruel): TEST ME !
171 command = ["switch", "--relocate",
172 from_info['Repository Root'],
173 to_info['Repository Root'],
174 self.relpath]
175 RunSVN(command, self._root_dir)
176 from_info['URL'] = from_info['URL'].replace(
177 from_info['Repository Root'],
178 to_info['Repository Root'])
179 else:
180 if CaptureSVNStatus(checkout_path):
181 raise gclient_utils.Error("Can't switch the checkout to %s; UUID "
182 "don't match and there is local changes "
183 "in %s. Delete the directory and "
184 "try again." % (url, checkout_path))
185 # Ok delete it.
186 print("\n_____ switching %s to a new checkout" % self.relpath)
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +0000187 gclient_utils.RemoveDirectory(checkout_path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000188 # We need to checkout.
189 command = ['checkout', url, checkout_path]
190 if revision:
191 command.extend(['--revision', str(revision)])
192 RunSVNAndGetFileList(command, self._root_dir, file_list)
193 return
194
195
196 # If the provided url has a revision number that matches the revision
197 # number of the existing directory, then we don't need to bother updating.
198 if not options.force and from_info['Revision'] == revision:
199 if options.verbose or not forced_revision:
200 print("\n_____ %s%s" % (self.relpath, rev_str))
201 return
202
203 command = ["update", checkout_path]
204 if revision:
205 command.extend(['--revision', str(revision)])
206 RunSVNAndGetFileList(command, self._root_dir, file_list)
207
208 def revert(self, options, args, file_list):
209 """Reverts local modifications. Subversion specific.
210
211 All reverted files will be appended to file_list, even if Subversion
212 doesn't know about them.
213 """
214 path = os.path.join(self._root_dir, self.relpath)
215 if not os.path.isdir(path):
216 # svn revert won't work if the directory doesn't exist. It needs to
217 # checkout instead.
218 print("\n_____ %s is missing, synching instead" % self.relpath)
219 # Don't reuse the args.
220 return self.update(options, [], file_list)
221
maruel@chromium.org754960e2009-09-21 12:31:05 +0000222 for file in CaptureSVNStatus(path):
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000223 file_path = os.path.join(path, file[1])
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000224 if file[0][0] == 'X':
maruel@chromium.org754960e2009-09-21 12:31:05 +0000225 # Ignore externals.
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000226 logging.info('Ignoring external %s' % file_path)
maruel@chromium.org754960e2009-09-21 12:31:05 +0000227 continue
228
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000229 if logging.getLogger().isEnabledFor(logging.INFO):
230 logging.info('%s%s' % (file[0], file[1]))
231 else:
232 print(file_path)
233 if file[0].isspace():
234 logging.error('No idea what is the status of %s.\n'
235 'You just found a bug in gclient, please ping '
236 'maruel@chromium.org ASAP!' % file_path)
237 # svn revert is really stupid. It fails on inconsistent line-endings,
238 # on switched directories, etc. So take no chance and delete everything!
239 try:
240 if not os.path.exists(file_path):
241 pass
242 elif os.path.isfile(file_path):
maruel@chromium.org754960e2009-09-21 12:31:05 +0000243 logging.info('os.remove(%s)' % file_path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000244 os.remove(file_path)
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000245 elif os.path.isdir(file_path):
maruel@chromium.org754960e2009-09-21 12:31:05 +0000246 logging.info('gclient_utils.RemoveDirectory(%s)' % file_path)
bradnelson@google.com8f9c69f2009-09-17 00:48:28 +0000247 gclient_utils.RemoveDirectory(file_path)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000248 else:
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000249 logging.error('no idea what is %s.\nYou just found a bug in gclient'
250 ', please ping maruel@chromium.org ASAP!' % file_path)
251 except EnvironmentError:
252 logging.error('Failed to remove %s.' % file_path)
253
254 # svn revert is so broken we don't even use it. Using
255 # "svn up --revision BASE" achieve the same effect.
256 RunSVNAndGetFileList(['update', '--revision', 'BASE'], path, file_list)
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000257
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000258 def status(self, options, args, file_list):
259 """Display status information."""
260 path = os.path.join(self._root_dir, self.relpath)
261 command = ['status']
262 command.extend(args)
263 if not os.path.isdir(path):
264 # svn status won't work if the directory doesn't exist.
265 print("\n________ couldn't run \'%s\' in \'%s\':\nThe directory "
266 "does not exist."
267 % (' '.join(command), path))
268 # There's no file list to retrieve.
269 else:
270 RunSVNAndGetFileList(command, path, file_list)
271
272 def pack(self, options, args, file_list):
273 """Generates a patch file which can be applied to the root of the
274 repository."""
275 path = os.path.join(self._root_dir, self.relpath)
276 command = ['diff']
277 command.extend(args)
278 # Simple class which tracks which file is being diffed and
279 # replaces instances of its file name in the original and
280 # working copy lines of the svn diff output.
281 class DiffFilterer(object):
282 index_string = "Index: "
283 original_prefix = "--- "
284 working_prefix = "+++ "
285
286 def __init__(self, relpath):
287 # Note that we always use '/' as the path separator to be
288 # consistent with svn's cygwin-style output on Windows
289 self._relpath = relpath.replace("\\", "/")
290 self._current_file = ""
291 self._replacement_file = ""
292
293 def SetCurrentFile(self, file):
294 self._current_file = file
295 # Note that we always use '/' as the path separator to be
296 # consistent with svn's cygwin-style output on Windows
297 self._replacement_file = self._relpath + '/' + file
298
299 def ReplaceAndPrint(self, line):
300 print(line.replace(self._current_file, self._replacement_file))
301
302 def Filter(self, line):
303 if (line.startswith(self.index_string)):
304 self.SetCurrentFile(line[len(self.index_string):])
305 self.ReplaceAndPrint(line)
306 else:
307 if (line.startswith(self.original_prefix) or
308 line.startswith(self.working_prefix)):
309 self.ReplaceAndPrint(line)
310 else:
311 print line
312
313 filterer = DiffFilterer(self.relpath)
314 RunSVNAndFilterOutput(command, path, False, False, filterer.Filter)
315
316
317# -----------------------------------------------------------------------------
318# SVN utils:
319
320
321def RunSVN(args, in_directory):
322 """Runs svn, sending output to stdout.
323
324 Args:
325 args: A sequence of command line parameters to be passed to svn.
326 in_directory: The directory where svn is to be run.
327
328 Raises:
329 Error: An error occurred while running the svn command.
330 """
331 c = [SVN_COMMAND]
332 c.extend(args)
333
334 gclient_utils.SubprocessCall(c, in_directory)
335
336
337def CaptureSVN(args, in_directory=None, print_error=True):
338 """Runs svn, capturing output sent to stdout as a string.
339
340 Args:
341 args: A sequence of command line parameters to be passed to svn.
342 in_directory: The directory where svn is to be run.
343
344 Returns:
345 The output sent to stdout as a string.
346 """
347 c = [SVN_COMMAND]
348 c.extend(args)
349
350 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
351 # the svn.exe executable, but shell=True makes subprocess on Linux fail
352 # when it's called with a list because it only tries to execute the
353 # first string ("svn").
354 stderr = None
355 if not print_error:
356 stderr = subprocess.PIPE
357 return subprocess.Popen(c,
358 cwd=in_directory,
359 shell=(sys.platform == 'win32'),
360 stdout=subprocess.PIPE,
361 stderr=stderr).communicate()[0]
362
363
364def RunSVNAndGetFileList(args, in_directory, file_list):
365 """Runs svn checkout, update, or status, output to stdout.
366
367 The first item in args must be either "checkout", "update", or "status".
368
369 svn's stdout is parsed to collect a list of files checked out or updated.
370 These files are appended to file_list. svn's stdout is also printed to
371 sys.stdout as in RunSVN.
372
373 Args:
374 args: A sequence of command line parameters to be passed to svn.
375 in_directory: The directory where svn is to be run.
376
377 Raises:
378 Error: An error occurred while running the svn command.
379 """
380 command = [SVN_COMMAND]
381 command.extend(args)
382
383 # svn update and svn checkout use the same pattern: the first three columns
384 # are for file status, property status, and lock status. This is followed
385 # by two spaces, and then the path to the file.
386 update_pattern = '^... (.*)$'
387
388 # The first three columns of svn status are the same as for svn update and
389 # svn checkout. The next three columns indicate addition-with-history,
390 # switch, and remote lock status. This is followed by one space, and then
391 # the path to the file.
392 status_pattern = '^...... (.*)$'
393
394 # args[0] must be a supported command. This will blow up if it's something
395 # else, which is good. Note that the patterns are only effective when
396 # these commands are used in their ordinary forms, the patterns are invalid
397 # for "svn status --show-updates", for example.
398 pattern = {
399 'checkout': update_pattern,
400 'status': status_pattern,
401 'update': update_pattern,
402 }[args[0]]
403
404 compiled_pattern = re.compile(pattern)
405
406 def CaptureMatchingLines(line):
407 match = compiled_pattern.search(line)
408 if match:
409 file_list.append(match.group(1))
410
411 RunSVNAndFilterOutput(args,
412 in_directory,
413 True,
414 True,
415 CaptureMatchingLines)
416
417def RunSVNAndFilterOutput(args,
418 in_directory,
419 print_messages,
420 print_stdout,
421 filter):
422 """Runs svn checkout, update, status, or diff, optionally outputting
423 to stdout.
424
425 The first item in args must be either "checkout", "update",
426 "status", or "diff".
427
428 svn's stdout is passed line-by-line to the given filter function. If
429 print_stdout is true, it is also printed to sys.stdout as in RunSVN.
430
431 Args:
432 args: A sequence of command line parameters to be passed to svn.
433 in_directory: The directory where svn is to be run.
434 print_messages: Whether to print status messages to stdout about
435 which Subversion commands are being run.
436 print_stdout: Whether to forward Subversion's output to stdout.
437 filter: A function taking one argument (a string) which will be
438 passed each line (with the ending newline character removed) of
439 Subversion's output for filtering.
440
441 Raises:
442 Error: An error occurred while running the svn command.
443 """
444 command = [SVN_COMMAND]
445 command.extend(args)
446
447 gclient_utils.SubprocessCallAndFilter(command,
448 in_directory,
449 print_messages,
450 print_stdout,
451 filter=filter)
452
453def CaptureSVNInfo(relpath, in_directory=None, print_error=True):
454 """Returns a dictionary from the svn info output for the given file.
455
456 Args:
457 relpath: The directory where the working copy resides relative to
458 the directory given by in_directory.
459 in_directory: The directory where svn is to be run.
460 """
461 output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error)
462 dom = gclient_utils.ParseXML(output)
463 result = {}
464 if dom:
465 GetNamedNodeText = gclient_utils.GetNamedNodeText
466 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
467 def C(item, f):
468 if item is not None: return f(item)
469 # /info/entry/
470 # url
471 # reposityory/(root|uuid)
472 # wc-info/(schedule|depth)
473 # commit/(author|date)
474 # str() the results because they may be returned as Unicode, which
475 # interferes with the higher layers matching up things in the deps
476 # dictionary.
477 # TODO(maruel): Fix at higher level instead (!)
478 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
479 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
480 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
481 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
482 int)
483 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
484 str)
485 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
486 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
487 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
488 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
489 return result
490
491
492def CaptureSVNHeadRevision(url):
493 """Get the head revision of a SVN repository.
494
495 Returns:
496 Int head revision
497 """
498 info = CaptureSVN(["info", "--xml", url], os.getcwd())
499 dom = xml.dom.minidom.parseString(info)
500 return int(dom.getElementsByTagName('entry')[0].getAttribute('revision'))
501
502
503def CaptureSVNStatus(files):
504 """Returns the svn 1.5 svn status emulated output.
505
506 @files can be a string (one file) or a list of files.
507
508 Returns an array of (status, file) tuples."""
509 command = ["status", "--xml"]
510 if not files:
511 pass
512 elif isinstance(files, basestring):
513 command.append(files)
514 else:
515 command.extend(files)
516
517 status_letter = {
518 None: ' ',
519 '': ' ',
520 'added': 'A',
521 'conflicted': 'C',
522 'deleted': 'D',
523 'external': 'X',
524 'ignored': 'I',
525 'incomplete': '!',
526 'merged': 'G',
527 'missing': '!',
528 'modified': 'M',
529 'none': ' ',
530 'normal': ' ',
531 'obstructed': '~',
532 'replaced': 'R',
533 'unversioned': '?',
534 }
535 dom = gclient_utils.ParseXML(CaptureSVN(command))
536 results = []
537 if dom:
538 # /status/target/entry/(wc-status|commit|author|date)
539 for target in dom.getElementsByTagName('target'):
540 base_path = target.getAttribute('path')
541 for entry in target.getElementsByTagName('entry'):
542 file = entry.getAttribute('path')
543 wc_status = entry.getElementsByTagName('wc-status')
544 assert len(wc_status) == 1
545 # Emulate svn 1.5 status ouput...
546 statuses = [' ' for i in range(7)]
547 # Col 0
548 xml_item_status = wc_status[0].getAttribute('item')
549 if xml_item_status in status_letter:
550 statuses[0] = status_letter[xml_item_status]
551 else:
552 raise Exception('Unknown item status "%s"; please implement me!' %
553 xml_item_status)
554 # Col 1
555 xml_props_status = wc_status[0].getAttribute('props')
556 if xml_props_status == 'modified':
557 statuses[1] = 'M'
558 elif xml_props_status == 'conflicted':
559 statuses[1] = 'C'
560 elif (not xml_props_status or xml_props_status == 'none' or
561 xml_props_status == 'normal'):
562 pass
563 else:
564 raise Exception('Unknown props status "%s"; please implement me!' %
565 xml_props_status)
566 # Col 2
567 if wc_status[0].getAttribute('wc-locked') == 'true':
568 statuses[2] = 'L'
569 # Col 3
570 if wc_status[0].getAttribute('copied') == 'true':
571 statuses[3] = '+'
maruel@chromium.orgaa3dd472009-09-21 19:02:48 +0000572 # Col 4
573 if wc_status[0].getAttribute('switched') == 'true':
574 statuses[4] = 'S'
575 # TODO(maruel): Col 5 and 6
maruel@chromium.org5f3eee32009-09-17 00:34:30 +0000576 item = (''.join(statuses), file)
577 results.append(item)
578 return results