Reapply 32057, 32058, 32059, 32062 and fixes problems introduced by these changes.
Noteworthy change is scm.SVN.GetFileProperty calls Capture instead of Run.
TEST=unit tests
BUG=none
Review URL: http://codereview.chromium.org/399009
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@32181 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/scm.py b/scm.py
index 1284815..21363d3 100644
--- a/scm.py
+++ b/scm.py
@@ -2,335 +2,414 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-"""SCM-specific functions."""
+"""SCM-specific utility classes."""
import os
import re
import subprocess
import sys
+import tempfile
import xml.dom.minidom
import gclient_utils
-SVN_COMMAND = "svn"
-GIT_COMMAND = "git"
+class GIT(object):
+ COMMAND = "git"
-# -----------------------------------------------------------------------------
-# Git utils:
+ @staticmethod
+ def Capture(args, in_directory=None, print_error=True):
+ """Runs git, capturing output sent to stdout as a string.
+
+ Args:
+ args: A sequence of command line parameters to be passed to git.
+ in_directory: The directory where git is to be run.
+
+ Returns:
+ The output sent to stdout as a string.
+ """
+ c = [GIT.COMMAND]
+ c.extend(args)
+
+ # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
+ # the git.exe executable, but shell=True makes subprocess on Linux fail
+ # when it's called with a list because it only tries to execute the
+ # first string ("git").
+ stderr = None
+ if not print_error:
+ stderr = subprocess.PIPE
+ return subprocess.Popen(c,
+ cwd=in_directory,
+ shell=sys.platform.startswith('win'),
+ stdout=subprocess.PIPE,
+ stderr=stderr).communicate()[0]
-def CaptureGit(args, in_directory=None, print_error=True):
- """Runs git, capturing output sent to stdout as a string.
+ @staticmethod
+ def CaptureStatus(files, upstream_branch='origin'):
+ """Returns git status.
- Args:
- args: A sequence of command line parameters to be passed to git.
- in_directory: The directory where git is to be run.
+ @files can be a string (one file) or a list of files.
- Returns:
- The output sent to stdout as a string.
- """
- c = [GIT_COMMAND]
- c.extend(args)
+ Returns an array of (status, file) tuples."""
+ command = ["diff", "--name-status", "-r", "%s.." % upstream_branch]
+ if not files:
+ pass
+ elif isinstance(files, basestring):
+ command.append(files)
+ else:
+ command.extend(files)
- # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
- # the git.exe executable, but shell=True makes subprocess on Linux fail
- # when it's called with a list because it only tries to execute the
- # first string ("git").
- stderr = None
- if not print_error:
- stderr = subprocess.PIPE
- return subprocess.Popen(c,
- cwd=in_directory,
- shell=sys.platform.startswith('win'),
- stdout=subprocess.PIPE,
- stderr=stderr).communicate()[0]
+ status = GIT.Capture(command).rstrip()
+ results = []
+ if status:
+ for statusline in status.split('\n'):
+ m = re.match('^(\w)\t(.+)$', statusline)
+ if not m:
+ raise Exception("status currently unsupported: %s" % statusline)
+ results.append(('%s ' % m.group(1), m.group(2)))
+ return results
-def CaptureGitStatus(files, upstream_branch='origin'):
- """Returns git status.
+class SVN(object):
+ COMMAND = "svn"
- @files can be a string (one file) or a list of files.
+ @staticmethod
+ def Run(args, in_directory):
+ """Runs svn, sending output to stdout.
- Returns an array of (status, file) tuples."""
- command = ["diff", "--name-status", "-r", "%s.." % upstream_branch]
- if not files:
- pass
- elif isinstance(files, basestring):
- command.append(files)
- else:
- command.extend(files)
+ Args:
+ args: A sequence of command line parameters to be passed to svn.
+ in_directory: The directory where svn is to be run.
- status = CaptureGit(command).rstrip()
- results = []
- if status:
- for statusline in status.split('\n'):
- m = re.match('^(\w)\t(.+)$', statusline)
- if not m:
- raise Exception("status currently unsupported: %s" % statusline)
- results.append(('%s ' % m.group(1), m.group(2)))
- return results
+ Raises:
+ Error: An error occurred while running the svn command.
+ """
+ c = [SVN.COMMAND]
+ c.extend(args)
+ gclient_utils.SubprocessCall(c, in_directory)
-# -----------------------------------------------------------------------------
-# SVN utils:
+ @staticmethod
+ def Capture(args, in_directory=None, print_error=True):
+ """Runs svn, capturing output sent to stdout as a string.
+ Args:
+ args: A sequence of command line parameters to be passed to svn.
+ in_directory: The directory where svn is to be run.
-def RunSVN(args, in_directory):
- """Runs svn, sending output to stdout.
+ Returns:
+ The output sent to stdout as a string.
+ """
+ c = [SVN.COMMAND]
+ c.extend(args)
- Args:
- args: A sequence of command line parameters to be passed to svn.
- in_directory: The directory where svn is to be run.
+ # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
+ # the svn.exe executable, but shell=True makes subprocess on Linux fail
+ # when it's called with a list because it only tries to execute the
+ # first string ("svn").
+ stderr = None
+ if not print_error:
+ stderr = subprocess.PIPE
+ return subprocess.Popen(c,
+ cwd=in_directory,
+ shell=(sys.platform == 'win32'),
+ stdout=subprocess.PIPE,
+ stderr=stderr).communicate()[0]
- Raises:
- Error: An error occurred while running the svn command.
- """
- c = [SVN_COMMAND]
- c.extend(args)
+ @staticmethod
+ def RunAndGetFileList(options, args, in_directory, file_list):
+ """Runs svn checkout, update, or status, output to stdout.
- gclient_utils.SubprocessCall(c, in_directory)
+ The first item in args must be either "checkout", "update", or "status".
+ svn's stdout is parsed to collect a list of files checked out or updated.
+ These files are appended to file_list. svn's stdout is also printed to
+ sys.stdout as in Run.
-def CaptureSVN(args, in_directory=None, print_error=True):
- """Runs svn, capturing output sent to stdout as a string.
+ Args:
+ options: command line options to gclient
+ args: A sequence of command line parameters to be passed to svn.
+ in_directory: The directory where svn is to be run.
- Args:
- args: A sequence of command line parameters to be passed to svn.
- in_directory: The directory where svn is to be run.
+ Raises:
+ Error: An error occurred while running the svn command.
+ """
+ command = [SVN.COMMAND]
+ command.extend(args)
- Returns:
- The output sent to stdout as a string.
- """
- c = [SVN_COMMAND]
- c.extend(args)
+ # svn update and svn checkout use the same pattern: the first three columns
+ # are for file status, property status, and lock status. This is followed
+ # by two spaces, and then the path to the file.
+ update_pattern = '^... (.*)$'
- # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
- # the svn.exe executable, but shell=True makes subprocess on Linux fail
- # when it's called with a list because it only tries to execute the
- # first string ("svn").
- stderr = None
- if not print_error:
- stderr = subprocess.PIPE
- return subprocess.Popen(c,
- cwd=in_directory,
- shell=(sys.platform == 'win32'),
- stdout=subprocess.PIPE,
- stderr=stderr).communicate()[0]
+ # The first three columns of svn status are the same as for svn update and
+ # svn checkout. The next three columns indicate addition-with-history,
+ # switch, and remote lock status. This is followed by one space, and then
+ # the path to the file.
+ status_pattern = '^...... (.*)$'
+ # args[0] must be a supported command. This will blow up if it's something
+ # else, which is good. Note that the patterns are only effective when
+ # these commands are used in their ordinary forms, the patterns are invalid
+ # for "svn status --show-updates", for example.
+ pattern = {
+ 'checkout': update_pattern,
+ 'status': status_pattern,
+ 'update': update_pattern,
+ }[args[0]]
-def RunSVNAndGetFileList(options, args, in_directory, file_list):
- """Runs svn checkout, update, or status, output to stdout.
+ compiled_pattern = re.compile(pattern)
- The first item in args must be either "checkout", "update", or "status".
+ def CaptureMatchingLines(line):
+ match = compiled_pattern.search(line)
+ if match:
+ file_list.append(match.group(1))
- svn's stdout is parsed to collect a list of files checked out or updated.
- These files are appended to file_list. svn's stdout is also printed to
- sys.stdout as in RunSVN.
+ SVN.RunAndFilterOutput(args,
+ in_directory,
+ options.verbose,
+ True,
+ CaptureMatchingLines)
- Args:
- options: command line options to gclient
- args: A sequence of command line parameters to be passed to svn.
- in_directory: The directory where svn is to be run.
+ @staticmethod
+ def RunAndFilterOutput(args,
+ in_directory,
+ print_messages,
+ print_stdout,
+ filter):
+ """Runs svn checkout, update, status, or diff, optionally outputting
+ to stdout.
- Raises:
- Error: An error occurred while running the svn command.
- """
- command = [SVN_COMMAND]
- command.extend(args)
+ The first item in args must be either "checkout", "update",
+ "status", or "diff".
- # svn update and svn checkout use the same pattern: the first three columns
- # are for file status, property status, and lock status. This is followed
- # by two spaces, and then the path to the file.
- update_pattern = '^... (.*)$'
+ svn's stdout is passed line-by-line to the given filter function. If
+ print_stdout is true, it is also printed to sys.stdout as in Run.
- # The first three columns of svn status are the same as for svn update and
- # svn checkout. The next three columns indicate addition-with-history,
- # switch, and remote lock status. This is followed by one space, and then
- # the path to the file.
- status_pattern = '^...... (.*)$'
+ Args:
+ args: A sequence of command line parameters to be passed to svn.
+ in_directory: The directory where svn is to be run.
+ print_messages: Whether to print status messages to stdout about
+ which Subversion commands are being run.
+ print_stdout: Whether to forward Subversion's output to stdout.
+ filter: A function taking one argument (a string) which will be
+ passed each line (with the ending newline character removed) of
+ Subversion's output for filtering.
- # args[0] must be a supported command. This will blow up if it's something
- # else, which is good. Note that the patterns are only effective when
- # these commands are used in their ordinary forms, the patterns are invalid
- # for "svn status --show-updates", for example.
- pattern = {
- 'checkout': update_pattern,
- 'status': status_pattern,
- 'update': update_pattern,
- }[args[0]]
+ Raises:
+ Error: An error occurred while running the svn command.
+ """
+ command = [SVN.COMMAND]
+ command.extend(args)
- compiled_pattern = re.compile(pattern)
+ gclient_utils.SubprocessCallAndFilter(command,
+ in_directory,
+ print_messages,
+ print_stdout,
+ filter=filter)
- def CaptureMatchingLines(line):
- match = compiled_pattern.search(line)
- if match:
- file_list.append(match.group(1))
+ @staticmethod
+ def CaptureInfo(relpath, in_directory=None, print_error=True):
+ """Returns a dictionary from the svn info output for the given file.
- RunSVNAndFilterOutput(args,
- in_directory,
- options.verbose,
- True,
- CaptureMatchingLines)
+ Args:
+ relpath: The directory where the working copy resides relative to
+ the directory given by in_directory.
+ in_directory: The directory where svn is to be run.
+ """
+ output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
+ dom = gclient_utils.ParseXML(output)
+ result = {}
+ if dom:
+ GetNamedNodeText = gclient_utils.GetNamedNodeText
+ GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
+ def C(item, f):
+ if item is not None: return f(item)
+ # /info/entry/
+ # url
+ # reposityory/(root|uuid)
+ # wc-info/(schedule|depth)
+ # commit/(author|date)
+ # str() the results because they may be returned as Unicode, which
+ # interferes with the higher layers matching up things in the deps
+ # dictionary.
+ # TODO(maruel): Fix at higher level instead (!)
+ result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
+ result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
+ result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
+ result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
+ 'revision'),
+ int)
+ result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
+ str)
+ # Differs across versions.
+ if result['Node Kind'] == 'dir':
+ result['Node Kind'] = 'directory'
+ result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
+ result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
+ result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
+ result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
+ return result
-def RunSVNAndFilterOutput(args,
- in_directory,
- print_messages,
- print_stdout,
- filter):
- """Runs svn checkout, update, status, or diff, optionally outputting
- to stdout.
+ @staticmethod
+ def CaptureHeadRevision(url):
+ """Get the head revision of a SVN repository.
- The first item in args must be either "checkout", "update",
- "status", or "diff".
+ Returns:
+ Int head revision
+ """
+ info = SVN.Capture(["info", "--xml", url], os.getcwd())
+ dom = xml.dom.minidom.parseString(info)
+ return dom.getElementsByTagName('entry')[0].getAttribute('revision')
- svn's stdout is passed line-by-line to the given filter function. If
- print_stdout is true, it is also printed to sys.stdout as in RunSVN.
+ @staticmethod
+ def CaptureStatus(files):
+ """Returns the svn 1.5 svn status emulated output.
- Args:
- args: A sequence of command line parameters to be passed to svn.
- in_directory: The directory where svn is to be run.
- print_messages: Whether to print status messages to stdout about
- which Subversion commands are being run.
- print_stdout: Whether to forward Subversion's output to stdout.
- filter: A function taking one argument (a string) which will be
- passed each line (with the ending newline character removed) of
- Subversion's output for filtering.
+ @files can be a string (one file) or a list of files.
- Raises:
- Error: An error occurred while running the svn command.
- """
- command = [SVN_COMMAND]
- command.extend(args)
+ Returns an array of (status, file) tuples."""
+ command = ["status", "--xml"]
+ if not files:
+ pass
+ elif isinstance(files, basestring):
+ command.append(files)
+ else:
+ command.extend(files)
- gclient_utils.SubprocessCallAndFilter(command,
- in_directory,
- print_messages,
- print_stdout,
- filter=filter)
+ status_letter = {
+ None: ' ',
+ '': ' ',
+ 'added': 'A',
+ 'conflicted': 'C',
+ 'deleted': 'D',
+ 'external': 'X',
+ 'ignored': 'I',
+ 'incomplete': '!',
+ 'merged': 'G',
+ 'missing': '!',
+ 'modified': 'M',
+ 'none': ' ',
+ 'normal': ' ',
+ 'obstructed': '~',
+ 'replaced': 'R',
+ 'unversioned': '?',
+ }
+ dom = gclient_utils.ParseXML(SVN.Capture(command))
+ results = []
+ if dom:
+ # /status/target/entry/(wc-status|commit|author|date)
+ for target in dom.getElementsByTagName('target'):
+ #base_path = target.getAttribute('path')
+ for entry in target.getElementsByTagName('entry'):
+ file_path = entry.getAttribute('path')
+ wc_status = entry.getElementsByTagName('wc-status')
+ assert len(wc_status) == 1
+ # Emulate svn 1.5 status ouput...
+ statuses = [' '] * 7
+ # Col 0
+ xml_item_status = wc_status[0].getAttribute('item')
+ if xml_item_status in status_letter:
+ statuses[0] = status_letter[xml_item_status]
+ else:
+ raise Exception('Unknown item status "%s"; please implement me!' %
+ xml_item_status)
+ # Col 1
+ xml_props_status = wc_status[0].getAttribute('props')
+ if xml_props_status == 'modified':
+ statuses[1] = 'M'
+ elif xml_props_status == 'conflicted':
+ statuses[1] = 'C'
+ elif (not xml_props_status or xml_props_status == 'none' or
+ xml_props_status == 'normal'):
+ pass
+ else:
+ raise Exception('Unknown props status "%s"; please implement me!' %
+ xml_props_status)
+ # Col 2
+ if wc_status[0].getAttribute('wc-locked') == 'true':
+ statuses[2] = 'L'
+ # Col 3
+ if wc_status[0].getAttribute('copied') == 'true':
+ statuses[3] = '+'
+ # Col 4
+ if wc_status[0].getAttribute('switched') == 'true':
+ statuses[4] = 'S'
+ # TODO(maruel): Col 5 and 6
+ item = (''.join(statuses), file_path)
+ results.append(item)
+ return results
-def CaptureSVNInfo(relpath, in_directory=None, print_error=True):
- """Returns a dictionary from the svn info output for the given file.
+ @staticmethod
+ def IsMoved(filename):
+ """Determine if a file has been added through svn mv"""
+ info = SVN.CaptureInfo(filename)
+ return (info.get('Copied From URL') and
+ info.get('Copied From Rev') and
+ info.get('Schedule') == 'add')
- Args:
- relpath: The directory where the working copy resides relative to
- the directory given by in_directory.
- in_directory: The directory where svn is to be run.
- """
- output = CaptureSVN(["info", "--xml", relpath], in_directory, print_error)
- dom = gclient_utils.ParseXML(output)
- result = {}
- if dom:
- GetNamedNodeText = gclient_utils.GetNamedNodeText
- GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
- def C(item, f):
- if item is not None: return f(item)
- # /info/entry/
- # url
- # reposityory/(root|uuid)
- # wc-info/(schedule|depth)
- # commit/(author|date)
- # str() the results because they may be returned as Unicode, which
- # interferes with the higher layers matching up things in the deps
- # dictionary.
- # TODO(maruel): Fix at higher level instead (!)
- result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
- result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
- result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
- result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry', 'revision'),
- int)
- result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
- str)
- result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
- result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
- result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
- result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
- return result
+ @staticmethod
+ def GetFileProperty(file, property_name):
+ """Returns the value of an SVN property for the given file.
+ Args:
+ file: The file to check
+ property_name: The name of the SVN property, e.g. "svn:mime-type"
-def CaptureSVNHeadRevision(url):
- """Get the head revision of a SVN repository.
+ Returns:
+ The value of the property, which will be the empty string if the property
+ is not set on the file. If the file is not under version control, the
+ empty string is also returned.
+ """
+ output = SVN.Capture(["propget", property_name, file])
+ if (output.startswith("svn: ") and
+ output.endswith("is not under version control")):
+ return ""
+ else:
+ return output
- Returns:
- Int head revision
- """
- info = CaptureSVN(["info", "--xml", url], os.getcwd())
- dom = xml.dom.minidom.parseString(info)
- return dom.getElementsByTagName('entry')[0].getAttribute('revision')
+ @staticmethod
+ def DiffItem(filename):
+ """Diff a single file"""
+ # Use svn info output instead of os.path.isdir because the latter fails
+ # when the file is deleted.
+ if SVN.CaptureInfo(filename).get("Node Kind") == "directory":
+ return None
+ # If the user specified a custom diff command in their svn config file,
+ # then it'll be used when we do svn diff, which we don't want to happen
+ # since we want the unified diff. Using --diff-cmd=diff doesn't always
+ # work, since they can have another diff executable in their path that
+ # gives different line endings. So we use a bogus temp directory as the
+ # config directory, which gets around these problems.
+ if sys.platform.startswith("win"):
+ parent_dir = tempfile.gettempdir()
+ else:
+ parent_dir = sys.path[0] # tempdir is not secure.
+ bogus_dir = os.path.join(parent_dir, "temp_svn_config")
+ if not os.path.exists(bogus_dir):
+ os.mkdir(bogus_dir)
+ # Grabs the diff data.
+ data = SVN.Capture(["diff", "--config-dir", bogus_dir, filename], None)
-
-def CaptureSVNStatus(files):
- """Returns the svn 1.5 svn status emulated output.
-
- @files can be a string (one file) or a list of files.
-
- Returns an array of (status, file) tuples."""
- command = ["status", "--xml"]
- if not files:
- pass
- elif isinstance(files, basestring):
- command.append(files)
- else:
- command.extend(files)
-
- status_letter = {
- None: ' ',
- '': ' ',
- 'added': 'A',
- 'conflicted': 'C',
- 'deleted': 'D',
- 'external': 'X',
- 'ignored': 'I',
- 'incomplete': '!',
- 'merged': 'G',
- 'missing': '!',
- 'modified': 'M',
- 'none': ' ',
- 'normal': ' ',
- 'obstructed': '~',
- 'replaced': 'R',
- 'unversioned': '?',
- }
- dom = gclient_utils.ParseXML(CaptureSVN(command))
- results = []
- if dom:
- # /status/target/entry/(wc-status|commit|author|date)
- for target in dom.getElementsByTagName('target'):
- for entry in target.getElementsByTagName('entry'):
- file_path = entry.getAttribute('path')
- wc_status = entry.getElementsByTagName('wc-status')
- assert len(wc_status) == 1
- # Emulate svn 1.5 status ouput...
- statuses = [' '] * 7
- # Col 0
- xml_item_status = wc_status[0].getAttribute('item')
- if xml_item_status in status_letter:
- statuses[0] = status_letter[xml_item_status]
- else:
- raise Exception('Unknown item status "%s"; please implement me!' %
- xml_item_status)
- # Col 1
- xml_props_status = wc_status[0].getAttribute('props')
- if xml_props_status == 'modified':
- statuses[1] = 'M'
- elif xml_props_status == 'conflicted':
- statuses[1] = 'C'
- elif (not xml_props_status or xml_props_status == 'none' or
- xml_props_status == 'normal'):
- pass
- else:
- raise Exception('Unknown props status "%s"; please implement me!' %
- xml_props_status)
- # Col 2
- if wc_status[0].getAttribute('wc-locked') == 'true':
- statuses[2] = 'L'
- # Col 3
- if wc_status[0].getAttribute('copied') == 'true':
- statuses[3] = '+'
- # Col 4
- if wc_status[0].getAttribute('switched') == 'true':
- statuses[4] = 'S'
- # TODO(maruel): Col 5 and 6
- item = (''.join(statuses), file_path)
- results.append(item)
- return results
+ # We know the diff will be incorrectly formatted. Fix it.
+ if SVN.IsMoved(filename):
+ # The file is "new" in the patch sense. Generate a homebrew diff.
+ # We can't use ReadFile() since it's not using binary mode.
+ file_handle = open(filename, 'rb')
+ file_content = file_handle.read()
+ file_handle.close()
+ # Prepend '+' to every lines.
+ file_content = ['+' + i for i in file_content.splitlines(True)]
+ nb_lines = len(file_content)
+ # We need to use / since patch on unix will fail otherwise.
+ filename = filename.replace('\\', '/')
+ data = "Index: %s\n" % filename
+ data += ("============================================================="
+ "======\n")
+ # Note: Should we use /dev/null instead?
+ data += "--- %s\n" % filename
+ data += "+++ %s\n" % filename
+ data += "@@ -0,0 +1,%d @@\n" % nb_lines
+ data += ''.join(file_content)
+ return data