blob: 1c1df13a482573f34466099405967ee5f5fc9a9d [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.org3c55d982010-05-06 14:25:44 +00007import cStringIO
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00008import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00009import os
10import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000011import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000012import subprocess
13import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000014import tempfile
maruel@chromium.orgfd876172010-04-30 14:01:05 +000015import time
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000016import xml.dom.minidom
17
18import gclient_utils
19
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000020def ValidateEmail(email):
maruel@chromium.org6e29d572010-06-04 17:32:20 +000021 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
22 is not None)
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000023
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000024
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000025def GetCasedPath(path):
26 """Elcheapos way to get the real path case on Windows."""
27 if sys.platform.startswith('win') and os.path.exists(path):
28 # Reconstruct the path.
29 path = os.path.abspath(path)
30 paths = path.split('\\')
31 for i in range(len(paths)):
32 if i == 0:
33 # Skip drive letter.
34 continue
35 subpath = '\\'.join(paths[:i+1])
36 prev = len('\\'.join(paths[:i]))
37 # glob.glob will return the cased path for the last item only. This is why
38 # we are calling it in a loop. Extract the data we want and put it back
39 # into the list.
40 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
41 path = '\\'.join(paths)
42 return path
43
44
maruel@chromium.org3c55d982010-05-06 14:25:44 +000045def GenFakeDiff(filename):
46 """Generates a fake diff from a file."""
47 file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True)
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +000048 filename = filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +000049 nb_lines = len(file_content)
50 # We need to use / since patch on unix will fail otherwise.
51 data = cStringIO.StringIO()
52 data.write("Index: %s\n" % filename)
53 data.write('=' * 67 + '\n')
54 # Note: Should we use /dev/null instead?
55 data.write("--- %s\n" % filename)
56 data.write("+++ %s\n" % filename)
57 data.write("@@ -0,0 +1,%d @@\n" % nb_lines)
58 # Prepend '+' to every lines.
59 for line in file_content:
60 data.write('+')
61 data.write(line)
62 result = data.getvalue()
63 data.close()
64 return result
65
66
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000067class GIT(object):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000068 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000069 def Capture(args, in_directory=None, print_error=True, error_ok=False):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000070 """Runs git, capturing output sent to stdout as a string.
71
72 Args:
73 args: A sequence of command line parameters to be passed to git.
74 in_directory: The directory where git is to be run.
75
76 Returns:
77 The output sent to stdout as a string.
78 """
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000079 try:
maruel@chromium.org17d01792010-09-01 18:07:10 +000080 return gclient_utils.CheckCall(['git'] + args, in_directory, print_error)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000081 except gclient_utils.CheckCallError:
82 if error_ok:
nasser@codeaurora.orgcd968c12010-02-01 06:05:00 +000083 return ('', '')
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000084 raise
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000085
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000086 @staticmethod
msb@chromium.org786fb682010-06-02 15:16:23 +000087 def CaptureStatus(files, upstream_branch=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000088 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000089
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000090 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000091
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000092 Returns an array of (status, file) tuples."""
msb@chromium.org786fb682010-06-02 15:16:23 +000093 if upstream_branch is None:
94 upstream_branch = GIT.GetUpstreamBranch(os.getcwd())
95 if upstream_branch is None:
96 raise Exception("Cannot determine upstream branch")
bauerb@chromium.org14ec5042010-03-30 18:19:09 +000097 command = ["diff", "--name-status", "-r", "%s..." % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000098 if not files:
99 pass
100 elif isinstance(files, basestring):
101 command.append(files)
102 else:
103 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000104
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000105 status = GIT.Capture(command)[0].rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000106 results = []
107 if status:
108 for statusline in status.split('\n'):
109 m = re.match('^(\w)\t(.+)$', statusline)
110 if not m:
111 raise Exception("status currently unsupported: %s" % statusline)
112 results.append(('%s ' % m.group(1), m.group(2)))
113 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000114
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000115 @staticmethod
116 def GetEmail(repo_root):
117 """Retrieves the user email address if known."""
118 # We could want to look at the svn cred when it has a svn remote but it
119 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000120 return GIT.Capture(['config', 'user.email'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000121 repo_root, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000122
123 @staticmethod
124 def ShortBranchName(branch):
125 """Converts a name like 'refs/heads/foo' to just 'foo'."""
126 return branch.replace('refs/heads/', '')
127
128 @staticmethod
129 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000130 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000131 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000132
133 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000134 def GetBranch(cwd):
135 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000136 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000137
138 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000139 def IsGitSvn(cwd):
140 """Returns true if this repo looks like it's using git-svn."""
141 # If you have any "svn-remote.*" config keys, we think you're using svn.
142 try:
143 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd)
144 return True
145 except gclient_utils.CheckCallError:
146 return False
147
148 @staticmethod
149 def GetSVNBranch(cwd):
150 """Returns the svn branch name if found."""
151 # Try to figure out which remote branch we're based on.
152 # Strategy:
153 # 1) find all git-svn branches and note their svn URLs.
154 # 2) iterate through our branch history and match up the URLs.
155
156 # regexp matching the git-svn line that contains the URL.
157 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
158
159 # Get the refname and svn url for all refs/remotes/*.
160 remotes = GIT.Capture(
161 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000162 cwd)[0].splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000163 svn_refs = {}
164 for ref in remotes:
165 match = git_svn_re.search(
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000166 GIT.Capture(['cat-file', '-p', ref], cwd)[0])
sky@chromium.org42d8da52010-04-23 18:25:07 +0000167 # Prefer origin/HEAD over all others.
168 if match and (match.group(1) not in svn_refs or
169 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000170 svn_refs[match.group(1)] = ref
171
172 svn_branch = ''
173 if len(svn_refs) == 1:
174 # Only one svn branch exists -- seems like a good candidate.
175 svn_branch = svn_refs.values()[0]
176 elif len(svn_refs) > 1:
177 # We have more than one remote branch available. We don't
178 # want to go through all of history, so read a line from the
179 # pipe at a time.
180 # The -100 is an arbitrary limit so we don't search forever.
181 cmd = ['git', 'log', '-100', '--pretty=medium']
maruel@chromium.org3a292682010-08-23 18:54:55 +0000182 proc = gclient_utils.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000183 for line in proc.stdout:
184 match = git_svn_re.match(line)
185 if match:
186 url = match.group(1)
187 if url in svn_refs:
188 svn_branch = svn_refs[url]
189 proc.stdout.close() # Cut pipe.
190 break
191 return svn_branch
192
193 @staticmethod
194 def FetchUpstreamTuple(cwd):
195 """Returns a tuple containg remote and remote ref,
196 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000197 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000198 """
199 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000200 branch = GIT.GetBranch(cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000201 upstream_branch = None
202 upstream_branch = GIT.Capture(
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000203 ['config', 'branch.%s.merge' % branch], in_directory=cwd,
204 error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000205 if upstream_branch:
206 remote = GIT.Capture(
207 ['config', 'branch.%s.remote' % branch],
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000208 in_directory=cwd, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000209 else:
210 # Fall back on trying a git-svn upstream branch.
211 if GIT.IsGitSvn(cwd):
212 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000213 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000214 # Else, try to guess the origin remote.
215 remote_branches = GIT.Capture(
216 ['branch', '-r'], in_directory=cwd)[0].split()
217 if 'origin/master' in remote_branches:
218 # Fall back on origin/master if it exits.
219 remote = 'origin'
220 upstream_branch = 'refs/heads/master'
221 elif 'origin/trunk' in remote_branches:
222 # Fall back on origin/trunk if it exists. Generally a shared
223 # git-svn clone
224 remote = 'origin'
225 upstream_branch = 'refs/heads/trunk'
226 else:
227 # Give up.
228 remote = None
229 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000230 return remote, upstream_branch
231
232 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000233 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000234 """Gets the current branch's upstream branch."""
235 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000236 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000237 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
238 return upstream_branch
239
240 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000241 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
242 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000243 """Diffs against the upstream branch or optionally another branch.
244
245 full_move means that move or copy operations should completely recreate the
246 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000247 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000248 branch = GIT.GetUpstreamBranch(cwd)
evan@chromium.org400f3e72010-05-19 14:23:36 +0000249 command = ['diff', '-p', '--no-prefix', '--no-ext-diff',
250 branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000251 if not full_move:
252 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000253 # TODO(maruel): --binary support.
254 if files:
255 command.append('--')
256 command.extend(files)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000257 diff = GIT.Capture(command, cwd)[0].splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000258 for i in range(len(diff)):
259 # In the case of added files, replace /dev/null with the path to the
260 # file being added.
261 if diff[i].startswith('--- /dev/null'):
262 diff[i] = '--- %s' % diff[i+1][4:]
263 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000264
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000265 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000266 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
267 """Returns the list of modified files between two branches."""
268 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000269 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000270 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000271 return GIT.Capture(command, cwd)[0].splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000272
273 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000274 def GetPatchName(cwd):
275 """Constructs a name for this patch."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000276 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd)[0].strip()
maruel@chromium.org862ff8e2010-08-06 15:29:16 +0000277 return "%s#%s" % (GIT.GetBranch(cwd), short_sha)
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000278
279 @staticmethod
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000280 def GetCheckoutRoot(path):
281 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000282 """
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000283 root = GIT.Capture(['rev-parse', '--show-cdup'], path)[0].strip()
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000284 return os.path.abspath(os.path.join(path, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000285
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000286 @staticmethod
287 def AssertVersion(min_version):
288 """Asserts git's version is at least min_version."""
289 def only_int(val):
290 if val.isdigit():
291 return int(val)
292 else:
293 return 0
294 current_version = GIT.Capture(['--version'])[0].split()[-1]
295 current_version_list = map(only_int, current_version.split('.'))
296 for min_ver in map(int, min_version.split('.')):
297 ver = current_version_list.pop(0)
298 if ver < min_ver:
299 return (False, current_version)
300 elif ver > min_ver:
301 return (True, current_version)
302 return (True, current_version)
303
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000304
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000305class SVN(object):
tony@chromium.org57564662010-04-14 02:35:12 +0000306 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000307
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000308 @staticmethod
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000309 def Run(args, **kwargs):
maruel@chromium.org17d01792010-09-01 18:07:10 +0000310 """Wrappers to gclient_utils.CheckCallAndFilterAndHeader()."""
311 return gclient_utils.CheckCallAndFilterAndHeader(['svn'] + args,
312 always=True, **kwargs)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000313
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000314 @staticmethod
315 def Capture(args, in_directory=None, print_error=True):
316 """Runs svn, capturing output sent to stdout as a string.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000317
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000318 Args:
319 args: A sequence of command line parameters to be passed to svn.
320 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000321
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000322 Returns:
323 The output sent to stdout as a string.
324 """
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000325 stderr = None
326 if not print_error:
327 stderr = subprocess.PIPE
maruel@chromium.org17d01792010-09-01 18:07:10 +0000328 return gclient_utils.Popen(['svn'] + args, cwd=in_directory,
329 stdout=subprocess.PIPE, stderr=stderr).communicate()[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000330
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000331 @staticmethod
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000332 def RunAndGetFileList(verbose, args, cwd, file_list, stdout=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000333 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000334
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000335 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000336
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000337 svn's stdout is parsed to collect a list of files checked out or updated.
338 These files are appended to file_list. svn's stdout is also printed to
339 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000340
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000341 Args:
maruel@chromium.org03807072010-08-16 17:18:44 +0000342 verbose: If True, uses verbose output
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000343 args: A sequence of command line parameters to be passed to svn.
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000344 cwd: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000345
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000346 Raises:
347 Error: An error occurred while running the svn command.
348 """
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000349 stdout = stdout or sys.stdout
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000350
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000351 # svn update and svn checkout use the same pattern: the first three columns
352 # are for file status, property status, and lock status. This is followed
353 # by two spaces, and then the path to the file.
354 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000355
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000356 # The first three columns of svn status are the same as for svn update and
357 # svn checkout. The next three columns indicate addition-with-history,
358 # switch, and remote lock status. This is followed by one space, and then
359 # the path to the file.
360 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000361
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000362 # args[0] must be a supported command. This will blow up if it's something
363 # else, which is good. Note that the patterns are only effective when
364 # these commands are used in their ordinary forms, the patterns are invalid
365 # for "svn status --show-updates", for example.
366 pattern = {
367 'checkout': update_pattern,
368 'status': status_pattern,
369 'update': update_pattern,
370 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000371 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000372 # Place an upper limit.
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000373 for _ in range(10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000374 previous_list_len = len(file_list)
375 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000376
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000377 def CaptureMatchingLines(line):
378 match = compiled_pattern.search(line)
379 if match:
380 file_list.append(match.group(1))
381 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000382 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000383
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000384 try:
maruel@chromium.org17d01792010-09-01 18:07:10 +0000385 gclient_utils.CheckCallAndFilterAndHeader(
386 ['svn'] + args,
387 cwd=cwd,
388 always=verbose,
389 filter_fn=CaptureMatchingLines,
390 stdout=stdout)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000391 except gclient_utils.Error:
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000392 def IsKnownFailure():
393 for x in failure:
394 if (x.startswith('svn: OPTIONS of') or
395 x.startswith('svn: PROPFIND of') or
396 x.startswith('svn: REPORT of') or
maruel@chromium.orgf61fc932010-08-19 13:05:24 +0000397 x.startswith('svn: Unknown hostname') or
398 x.startswith('svn: Server sent unexpected return value')):
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000399 return True
400 return False
401
maruel@chromium.org953586a2010-06-15 14:22:24 +0000402 # Subversion client is really misbehaving with Google Code.
403 if args[0] == 'checkout':
404 # Ensure at least one file was checked out, otherwise *delete* the
405 # directory.
406 if len(file_list) == previous_list_len:
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000407 if not IsKnownFailure():
maruel@chromium.org953586a2010-06-15 14:22:24 +0000408 # No known svn error was found, bail out.
409 raise
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000410 # No file were checked out, so make sure the directory is
411 # deleted in case it's messed up and try again.
412 # Warning: It's bad, it assumes args[2] is the directory
413 # argument.
414 if os.path.isdir(args[2]):
415 gclient_utils.RemoveDirectory(args[2])
maruel@chromium.org953586a2010-06-15 14:22:24 +0000416 else:
417 # Progress was made, convert to update since an aborted checkout
418 # is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000419 args = ['update'] + args[1:]
maruel@chromium.org953586a2010-06-15 14:22:24 +0000420 else:
421 # It was an update or export.
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000422 # We enforce that some progress has been made or a known failure.
423 if len(file_list) == previous_list_len and not IsKnownFailure():
424 # No known svn error was found and no progress, bail out.
425 raise
maruel@chromium.org953586a2010-06-15 14:22:24 +0000426 print "Sleeping 15 seconds and retrying...."
427 time.sleep(15)
428 continue
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000429 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000430
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000431 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000432 def CaptureInfo(relpath, in_directory=None, print_error=True):
433 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000434
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000435 Args:
436 relpath: The directory where the working copy resides relative to
437 the directory given by in_directory.
438 in_directory: The directory where svn is to be run.
439 """
440 output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
441 dom = gclient_utils.ParseXML(output)
442 result = {}
443 if dom:
444 GetNamedNodeText = gclient_utils.GetNamedNodeText
445 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
446 def C(item, f):
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000447 if item is not None:
448 return f(item)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000449 # /info/entry/
450 # url
451 # reposityory/(root|uuid)
452 # wc-info/(schedule|depth)
453 # commit/(author|date)
454 # str() the results because they may be returned as Unicode, which
455 # interferes with the higher layers matching up things in the deps
456 # dictionary.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000457 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
458 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
459 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
460 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
461 'revision'),
462 int)
463 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
464 str)
465 # Differs across versions.
466 if result['Node Kind'] == 'dir':
467 result['Node Kind'] = 'directory'
468 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
469 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
470 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
471 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
472 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000473
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000474 @staticmethod
475 def CaptureHeadRevision(url):
476 """Get the head revision of a SVN repository.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000477
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000478 Returns:
479 Int head revision
480 """
481 info = SVN.Capture(["info", "--xml", url], os.getcwd())
482 dom = xml.dom.minidom.parseString(info)
483 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000484
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000485 @staticmethod
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000486 def CaptureBaseRevision(cwd):
487 """Get the base revision of a SVN repository.
488
489 Returns:
490 Int base revision
491 """
492 info = SVN.Capture(["info", "--xml"], cwd)
493 dom = xml.dom.minidom.parseString(info)
494 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
495
496 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000497 def CaptureStatus(files):
498 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000499
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000500 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000501
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000502 Returns an array of (status, file) tuples."""
503 command = ["status", "--xml"]
504 if not files:
505 pass
506 elif isinstance(files, basestring):
507 command.append(files)
508 else:
509 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000510
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000511 status_letter = {
512 None: ' ',
513 '': ' ',
514 'added': 'A',
515 'conflicted': 'C',
516 'deleted': 'D',
517 'external': 'X',
518 'ignored': 'I',
519 'incomplete': '!',
520 'merged': 'G',
521 'missing': '!',
522 'modified': 'M',
523 'none': ' ',
524 'normal': ' ',
525 'obstructed': '~',
526 'replaced': 'R',
527 'unversioned': '?',
528 }
529 dom = gclient_utils.ParseXML(SVN.Capture(command))
530 results = []
531 if dom:
532 # /status/target/entry/(wc-status|commit|author|date)
533 for target in dom.getElementsByTagName('target'):
534 #base_path = target.getAttribute('path')
535 for entry in target.getElementsByTagName('entry'):
536 file_path = entry.getAttribute('path')
537 wc_status = entry.getElementsByTagName('wc-status')
538 assert len(wc_status) == 1
539 # Emulate svn 1.5 status ouput...
540 statuses = [' '] * 7
541 # Col 0
542 xml_item_status = wc_status[0].getAttribute('item')
543 if xml_item_status in status_letter:
544 statuses[0] = status_letter[xml_item_status]
545 else:
546 raise Exception('Unknown item status "%s"; please implement me!' %
547 xml_item_status)
548 # Col 1
549 xml_props_status = wc_status[0].getAttribute('props')
550 if xml_props_status == 'modified':
551 statuses[1] = 'M'
552 elif xml_props_status == 'conflicted':
553 statuses[1] = 'C'
554 elif (not xml_props_status or xml_props_status == 'none' or
555 xml_props_status == 'normal'):
556 pass
557 else:
558 raise Exception('Unknown props status "%s"; please implement me!' %
559 xml_props_status)
560 # Col 2
561 if wc_status[0].getAttribute('wc-locked') == 'true':
562 statuses[2] = 'L'
563 # Col 3
564 if wc_status[0].getAttribute('copied') == 'true':
565 statuses[3] = '+'
566 # Col 4
567 if wc_status[0].getAttribute('switched') == 'true':
568 statuses[4] = 'S'
569 # TODO(maruel): Col 5 and 6
570 item = (''.join(statuses), file_path)
571 results.append(item)
572 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000573
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000574 @staticmethod
575 def IsMoved(filename):
576 """Determine if a file has been added through svn mv"""
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000577 return SVN.IsMovedInfo(SVN.CaptureInfo(filename))
578
579 @staticmethod
580 def IsMovedInfo(info):
581 """Determine if a file has been added through svn mv"""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000582 return (info.get('Copied From URL') and
583 info.get('Copied From Rev') and
584 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000585
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000586 @staticmethod
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000587 def GetFileProperty(filename, property_name):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000588 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000589
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000590 Args:
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000591 filename: The file to check
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000592 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000593
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000594 Returns:
595 The value of the property, which will be the empty string if the property
596 is not set on the file. If the file is not under version control, the
597 empty string is also returned.
598 """
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000599 output = SVN.Capture(["propget", property_name, filename])
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000600 if (output.startswith("svn: ") and
601 output.endswith("is not under version control")):
602 return ""
603 else:
604 return output
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000605
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000606 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000607 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000608 """Diffs a single file.
609
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000610 Should be simple, eh? No it isn't.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000611 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000612 expected relative path.
613 full_move means that move or copy operations should completely recreate the
614 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000615 # If the user specified a custom diff command in their svn config file,
616 # then it'll be used when we do svn diff, which we don't want to happen
617 # since we want the unified diff. Using --diff-cmd=diff doesn't always
618 # work, since they can have another diff executable in their path that
619 # gives different line endings. So we use a bogus temp directory as the
620 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000621 bogus_dir = tempfile.mkdtemp()
622 try:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000623 # Use "svn info" output instead of os.path.isdir because the latter fails
624 # when the file is deleted.
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000625 return SVN._DiffItemInternal(filename, SVN.CaptureInfo(filename),
626 bogus_dir,
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000627 full_move=full_move, revision=revision)
628 finally:
629 shutil.rmtree(bogus_dir)
630
631 @staticmethod
632 def _DiffItemInternal(filename, info, bogus_dir, full_move=False,
633 revision=None):
634 """Grabs the diff data."""
635 command = ["diff", "--config-dir", bogus_dir, filename]
636 if revision:
637 command.extend(['--revision', revision])
638 data = None
639 if SVN.IsMovedInfo(info):
640 if full_move:
641 if info.get("Node Kind") == "directory":
642 # Things become tricky here. It's a directory copy/move. We need to
643 # diff all the files inside it.
644 # This will put a lot of pressure on the heap. This is why StringIO
645 # is used and converted back into a string at the end. The reason to
646 # return a string instead of a StringIO is that StringIO.write()
647 # doesn't accept a StringIO object. *sigh*.
648 for (dirpath, dirnames, filenames) in os.walk(filename):
649 # Cleanup all files starting with a '.'.
650 for d in dirnames:
651 if d.startswith('.'):
652 dirnames.remove(d)
653 for f in filenames:
654 if f.startswith('.'):
655 filenames.remove(f)
656 for f in filenames:
657 if data is None:
658 data = cStringIO.StringIO()
659 data.write(GenFakeDiff(os.path.join(dirpath, f)))
660 if data:
661 tmp = data.getvalue()
662 data.close()
663 data = tmp
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000664 else:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000665 data = GenFakeDiff(filename)
666 else:
667 if info.get("Node Kind") != "directory":
maruel@chromium.org0836c562010-01-22 01:10:06 +0000668 # svn diff on a mv/cp'd file outputs nothing if there was no change.
669 data = SVN.Capture(command, None)
670 if not data:
671 # We put in an empty Index entry so upload.py knows about them.
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +0000672 data = "Index: %s\n" % filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000673 # Otherwise silently ignore directories.
674 else:
675 if info.get("Node Kind") != "directory":
676 # Normal simple case.
maruel@chromium.org0836c562010-01-22 01:10:06 +0000677 data = SVN.Capture(command, None)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000678 # Otherwise silently ignore directories.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000679 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000680
681 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000682 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000683 """Returns a string containing the diff for the given file list.
684
685 The files in the list should either be absolute paths or relative to the
686 given root. If no root directory is provided, the repository root will be
687 used.
688 The diff will always use relative paths.
689 """
690 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000691 root = root or SVN.GetCheckoutRoot(previous_cwd)
692 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000693 def RelativePath(path, root):
694 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000695 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000696 return path[len(root):]
697 return path
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000698 # If the user specified a custom diff command in their svn config file,
699 # then it'll be used when we do svn diff, which we don't want to happen
700 # since we want the unified diff. Using --diff-cmd=diff doesn't always
701 # work, since they can have another diff executable in their path that
702 # gives different line endings. So we use a bogus temp directory as the
703 # config directory, which gets around these problems.
704 bogus_dir = tempfile.mkdtemp()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000705 try:
706 os.chdir(root)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000707 # Cleanup filenames
708 filenames = [RelativePath(f, root) for f in filenames]
709 # Get information about the modified items (files and directories)
710 data = dict([(f, SVN.CaptureInfo(f)) for f in filenames])
gavinp@google.com3fda4cc2010-06-29 13:29:27 +0000711 diffs = []
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000712 if full_move:
713 # Eliminate modified files inside moved/copied directory.
714 for (filename, info) in data.iteritems():
715 if SVN.IsMovedInfo(info) and info.get("Node Kind") == "directory":
716 # Remove files inside the directory.
717 filenames = [f for f in filenames
718 if not f.startswith(filename + os.path.sep)]
719 for filename in data.keys():
720 if not filename in filenames:
721 # Remove filtered out items.
722 del data[filename]
gavinp@google.com3fda4cc2010-06-29 13:29:27 +0000723 else:
724 metaheaders = []
725 for (filename, info) in data.iteritems():
726 if SVN.IsMovedInfo(info):
727 # for now, the most common case is a head copy,
728 # so let's just encode that as a straight up cp.
729 srcurl = info.get('Copied From URL')
730 root = info.get('Repository Root')
731 rev = int(info.get('Copied From Rev'))
732 assert srcurl.startswith(root)
733 src = srcurl[len(root)+1:]
734 srcinfo = SVN.CaptureInfo(srcurl)
735 if (srcinfo.get('Revision') != rev and
736 SVN.Capture(['diff', '-r', '%d:head' % rev, srcurl])):
737 metaheaders.append("#$ svn cp -r %d %s %s "
738 "### WARNING: note non-trunk copy\n" %
739 (rev, src, filename))
740 else:
741 metaheaders.append("#$ cp %s %s\n" % (src,
742 filename))
743
744 if metaheaders:
745 diffs.append("### BEGIN SVN COPY METADATA\n")
746 diffs.extend(metaheaders)
747 diffs.append("### END SVN COPY METADATA\n")
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000748 # Now ready to do the actual diff.
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000749 for filename in sorted(data.iterkeys()):
750 diffs.append(SVN._DiffItemInternal(filename, data[filename], bogus_dir,
751 full_move=full_move,
752 revision=revision))
753 # Use StringIO since it can be messy when diffing a directory move with
754 # full_move=True.
755 buf = cStringIO.StringIO()
756 for d in filter(None, diffs):
757 buf.write(d)
758 result = buf.getvalue()
759 buf.close()
760 return result
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000761 finally:
762 os.chdir(previous_cwd)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000763 shutil.rmtree(bogus_dir)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000764
765 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000766 def GetEmail(repo_root):
767 """Retrieves the svn account which we assume is an email address."""
768 infos = SVN.CaptureInfo(repo_root)
769 uuid = infos.get('UUID')
770 root = infos.get('Repository Root')
771 if not root:
772 return None
773
774 # Should check for uuid but it is incorrectly saved for https creds.
775 realm = root.rsplit('/', 1)[0]
776 if root.startswith('https') or not uuid:
777 regexp = re.compile(r'<%s:\d+>.*' % realm)
778 else:
779 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
780 if regexp is None:
781 return None
782 if sys.platform.startswith('win'):
783 if not 'APPDATA' in os.environ:
784 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000785 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
786 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000787 else:
788 if not 'HOME' in os.environ:
789 return None
790 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
791 'svn.simple')
792 for credfile in os.listdir(auth_dir):
793 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
794 if regexp.match(cred_info.get('svn:realmstring')):
795 return cred_info.get('username')
796
797 @staticmethod
798 def ReadSimpleAuth(filename):
799 f = open(filename, 'r')
800 values = {}
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000801 def ReadOneItem(item_type):
802 m = re.match(r'%s (\d+)' % item_type, f.readline())
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000803 if not m:
804 return None
805 data = f.read(int(m.group(1)))
806 if f.read(1) != '\n':
807 return None
808 return data
809
810 while True:
811 key = ReadOneItem('K')
812 if not key:
813 break
814 value = ReadOneItem('V')
815 if not value:
816 break
817 values[key] = value
818 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000819
820 @staticmethod
821 def GetCheckoutRoot(directory):
822 """Returns the top level directory of the current repository.
823
824 The directory is returned as an absolute path.
825 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000826 directory = os.path.abspath(directory)
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000827 infos = SVN.CaptureInfo(directory, print_error=False)
828 cur_dir_repo_root = infos.get("Repository Root")
829 if not cur_dir_repo_root:
830 return None
831
832 while True:
833 parent = os.path.dirname(directory)
834 if (SVN.CaptureInfo(parent, print_error=False).get(
835 "Repository Root") != cur_dir_repo_root):
836 break
837 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000838 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000839
840 @staticmethod
841 def AssertVersion(min_version):
842 """Asserts svn's version is at least min_version."""
843 def only_int(val):
844 if val.isdigit():
845 return int(val)
846 else:
847 return 0
848 if not SVN.current_version:
849 SVN.current_version = SVN.Capture(['--version']).split()[2]
850 current_version_list = map(only_int, SVN.current_version.split('.'))
851 for min_ver in map(int, min_version.split('.')):
852 ver = current_version_list.pop(0)
853 if ver < min_ver:
854 return (False, SVN.current_version)
855 elif ver > min_ver:
856 return (True, SVN.current_version)
857 return (True, SVN.current_version)