blob: 5f3a9bee1afcd91ce9a6da29bccf7331dc21c9bf [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.org3c55d982010-05-06 14:25:44 +00007import cStringIO
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00008import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00009import os
10import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000011import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000012import subprocess
13import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000014import tempfile
maruel@chromium.orgfd876172010-04-30 14:01:05 +000015import time
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000016import xml.dom.minidom
17
18import gclient_utils
19
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000020def ValidateEmail(email):
21 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
22 is not None)
23
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000024
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000025def GetCasedPath(path):
26 """Elcheapos way to get the real path case on Windows."""
27 if sys.platform.startswith('win') and os.path.exists(path):
28 # Reconstruct the path.
29 path = os.path.abspath(path)
30 paths = path.split('\\')
31 for i in range(len(paths)):
32 if i == 0:
33 # Skip drive letter.
34 continue
35 subpath = '\\'.join(paths[:i+1])
36 prev = len('\\'.join(paths[:i]))
37 # glob.glob will return the cased path for the last item only. This is why
38 # we are calling it in a loop. Extract the data we want and put it back
39 # into the list.
40 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
41 path = '\\'.join(paths)
42 return path
43
44
maruel@chromium.org3c55d982010-05-06 14:25:44 +000045def GenFakeDiff(filename):
46 """Generates a fake diff from a file."""
47 file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True)
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +000048 filename = filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +000049 nb_lines = len(file_content)
50 # We need to use / since patch on unix will fail otherwise.
51 data = cStringIO.StringIO()
52 data.write("Index: %s\n" % filename)
53 data.write('=' * 67 + '\n')
54 # Note: Should we use /dev/null instead?
55 data.write("--- %s\n" % filename)
56 data.write("+++ %s\n" % filename)
57 data.write("@@ -0,0 +1,%d @@\n" % nb_lines)
58 # Prepend '+' to every lines.
59 for line in file_content:
60 data.write('+')
61 data.write(line)
62 result = data.getvalue()
63 data.close()
64 return result
65
66
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000067class GIT(object):
68 COMMAND = "git"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000069
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000070 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000071 def Capture(args, in_directory=None, print_error=True, error_ok=False):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000072 """Runs git, capturing output sent to stdout as a string.
73
74 Args:
75 args: A sequence of command line parameters to be passed to git.
76 in_directory: The directory where git is to be run.
77
78 Returns:
79 The output sent to stdout as a string.
80 """
81 c = [GIT.COMMAND]
82 c.extend(args)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000083 try:
84 return gclient_utils.CheckCall(c, in_directory, print_error)
85 except gclient_utils.CheckCallError:
86 if error_ok:
nasser@codeaurora.orgcd968c12010-02-01 06:05:00 +000087 return ('', '')
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000088 raise
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000089
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000090 @staticmethod
msb@chromium.org786fb682010-06-02 15:16:23 +000091 def CaptureStatus(files, upstream_branch=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000092 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000093
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000094 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000095
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000096 Returns an array of (status, file) tuples."""
msb@chromium.org786fb682010-06-02 15:16:23 +000097 if upstream_branch is None:
98 upstream_branch = GIT.GetUpstreamBranch(os.getcwd())
99 if upstream_branch is None:
100 raise Exception("Cannot determine upstream branch")
bauerb@chromium.org14ec5042010-03-30 18:19:09 +0000101 command = ["diff", "--name-status", "-r", "%s..." % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000102 if not files:
103 pass
104 elif isinstance(files, basestring):
105 command.append(files)
106 else:
107 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000108
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000109 status = GIT.Capture(command)[0].rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000110 results = []
111 if status:
112 for statusline in status.split('\n'):
113 m = re.match('^(\w)\t(.+)$', statusline)
114 if not m:
115 raise Exception("status currently unsupported: %s" % statusline)
116 results.append(('%s ' % m.group(1), m.group(2)))
117 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000118
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000119 @staticmethod
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000120 def RunAndFilterOutput(args,
121 in_directory,
122 print_messages,
123 print_stdout,
124 filter):
125 """Runs a command, optionally outputting to stdout.
126
127 stdout is passed line-by-line to the given filter function. If
128 print_stdout is true, it is also printed to sys.stdout as in Run.
129
130 Args:
131 args: A sequence of command line parameters to be passed.
msb@chromium.orgd6504212010-01-13 17:34:31 +0000132 in_directory: The directory where git is to be run.
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000133 print_messages: Whether to print status messages to stdout about
134 which commands are being run.
135 print_stdout: Whether to forward program's output to stdout.
136 filter: A function taking one argument (a string) which will be
137 passed each line (with the ending newline character removed) of
138 program's output for filtering.
139
140 Raises:
141 gclient_utils.Error: An error occurred while running the command.
142 """
143 command = [GIT.COMMAND]
144 command.extend(args)
145 gclient_utils.SubprocessCallAndFilter(command,
146 in_directory,
147 print_messages,
148 print_stdout,
149 filter=filter)
150
151 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000152 def GetEmail(repo_root):
153 """Retrieves the user email address if known."""
154 # We could want to look at the svn cred when it has a svn remote but it
155 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000156 return GIT.Capture(['config', 'user.email'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000157 repo_root, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000158
159 @staticmethod
160 def ShortBranchName(branch):
161 """Converts a name like 'refs/heads/foo' to just 'foo'."""
162 return branch.replace('refs/heads/', '')
163
164 @staticmethod
165 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000166 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000167 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000168
169 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000170 def GetBranch(cwd):
171 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000172 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000173
174 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000175 def IsGitSvn(cwd):
176 """Returns true if this repo looks like it's using git-svn."""
177 # If you have any "svn-remote.*" config keys, we think you're using svn.
178 try:
179 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd)
180 return True
181 except gclient_utils.CheckCallError:
182 return False
183
184 @staticmethod
185 def GetSVNBranch(cwd):
186 """Returns the svn branch name if found."""
187 # Try to figure out which remote branch we're based on.
188 # Strategy:
189 # 1) find all git-svn branches and note their svn URLs.
190 # 2) iterate through our branch history and match up the URLs.
191
192 # regexp matching the git-svn line that contains the URL.
193 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
194
195 # Get the refname and svn url for all refs/remotes/*.
196 remotes = GIT.Capture(
197 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000198 cwd)[0].splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000199 svn_refs = {}
200 for ref in remotes:
201 match = git_svn_re.search(
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000202 GIT.Capture(['cat-file', '-p', ref], cwd)[0])
sky@chromium.org42d8da52010-04-23 18:25:07 +0000203 # Prefer origin/HEAD over all others.
204 if match and (match.group(1) not in svn_refs or
205 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000206 svn_refs[match.group(1)] = ref
207
208 svn_branch = ''
209 if len(svn_refs) == 1:
210 # Only one svn branch exists -- seems like a good candidate.
211 svn_branch = svn_refs.values()[0]
212 elif len(svn_refs) > 1:
213 # We have more than one remote branch available. We don't
214 # want to go through all of history, so read a line from the
215 # pipe at a time.
216 # The -100 is an arbitrary limit so we don't search forever.
217 cmd = ['git', 'log', '-100', '--pretty=medium']
218 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
219 for line in proc.stdout:
220 match = git_svn_re.match(line)
221 if match:
222 url = match.group(1)
223 if url in svn_refs:
224 svn_branch = svn_refs[url]
225 proc.stdout.close() # Cut pipe.
226 break
227 return svn_branch
228
229 @staticmethod
230 def FetchUpstreamTuple(cwd):
231 """Returns a tuple containg remote and remote ref,
232 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000233 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000234 """
235 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000236 branch = GIT.GetBranch(cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000237 upstream_branch = None
238 upstream_branch = GIT.Capture(
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000239 ['config', 'branch.%s.merge' % branch], in_directory=cwd,
240 error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000241 if upstream_branch:
242 remote = GIT.Capture(
243 ['config', 'branch.%s.remote' % branch],
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000244 in_directory=cwd, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000245 else:
246 # Fall back on trying a git-svn upstream branch.
247 if GIT.IsGitSvn(cwd):
248 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000249 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000250 # Else, try to guess the origin remote.
251 remote_branches = GIT.Capture(
252 ['branch', '-r'], in_directory=cwd)[0].split()
253 if 'origin/master' in remote_branches:
254 # Fall back on origin/master if it exits.
255 remote = 'origin'
256 upstream_branch = 'refs/heads/master'
257 elif 'origin/trunk' in remote_branches:
258 # Fall back on origin/trunk if it exists. Generally a shared
259 # git-svn clone
260 remote = 'origin'
261 upstream_branch = 'refs/heads/trunk'
262 else:
263 # Give up.
264 remote = None
265 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000266 return remote, upstream_branch
267
268 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000269 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000270 """Gets the current branch's upstream branch."""
271 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000272 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000273 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
274 return upstream_branch
275
276 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000277 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
278 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000279 """Diffs against the upstream branch or optionally another branch.
280
281 full_move means that move or copy operations should completely recreate the
282 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000283 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000284 branch = GIT.GetUpstreamBranch(cwd)
evan@chromium.org400f3e72010-05-19 14:23:36 +0000285 command = ['diff', '-p', '--no-prefix', '--no-ext-diff',
286 branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000287 if not full_move:
288 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000289 # TODO(maruel): --binary support.
290 if files:
291 command.append('--')
292 command.extend(files)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000293 diff = GIT.Capture(command, cwd)[0].splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000294 for i in range(len(diff)):
295 # In the case of added files, replace /dev/null with the path to the
296 # file being added.
297 if diff[i].startswith('--- /dev/null'):
298 diff[i] = '--- %s' % diff[i+1][4:]
299 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000300
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000301 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000302 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
303 """Returns the list of modified files between two branches."""
304 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000305 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000306 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000307 return GIT.Capture(command, cwd)[0].splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000308
309 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000310 def GetPatchName(cwd):
311 """Constructs a name for this patch."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000312 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000313 return "%s-%s" % (GIT.GetBranch(cwd), short_sha)
314
315 @staticmethod
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000316 def GetCheckoutRoot(path):
317 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000318 """
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000319 root = GIT.Capture(['rev-parse', '--show-cdup'], path)[0].strip()
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000320 return os.path.abspath(os.path.join(path, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000321
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000322 @staticmethod
323 def AssertVersion(min_version):
324 """Asserts git's version is at least min_version."""
325 def only_int(val):
326 if val.isdigit():
327 return int(val)
328 else:
329 return 0
330 current_version = GIT.Capture(['--version'])[0].split()[-1]
331 current_version_list = map(only_int, current_version.split('.'))
332 for min_ver in map(int, min_version.split('.')):
333 ver = current_version_list.pop(0)
334 if ver < min_ver:
335 return (False, current_version)
336 elif ver > min_ver:
337 return (True, current_version)
338 return (True, current_version)
339
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000340
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000341class SVN(object):
342 COMMAND = "svn"
tony@chromium.org57564662010-04-14 02:35:12 +0000343 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000344
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000345 @staticmethod
346 def Run(args, in_directory):
347 """Runs svn, sending output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000348
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000349 Args:
350 args: A sequence of command line parameters to be passed to svn.
351 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000352
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000353 Raises:
354 Error: An error occurred while running the svn command.
355 """
356 c = [SVN.COMMAND]
357 c.extend(args)
maruel@chromium.org2185f002009-12-18 21:03:47 +0000358 # TODO(maruel): This is very gclient-specific.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000359 gclient_utils.SubprocessCall(c, in_directory)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000360
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000361 @staticmethod
362 def Capture(args, in_directory=None, print_error=True):
363 """Runs svn, capturing output sent to stdout as a string.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000364
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000365 Args:
366 args: A sequence of command line parameters to be passed to svn.
367 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000368
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000369 Returns:
370 The output sent to stdout as a string.
371 """
372 c = [SVN.COMMAND]
373 c.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000374
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000375 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
376 # the svn.exe executable, but shell=True makes subprocess on Linux fail
377 # when it's called with a list because it only tries to execute the
378 # first string ("svn").
379 stderr = None
380 if not print_error:
381 stderr = subprocess.PIPE
382 return subprocess.Popen(c,
383 cwd=in_directory,
384 shell=(sys.platform == 'win32'),
385 stdout=subprocess.PIPE,
386 stderr=stderr).communicate()[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000387
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000388 @staticmethod
389 def RunAndGetFileList(options, args, in_directory, file_list):
390 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000391
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000392 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000393
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000394 svn's stdout is parsed to collect a list of files checked out or updated.
395 These files are appended to file_list. svn's stdout is also printed to
396 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000397
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000398 Args:
399 options: command line options to gclient
400 args: A sequence of command line parameters to be passed to svn.
401 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000402
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000403 Raises:
404 Error: An error occurred while running the svn command.
405 """
406 command = [SVN.COMMAND]
407 command.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000408
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000409 # svn update and svn checkout use the same pattern: the first three columns
410 # are for file status, property status, and lock status. This is followed
411 # by two spaces, and then the path to the file.
412 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000413
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000414 # The first three columns of svn status are the same as for svn update and
415 # svn checkout. The next three columns indicate addition-with-history,
416 # switch, and remote lock status. This is followed by one space, and then
417 # the path to the file.
418 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000419
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000420 # args[0] must be a supported command. This will blow up if it's something
421 # else, which is good. Note that the patterns are only effective when
422 # these commands are used in their ordinary forms, the patterns are invalid
423 # for "svn status --show-updates", for example.
424 pattern = {
425 'checkout': update_pattern,
426 'status': status_pattern,
427 'update': update_pattern,
428 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000429 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000430 # Place an upper limit.
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000431 for _ in range(10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000432 previous_list_len = len(file_list)
433 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000434
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000435 def CaptureMatchingLines(line):
436 match = compiled_pattern.search(line)
437 if match:
438 file_list.append(match.group(1))
439 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000440 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000441
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000442 try:
443 SVN.RunAndFilterOutput(args,
444 in_directory,
445 options.verbose,
446 True,
447 CaptureMatchingLines)
448 except gclient_utils.Error:
maruel@chromium.org2de10252010-02-08 01:10:39 +0000449 # We enforce that some progress has been made or HTTP 502.
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000450 if (filter(lambda x: '502 Bad Gateway' in x, failure) or
maruel@chromium.org2de10252010-02-08 01:10:39 +0000451 (len(failure) and len(file_list) > previous_list_len)):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000452 if args[0] == 'checkout':
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000453 # An aborted checkout is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000454 args = ['update'] + args[1:]
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000455 print "Sleeping 15 seconds and retrying...."
456 time.sleep(15)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000457 continue
maruel@chromium.org2de10252010-02-08 01:10:39 +0000458 # No progress was made or an unknown error we aren't sure, bail out.
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000459 raise
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000460 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000461
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000462 @staticmethod
463 def RunAndFilterOutput(args,
464 in_directory,
465 print_messages,
466 print_stdout,
467 filter):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000468 """Runs a command, optionally outputting to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000469
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000470 stdout is passed line-by-line to the given filter function. If
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000471 print_stdout is true, it is also printed to sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000472
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000473 Args:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000474 args: A sequence of command line parameters to be passed.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000475 in_directory: The directory where svn is to be run.
476 print_messages: Whether to print status messages to stdout about
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000477 which commands are being run.
478 print_stdout: Whether to forward program's output to stdout.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000479 filter: A function taking one argument (a string) which will be
480 passed each line (with the ending newline character removed) of
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000481 program's output for filtering.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000482
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000483 Raises:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000484 gclient_utils.Error: An error occurred while running the command.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000485 """
486 command = [SVN.COMMAND]
487 command.extend(args)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000488 gclient_utils.SubprocessCallAndFilter(command,
489 in_directory,
490 print_messages,
491 print_stdout,
492 filter=filter)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000493
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000494 @staticmethod
495 def CaptureInfo(relpath, in_directory=None, print_error=True):
496 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000497
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000498 Args:
499 relpath: The directory where the working copy resides relative to
500 the directory given by in_directory.
501 in_directory: The directory where svn is to be run.
502 """
503 output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
504 dom = gclient_utils.ParseXML(output)
505 result = {}
506 if dom:
507 GetNamedNodeText = gclient_utils.GetNamedNodeText
508 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
509 def C(item, f):
510 if item is not None: return f(item)
511 # /info/entry/
512 # url
513 # reposityory/(root|uuid)
514 # wc-info/(schedule|depth)
515 # commit/(author|date)
516 # str() the results because they may be returned as Unicode, which
517 # interferes with the higher layers matching up things in the deps
518 # dictionary.
519 # TODO(maruel): Fix at higher level instead (!)
520 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
521 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
522 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
523 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
524 'revision'),
525 int)
526 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
527 str)
528 # Differs across versions.
529 if result['Node Kind'] == 'dir':
530 result['Node Kind'] = 'directory'
531 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
532 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
533 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
534 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
535 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000536
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000537 @staticmethod
538 def CaptureHeadRevision(url):
539 """Get the head revision of a SVN repository.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000540
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000541 Returns:
542 Int head revision
543 """
544 info = SVN.Capture(["info", "--xml", url], os.getcwd())
545 dom = xml.dom.minidom.parseString(info)
546 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000547
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000548 @staticmethod
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000549 def CaptureBaseRevision(cwd):
550 """Get the base revision of a SVN repository.
551
552 Returns:
553 Int base revision
554 """
555 info = SVN.Capture(["info", "--xml"], cwd)
556 dom = xml.dom.minidom.parseString(info)
557 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
558
559 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000560 def CaptureStatus(files):
561 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000562
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000563 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000564
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000565 Returns an array of (status, file) tuples."""
566 command = ["status", "--xml"]
567 if not files:
568 pass
569 elif isinstance(files, basestring):
570 command.append(files)
571 else:
572 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000573
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000574 status_letter = {
575 None: ' ',
576 '': ' ',
577 'added': 'A',
578 'conflicted': 'C',
579 'deleted': 'D',
580 'external': 'X',
581 'ignored': 'I',
582 'incomplete': '!',
583 'merged': 'G',
584 'missing': '!',
585 'modified': 'M',
586 'none': ' ',
587 'normal': ' ',
588 'obstructed': '~',
589 'replaced': 'R',
590 'unversioned': '?',
591 }
592 dom = gclient_utils.ParseXML(SVN.Capture(command))
593 results = []
594 if dom:
595 # /status/target/entry/(wc-status|commit|author|date)
596 for target in dom.getElementsByTagName('target'):
597 #base_path = target.getAttribute('path')
598 for entry in target.getElementsByTagName('entry'):
599 file_path = entry.getAttribute('path')
600 wc_status = entry.getElementsByTagName('wc-status')
601 assert len(wc_status) == 1
602 # Emulate svn 1.5 status ouput...
603 statuses = [' '] * 7
604 # Col 0
605 xml_item_status = wc_status[0].getAttribute('item')
606 if xml_item_status in status_letter:
607 statuses[0] = status_letter[xml_item_status]
608 else:
609 raise Exception('Unknown item status "%s"; please implement me!' %
610 xml_item_status)
611 # Col 1
612 xml_props_status = wc_status[0].getAttribute('props')
613 if xml_props_status == 'modified':
614 statuses[1] = 'M'
615 elif xml_props_status == 'conflicted':
616 statuses[1] = 'C'
617 elif (not xml_props_status or xml_props_status == 'none' or
618 xml_props_status == 'normal'):
619 pass
620 else:
621 raise Exception('Unknown props status "%s"; please implement me!' %
622 xml_props_status)
623 # Col 2
624 if wc_status[0].getAttribute('wc-locked') == 'true':
625 statuses[2] = 'L'
626 # Col 3
627 if wc_status[0].getAttribute('copied') == 'true':
628 statuses[3] = '+'
629 # Col 4
630 if wc_status[0].getAttribute('switched') == 'true':
631 statuses[4] = 'S'
632 # TODO(maruel): Col 5 and 6
633 item = (''.join(statuses), file_path)
634 results.append(item)
635 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000636
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000637 @staticmethod
638 def IsMoved(filename):
639 """Determine if a file has been added through svn mv"""
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000640 return SVN.IsMovedInfo(SVN.CaptureInfo(filename))
641
642 @staticmethod
643 def IsMovedInfo(info):
644 """Determine if a file has been added through svn mv"""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000645 return (info.get('Copied From URL') and
646 info.get('Copied From Rev') and
647 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000648
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000649 @staticmethod
650 def GetFileProperty(file, property_name):
651 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000652
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000653 Args:
654 file: The file to check
655 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000656
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000657 Returns:
658 The value of the property, which will be the empty string if the property
659 is not set on the file. If the file is not under version control, the
660 empty string is also returned.
661 """
662 output = SVN.Capture(["propget", property_name, file])
663 if (output.startswith("svn: ") and
664 output.endswith("is not under version control")):
665 return ""
666 else:
667 return output
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000668
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000669 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000670 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000671 """Diffs a single file.
672
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000673 Should be simple, eh? No it isn't.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000674 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000675 expected relative path.
676 full_move means that move or copy operations should completely recreate the
677 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000678 # If the user specified a custom diff command in their svn config file,
679 # then it'll be used when we do svn diff, which we don't want to happen
680 # since we want the unified diff. Using --diff-cmd=diff doesn't always
681 # work, since they can have another diff executable in their path that
682 # gives different line endings. So we use a bogus temp directory as the
683 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000684 bogus_dir = tempfile.mkdtemp()
685 try:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000686 # Use "svn info" output instead of os.path.isdir because the latter fails
687 # when the file is deleted.
688 return SVN._DiffItemInternal(SVN.CaptureInfo(filename),
689 full_move=full_move, revision=revision)
690 finally:
691 shutil.rmtree(bogus_dir)
692
693 @staticmethod
694 def _DiffItemInternal(filename, info, bogus_dir, full_move=False,
695 revision=None):
696 """Grabs the diff data."""
697 command = ["diff", "--config-dir", bogus_dir, filename]
698 if revision:
699 command.extend(['--revision', revision])
700 data = None
701 if SVN.IsMovedInfo(info):
702 if full_move:
703 if info.get("Node Kind") == "directory":
704 # Things become tricky here. It's a directory copy/move. We need to
705 # diff all the files inside it.
706 # This will put a lot of pressure on the heap. This is why StringIO
707 # is used and converted back into a string at the end. The reason to
708 # return a string instead of a StringIO is that StringIO.write()
709 # doesn't accept a StringIO object. *sigh*.
710 for (dirpath, dirnames, filenames) in os.walk(filename):
711 # Cleanup all files starting with a '.'.
712 for d in dirnames:
713 if d.startswith('.'):
714 dirnames.remove(d)
715 for f in filenames:
716 if f.startswith('.'):
717 filenames.remove(f)
718 for f in filenames:
719 if data is None:
720 data = cStringIO.StringIO()
721 data.write(GenFakeDiff(os.path.join(dirpath, f)))
722 if data:
723 tmp = data.getvalue()
724 data.close()
725 data = tmp
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000726 else:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000727 data = GenFakeDiff(filename)
728 else:
729 if info.get("Node Kind") != "directory":
maruel@chromium.org0836c562010-01-22 01:10:06 +0000730 # svn diff on a mv/cp'd file outputs nothing if there was no change.
731 data = SVN.Capture(command, None)
732 if not data:
733 # We put in an empty Index entry so upload.py knows about them.
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +0000734 data = "Index: %s\n" % filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000735 # Otherwise silently ignore directories.
736 else:
737 if info.get("Node Kind") != "directory":
738 # Normal simple case.
maruel@chromium.org0836c562010-01-22 01:10:06 +0000739 data = SVN.Capture(command, None)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000740 # Otherwise silently ignore directories.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000741 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000742
743 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000744 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000745 """Returns a string containing the diff for the given file list.
746
747 The files in the list should either be absolute paths or relative to the
748 given root. If no root directory is provided, the repository root will be
749 used.
750 The diff will always use relative paths.
751 """
752 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000753 root = root or SVN.GetCheckoutRoot(previous_cwd)
754 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000755 def RelativePath(path, root):
756 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000757 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000758 return path[len(root):]
759 return path
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000760 # If the user specified a custom diff command in their svn config file,
761 # then it'll be used when we do svn diff, which we don't want to happen
762 # since we want the unified diff. Using --diff-cmd=diff doesn't always
763 # work, since they can have another diff executable in their path that
764 # gives different line endings. So we use a bogus temp directory as the
765 # config directory, which gets around these problems.
766 bogus_dir = tempfile.mkdtemp()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000767 try:
768 os.chdir(root)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000769 # Cleanup filenames
770 filenames = [RelativePath(f, root) for f in filenames]
771 # Get information about the modified items (files and directories)
772 data = dict([(f, SVN.CaptureInfo(f)) for f in filenames])
773 if full_move:
774 # Eliminate modified files inside moved/copied directory.
775 for (filename, info) in data.iteritems():
776 if SVN.IsMovedInfo(info) and info.get("Node Kind") == "directory":
777 # Remove files inside the directory.
778 filenames = [f for f in filenames
779 if not f.startswith(filename + os.path.sep)]
780 for filename in data.keys():
781 if not filename in filenames:
782 # Remove filtered out items.
783 del data[filename]
784 # Now ready to do the actual diff.
785 diffs = []
786 for filename in sorted(data.iterkeys()):
787 diffs.append(SVN._DiffItemInternal(filename, data[filename], bogus_dir,
788 full_move=full_move,
789 revision=revision))
790 # Use StringIO since it can be messy when diffing a directory move with
791 # full_move=True.
792 buf = cStringIO.StringIO()
793 for d in filter(None, diffs):
794 buf.write(d)
795 result = buf.getvalue()
796 buf.close()
797 return result
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000798 finally:
799 os.chdir(previous_cwd)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000800 shutil.rmtree(bogus_dir)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000801
802 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000803 def GetEmail(repo_root):
804 """Retrieves the svn account which we assume is an email address."""
805 infos = SVN.CaptureInfo(repo_root)
806 uuid = infos.get('UUID')
807 root = infos.get('Repository Root')
808 if not root:
809 return None
810
811 # Should check for uuid but it is incorrectly saved for https creds.
812 realm = root.rsplit('/', 1)[0]
813 if root.startswith('https') or not uuid:
814 regexp = re.compile(r'<%s:\d+>.*' % realm)
815 else:
816 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
817 if regexp is None:
818 return None
819 if sys.platform.startswith('win'):
820 if not 'APPDATA' in os.environ:
821 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000822 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
823 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000824 else:
825 if not 'HOME' in os.environ:
826 return None
827 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
828 'svn.simple')
829 for credfile in os.listdir(auth_dir):
830 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
831 if regexp.match(cred_info.get('svn:realmstring')):
832 return cred_info.get('username')
833
834 @staticmethod
835 def ReadSimpleAuth(filename):
836 f = open(filename, 'r')
837 values = {}
838 def ReadOneItem(type):
839 m = re.match(r'%s (\d+)' % type, f.readline())
840 if not m:
841 return None
842 data = f.read(int(m.group(1)))
843 if f.read(1) != '\n':
844 return None
845 return data
846
847 while True:
848 key = ReadOneItem('K')
849 if not key:
850 break
851 value = ReadOneItem('V')
852 if not value:
853 break
854 values[key] = value
855 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000856
857 @staticmethod
858 def GetCheckoutRoot(directory):
859 """Returns the top level directory of the current repository.
860
861 The directory is returned as an absolute path.
862 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000863 directory = os.path.abspath(directory)
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000864 infos = SVN.CaptureInfo(directory, print_error=False)
865 cur_dir_repo_root = infos.get("Repository Root")
866 if not cur_dir_repo_root:
867 return None
868
869 while True:
870 parent = os.path.dirname(directory)
871 if (SVN.CaptureInfo(parent, print_error=False).get(
872 "Repository Root") != cur_dir_repo_root):
873 break
874 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000875 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000876
877 @staticmethod
878 def AssertVersion(min_version):
879 """Asserts svn's version is at least min_version."""
880 def only_int(val):
881 if val.isdigit():
882 return int(val)
883 else:
884 return 0
885 if not SVN.current_version:
886 SVN.current_version = SVN.Capture(['--version']).split()[2]
887 current_version_list = map(only_int, SVN.current_version.split('.'))
888 for min_ver in map(int, min_version.split('.')):
889 ver = current_version_list.pop(0)
890 if ver < min_ver:
891 return (False, SVN.current_version)
892 elif ver > min_ver:
893 return (True, SVN.current_version)
894 return (True, SVN.current_version)