blob: 11c977a64b0a50ac1cf78a0446a2ee43b7ddfac4 [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.org3c55d982010-05-06 14:25:44 +00007import cStringIO
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00008import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00009import os
10import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000011import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000012import subprocess
13import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000014import tempfile
maruel@chromium.orgfd876172010-04-30 14:01:05 +000015import time
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000016import xml.dom.minidom
17
18import gclient_utils
19
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000020def ValidateEmail(email):
21 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
22 is not None)
23
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000024
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000025def GetCasedPath(path):
26 """Elcheapos way to get the real path case on Windows."""
27 if sys.platform.startswith('win') and os.path.exists(path):
28 # Reconstruct the path.
29 path = os.path.abspath(path)
30 paths = path.split('\\')
31 for i in range(len(paths)):
32 if i == 0:
33 # Skip drive letter.
34 continue
35 subpath = '\\'.join(paths[:i+1])
36 prev = len('\\'.join(paths[:i]))
37 # glob.glob will return the cased path for the last item only. This is why
38 # we are calling it in a loop. Extract the data we want and put it back
39 # into the list.
40 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
41 path = '\\'.join(paths)
42 return path
43
44
maruel@chromium.org3c55d982010-05-06 14:25:44 +000045def GenFakeDiff(filename):
46 """Generates a fake diff from a file."""
47 file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True)
48 nb_lines = len(file_content)
49 # We need to use / since patch on unix will fail otherwise.
50 data = cStringIO.StringIO()
51 data.write("Index: %s\n" % filename)
52 data.write('=' * 67 + '\n')
53 # Note: Should we use /dev/null instead?
54 data.write("--- %s\n" % filename)
55 data.write("+++ %s\n" % filename)
56 data.write("@@ -0,0 +1,%d @@\n" % nb_lines)
57 # Prepend '+' to every lines.
58 for line in file_content:
59 data.write('+')
60 data.write(line)
61 result = data.getvalue()
62 data.close()
63 return result
64
65
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000066class GIT(object):
67 COMMAND = "git"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000068
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000069 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000070 def Capture(args, in_directory=None, print_error=True, error_ok=False):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000071 """Runs git, capturing output sent to stdout as a string.
72
73 Args:
74 args: A sequence of command line parameters to be passed to git.
75 in_directory: The directory where git is to be run.
76
77 Returns:
78 The output sent to stdout as a string.
79 """
80 c = [GIT.COMMAND]
81 c.extend(args)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000082 try:
83 return gclient_utils.CheckCall(c, in_directory, print_error)
84 except gclient_utils.CheckCallError:
85 if error_ok:
nasser@codeaurora.orgcd968c12010-02-01 06:05:00 +000086 return ('', '')
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000087 raise
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000088
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000089 @staticmethod
msb@chromium.org786fb682010-06-02 15:16:23 +000090 def CaptureStatus(files, upstream_branch=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000091 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000092
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000093 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000094
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000095 Returns an array of (status, file) tuples."""
msb@chromium.org786fb682010-06-02 15:16:23 +000096 if upstream_branch is None:
97 upstream_branch = GIT.GetUpstreamBranch(os.getcwd())
98 if upstream_branch is None:
99 raise Exception("Cannot determine upstream branch")
bauerb@chromium.org14ec5042010-03-30 18:19:09 +0000100 command = ["diff", "--name-status", "-r", "%s..." % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000101 if not files:
102 pass
103 elif isinstance(files, basestring):
104 command.append(files)
105 else:
106 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000107
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000108 status = GIT.Capture(command)[0].rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000109 results = []
110 if status:
111 for statusline in status.split('\n'):
112 m = re.match('^(\w)\t(.+)$', statusline)
113 if not m:
114 raise Exception("status currently unsupported: %s" % statusline)
115 results.append(('%s ' % m.group(1), m.group(2)))
116 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000117
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000118 @staticmethod
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000119 def RunAndFilterOutput(args,
120 in_directory,
121 print_messages,
122 print_stdout,
123 filter):
124 """Runs a command, optionally outputting to stdout.
125
126 stdout is passed line-by-line to the given filter function. If
127 print_stdout is true, it is also printed to sys.stdout as in Run.
128
129 Args:
130 args: A sequence of command line parameters to be passed.
msb@chromium.orgd6504212010-01-13 17:34:31 +0000131 in_directory: The directory where git is to be run.
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000132 print_messages: Whether to print status messages to stdout about
133 which commands are being run.
134 print_stdout: Whether to forward program's output to stdout.
135 filter: A function taking one argument (a string) which will be
136 passed each line (with the ending newline character removed) of
137 program's output for filtering.
138
139 Raises:
140 gclient_utils.Error: An error occurred while running the command.
141 """
142 command = [GIT.COMMAND]
143 command.extend(args)
144 gclient_utils.SubprocessCallAndFilter(command,
145 in_directory,
146 print_messages,
147 print_stdout,
148 filter=filter)
149
150 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000151 def GetEmail(repo_root):
152 """Retrieves the user email address if known."""
153 # We could want to look at the svn cred when it has a svn remote but it
154 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000155 return GIT.Capture(['config', 'user.email'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000156 repo_root, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000157
158 @staticmethod
159 def ShortBranchName(branch):
160 """Converts a name like 'refs/heads/foo' to just 'foo'."""
161 return branch.replace('refs/heads/', '')
162
163 @staticmethod
164 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000165 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000166 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000167
168 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000169 def GetBranch(cwd):
170 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000171 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000172
173 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000174 def IsGitSvn(cwd):
175 """Returns true if this repo looks like it's using git-svn."""
176 # If you have any "svn-remote.*" config keys, we think you're using svn.
177 try:
178 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd)
179 return True
180 except gclient_utils.CheckCallError:
181 return False
182
183 @staticmethod
184 def GetSVNBranch(cwd):
185 """Returns the svn branch name if found."""
186 # Try to figure out which remote branch we're based on.
187 # Strategy:
188 # 1) find all git-svn branches and note their svn URLs.
189 # 2) iterate through our branch history and match up the URLs.
190
191 # regexp matching the git-svn line that contains the URL.
192 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
193
194 # Get the refname and svn url for all refs/remotes/*.
195 remotes = GIT.Capture(
196 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000197 cwd)[0].splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000198 svn_refs = {}
199 for ref in remotes:
200 match = git_svn_re.search(
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000201 GIT.Capture(['cat-file', '-p', ref], cwd)[0])
sky@chromium.org42d8da52010-04-23 18:25:07 +0000202 # Prefer origin/HEAD over all others.
203 if match and (match.group(1) not in svn_refs or
204 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000205 svn_refs[match.group(1)] = ref
206
207 svn_branch = ''
208 if len(svn_refs) == 1:
209 # Only one svn branch exists -- seems like a good candidate.
210 svn_branch = svn_refs.values()[0]
211 elif len(svn_refs) > 1:
212 # We have more than one remote branch available. We don't
213 # want to go through all of history, so read a line from the
214 # pipe at a time.
215 # The -100 is an arbitrary limit so we don't search forever.
216 cmd = ['git', 'log', '-100', '--pretty=medium']
217 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
218 for line in proc.stdout:
219 match = git_svn_re.match(line)
220 if match:
221 url = match.group(1)
222 if url in svn_refs:
223 svn_branch = svn_refs[url]
224 proc.stdout.close() # Cut pipe.
225 break
226 return svn_branch
227
228 @staticmethod
229 def FetchUpstreamTuple(cwd):
230 """Returns a tuple containg remote and remote ref,
231 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000232 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000233 """
234 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000235 branch = GIT.GetBranch(cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000236 upstream_branch = None
237 upstream_branch = GIT.Capture(
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000238 ['config', 'branch.%s.merge' % branch], in_directory=cwd,
239 error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000240 if upstream_branch:
241 remote = GIT.Capture(
242 ['config', 'branch.%s.remote' % branch],
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000243 in_directory=cwd, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000244 else:
245 # Fall back on trying a git-svn upstream branch.
246 if GIT.IsGitSvn(cwd):
247 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000248 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000249 # Else, try to guess the origin remote.
250 remote_branches = GIT.Capture(
251 ['branch', '-r'], in_directory=cwd)[0].split()
252 if 'origin/master' in remote_branches:
253 # Fall back on origin/master if it exits.
254 remote = 'origin'
255 upstream_branch = 'refs/heads/master'
256 elif 'origin/trunk' in remote_branches:
257 # Fall back on origin/trunk if it exists. Generally a shared
258 # git-svn clone
259 remote = 'origin'
260 upstream_branch = 'refs/heads/trunk'
261 else:
262 # Give up.
263 remote = None
264 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000265 return remote, upstream_branch
266
267 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000268 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000269 """Gets the current branch's upstream branch."""
270 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000271 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000272 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
273 return upstream_branch
274
275 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000276 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
277 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000278 """Diffs against the upstream branch or optionally another branch.
279
280 full_move means that move or copy operations should completely recreate the
281 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000282 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000283 branch = GIT.GetUpstreamBranch(cwd)
evan@chromium.org400f3e72010-05-19 14:23:36 +0000284 command = ['diff', '-p', '--no-prefix', '--no-ext-diff',
285 branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000286 if not full_move:
287 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000288 # TODO(maruel): --binary support.
289 if files:
290 command.append('--')
291 command.extend(files)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000292 diff = GIT.Capture(command, cwd)[0].splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000293 for i in range(len(diff)):
294 # In the case of added files, replace /dev/null with the path to the
295 # file being added.
296 if diff[i].startswith('--- /dev/null'):
297 diff[i] = '--- %s' % diff[i+1][4:]
298 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000299
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000300 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000301 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
302 """Returns the list of modified files between two branches."""
303 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000304 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000305 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000306 return GIT.Capture(command, cwd)[0].splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000307
308 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000309 def GetPatchName(cwd):
310 """Constructs a name for this patch."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000311 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000312 return "%s-%s" % (GIT.GetBranch(cwd), short_sha)
313
314 @staticmethod
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000315 def GetCheckoutRoot(path):
316 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000317 """
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000318 root = GIT.Capture(['rev-parse', '--show-cdup'], path)[0].strip()
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000319 return os.path.abspath(os.path.join(path, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000320
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000321 @staticmethod
322 def AssertVersion(min_version):
323 """Asserts git's version is at least min_version."""
324 def only_int(val):
325 if val.isdigit():
326 return int(val)
327 else:
328 return 0
329 current_version = GIT.Capture(['--version'])[0].split()[-1]
330 current_version_list = map(only_int, current_version.split('.'))
331 for min_ver in map(int, min_version.split('.')):
332 ver = current_version_list.pop(0)
333 if ver < min_ver:
334 return (False, current_version)
335 elif ver > min_ver:
336 return (True, current_version)
337 return (True, current_version)
338
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000339
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000340class SVN(object):
341 COMMAND = "svn"
tony@chromium.org57564662010-04-14 02:35:12 +0000342 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000343
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000344 @staticmethod
345 def Run(args, in_directory):
346 """Runs svn, sending output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000347
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000348 Args:
349 args: A sequence of command line parameters to be passed to svn.
350 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000351
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000352 Raises:
353 Error: An error occurred while running the svn command.
354 """
355 c = [SVN.COMMAND]
356 c.extend(args)
maruel@chromium.org2185f002009-12-18 21:03:47 +0000357 # TODO(maruel): This is very gclient-specific.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000358 gclient_utils.SubprocessCall(c, in_directory)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000359
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000360 @staticmethod
361 def Capture(args, in_directory=None, print_error=True):
362 """Runs svn, capturing output sent to stdout as a string.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000363
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000364 Args:
365 args: A sequence of command line parameters to be passed to svn.
366 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000367
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000368 Returns:
369 The output sent to stdout as a string.
370 """
371 c = [SVN.COMMAND]
372 c.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000373
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000374 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
375 # the svn.exe executable, but shell=True makes subprocess on Linux fail
376 # when it's called with a list because it only tries to execute the
377 # first string ("svn").
378 stderr = None
379 if not print_error:
380 stderr = subprocess.PIPE
381 return subprocess.Popen(c,
382 cwd=in_directory,
383 shell=(sys.platform == 'win32'),
384 stdout=subprocess.PIPE,
385 stderr=stderr).communicate()[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000386
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000387 @staticmethod
388 def RunAndGetFileList(options, args, in_directory, file_list):
389 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000390
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000391 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000392
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000393 svn's stdout is parsed to collect a list of files checked out or updated.
394 These files are appended to file_list. svn's stdout is also printed to
395 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000396
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000397 Args:
398 options: command line options to gclient
399 args: A sequence of command line parameters to be passed to svn.
400 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000401
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000402 Raises:
403 Error: An error occurred while running the svn command.
404 """
405 command = [SVN.COMMAND]
406 command.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000407
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000408 # svn update and svn checkout use the same pattern: the first three columns
409 # are for file status, property status, and lock status. This is followed
410 # by two spaces, and then the path to the file.
411 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000412
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000413 # The first three columns of svn status are the same as for svn update and
414 # svn checkout. The next three columns indicate addition-with-history,
415 # switch, and remote lock status. This is followed by one space, and then
416 # the path to the file.
417 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000418
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000419 # args[0] must be a supported command. This will blow up if it's something
420 # else, which is good. Note that the patterns are only effective when
421 # these commands are used in their ordinary forms, the patterns are invalid
422 # for "svn status --show-updates", for example.
423 pattern = {
424 'checkout': update_pattern,
425 'status': status_pattern,
426 'update': update_pattern,
427 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000428 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000429 # Place an upper limit.
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000430 for _ in range(10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000431 previous_list_len = len(file_list)
432 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000433
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000434 def CaptureMatchingLines(line):
435 match = compiled_pattern.search(line)
436 if match:
437 file_list.append(match.group(1))
438 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000439 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000440
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000441 try:
442 SVN.RunAndFilterOutput(args,
443 in_directory,
444 options.verbose,
445 True,
446 CaptureMatchingLines)
447 except gclient_utils.Error:
maruel@chromium.org2de10252010-02-08 01:10:39 +0000448 # We enforce that some progress has been made or HTTP 502.
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000449 if (filter(lambda x: '502 Bad Gateway' in x, failure) or
maruel@chromium.org2de10252010-02-08 01:10:39 +0000450 (len(failure) and len(file_list) > previous_list_len)):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000451 if args[0] == 'checkout':
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000452 # An aborted checkout is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000453 args = ['update'] + args[1:]
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000454 print "Sleeping 15 seconds and retrying...."
455 time.sleep(15)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000456 continue
maruel@chromium.org2de10252010-02-08 01:10:39 +0000457 # No progress was made or an unknown error we aren't sure, bail out.
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000458 raise
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000459 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000460
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000461 @staticmethod
462 def RunAndFilterOutput(args,
463 in_directory,
464 print_messages,
465 print_stdout,
466 filter):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000467 """Runs a command, optionally outputting to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000468
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000469 stdout is passed line-by-line to the given filter function. If
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000470 print_stdout is true, it is also printed to sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000471
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000472 Args:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000473 args: A sequence of command line parameters to be passed.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000474 in_directory: The directory where svn is to be run.
475 print_messages: Whether to print status messages to stdout about
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000476 which commands are being run.
477 print_stdout: Whether to forward program's output to stdout.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000478 filter: A function taking one argument (a string) which will be
479 passed each line (with the ending newline character removed) of
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000480 program's output for filtering.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000481
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000482 Raises:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000483 gclient_utils.Error: An error occurred while running the command.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000484 """
485 command = [SVN.COMMAND]
486 command.extend(args)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000487 gclient_utils.SubprocessCallAndFilter(command,
488 in_directory,
489 print_messages,
490 print_stdout,
491 filter=filter)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000492
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000493 @staticmethod
494 def CaptureInfo(relpath, in_directory=None, print_error=True):
495 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000496
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000497 Args:
498 relpath: The directory where the working copy resides relative to
499 the directory given by in_directory.
500 in_directory: The directory where svn is to be run.
501 """
502 output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
503 dom = gclient_utils.ParseXML(output)
504 result = {}
505 if dom:
506 GetNamedNodeText = gclient_utils.GetNamedNodeText
507 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
508 def C(item, f):
509 if item is not None: return f(item)
510 # /info/entry/
511 # url
512 # reposityory/(root|uuid)
513 # wc-info/(schedule|depth)
514 # commit/(author|date)
515 # str() the results because they may be returned as Unicode, which
516 # interferes with the higher layers matching up things in the deps
517 # dictionary.
518 # TODO(maruel): Fix at higher level instead (!)
519 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
520 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
521 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
522 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
523 'revision'),
524 int)
525 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
526 str)
527 # Differs across versions.
528 if result['Node Kind'] == 'dir':
529 result['Node Kind'] = 'directory'
530 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
531 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
532 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
533 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
534 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000535
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000536 @staticmethod
537 def CaptureHeadRevision(url):
538 """Get the head revision of a SVN repository.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000539
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000540 Returns:
541 Int head revision
542 """
543 info = SVN.Capture(["info", "--xml", url], os.getcwd())
544 dom = xml.dom.minidom.parseString(info)
545 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000546
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000547 @staticmethod
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000548 def CaptureBaseRevision(cwd):
549 """Get the base revision of a SVN repository.
550
551 Returns:
552 Int base revision
553 """
554 info = SVN.Capture(["info", "--xml"], cwd)
555 dom = xml.dom.minidom.parseString(info)
556 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
557
558 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000559 def CaptureStatus(files):
560 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000561
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000562 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000563
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000564 Returns an array of (status, file) tuples."""
565 command = ["status", "--xml"]
566 if not files:
567 pass
568 elif isinstance(files, basestring):
569 command.append(files)
570 else:
571 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000572
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000573 status_letter = {
574 None: ' ',
575 '': ' ',
576 'added': 'A',
577 'conflicted': 'C',
578 'deleted': 'D',
579 'external': 'X',
580 'ignored': 'I',
581 'incomplete': '!',
582 'merged': 'G',
583 'missing': '!',
584 'modified': 'M',
585 'none': ' ',
586 'normal': ' ',
587 'obstructed': '~',
588 'replaced': 'R',
589 'unversioned': '?',
590 }
591 dom = gclient_utils.ParseXML(SVN.Capture(command))
592 results = []
593 if dom:
594 # /status/target/entry/(wc-status|commit|author|date)
595 for target in dom.getElementsByTagName('target'):
596 #base_path = target.getAttribute('path')
597 for entry in target.getElementsByTagName('entry'):
598 file_path = entry.getAttribute('path')
599 wc_status = entry.getElementsByTagName('wc-status')
600 assert len(wc_status) == 1
601 # Emulate svn 1.5 status ouput...
602 statuses = [' '] * 7
603 # Col 0
604 xml_item_status = wc_status[0].getAttribute('item')
605 if xml_item_status in status_letter:
606 statuses[0] = status_letter[xml_item_status]
607 else:
608 raise Exception('Unknown item status "%s"; please implement me!' %
609 xml_item_status)
610 # Col 1
611 xml_props_status = wc_status[0].getAttribute('props')
612 if xml_props_status == 'modified':
613 statuses[1] = 'M'
614 elif xml_props_status == 'conflicted':
615 statuses[1] = 'C'
616 elif (not xml_props_status or xml_props_status == 'none' or
617 xml_props_status == 'normal'):
618 pass
619 else:
620 raise Exception('Unknown props status "%s"; please implement me!' %
621 xml_props_status)
622 # Col 2
623 if wc_status[0].getAttribute('wc-locked') == 'true':
624 statuses[2] = 'L'
625 # Col 3
626 if wc_status[0].getAttribute('copied') == 'true':
627 statuses[3] = '+'
628 # Col 4
629 if wc_status[0].getAttribute('switched') == 'true':
630 statuses[4] = 'S'
631 # TODO(maruel): Col 5 and 6
632 item = (''.join(statuses), file_path)
633 results.append(item)
634 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000635
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000636 @staticmethod
637 def IsMoved(filename):
638 """Determine if a file has been added through svn mv"""
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000639 return SVN.IsMovedInfo(SVN.CaptureInfo(filename))
640
641 @staticmethod
642 def IsMovedInfo(info):
643 """Determine if a file has been added through svn mv"""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000644 return (info.get('Copied From URL') and
645 info.get('Copied From Rev') and
646 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000647
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000648 @staticmethod
649 def GetFileProperty(file, property_name):
650 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000651
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000652 Args:
653 file: The file to check
654 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000655
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000656 Returns:
657 The value of the property, which will be the empty string if the property
658 is not set on the file. If the file is not under version control, the
659 empty string is also returned.
660 """
661 output = SVN.Capture(["propget", property_name, file])
662 if (output.startswith("svn: ") and
663 output.endswith("is not under version control")):
664 return ""
665 else:
666 return output
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000667
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000668 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000669 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000670 """Diffs a single file.
671
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000672 Should be simple, eh? No it isn't.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000673 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000674 expected relative path.
675 full_move means that move or copy operations should completely recreate the
676 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000677 # If the user specified a custom diff command in their svn config file,
678 # then it'll be used when we do svn diff, which we don't want to happen
679 # since we want the unified diff. Using --diff-cmd=diff doesn't always
680 # work, since they can have another diff executable in their path that
681 # gives different line endings. So we use a bogus temp directory as the
682 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000683 bogus_dir = tempfile.mkdtemp()
684 try:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000685 # Use "svn info" output instead of os.path.isdir because the latter fails
686 # when the file is deleted.
687 return SVN._DiffItemInternal(SVN.CaptureInfo(filename),
688 full_move=full_move, revision=revision)
689 finally:
690 shutil.rmtree(bogus_dir)
691
692 @staticmethod
693 def _DiffItemInternal(filename, info, bogus_dir, full_move=False,
694 revision=None):
695 """Grabs the diff data."""
696 command = ["diff", "--config-dir", bogus_dir, filename]
697 if revision:
698 command.extend(['--revision', revision])
699 data = None
700 if SVN.IsMovedInfo(info):
701 if full_move:
702 if info.get("Node Kind") == "directory":
703 # Things become tricky here. It's a directory copy/move. We need to
704 # diff all the files inside it.
705 # This will put a lot of pressure on the heap. This is why StringIO
706 # is used and converted back into a string at the end. The reason to
707 # return a string instead of a StringIO is that StringIO.write()
708 # doesn't accept a StringIO object. *sigh*.
709 for (dirpath, dirnames, filenames) in os.walk(filename):
710 # Cleanup all files starting with a '.'.
711 for d in dirnames:
712 if d.startswith('.'):
713 dirnames.remove(d)
714 for f in filenames:
715 if f.startswith('.'):
716 filenames.remove(f)
717 for f in filenames:
718 if data is None:
719 data = cStringIO.StringIO()
720 data.write(GenFakeDiff(os.path.join(dirpath, f)))
721 if data:
722 tmp = data.getvalue()
723 data.close()
724 data = tmp
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000725 else:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000726 data = GenFakeDiff(filename)
727 else:
728 if info.get("Node Kind") != "directory":
maruel@chromium.org0836c562010-01-22 01:10:06 +0000729 # svn diff on a mv/cp'd file outputs nothing if there was no change.
730 data = SVN.Capture(command, None)
731 if not data:
732 # We put in an empty Index entry so upload.py knows about them.
733 data = "Index: %s\n" % filename
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000734 # Otherwise silently ignore directories.
735 else:
736 if info.get("Node Kind") != "directory":
737 # Normal simple case.
maruel@chromium.org0836c562010-01-22 01:10:06 +0000738 data = SVN.Capture(command, None)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000739 # Otherwise silently ignore directories.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000740 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000741
742 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000743 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000744 """Returns a string containing the diff for the given file list.
745
746 The files in the list should either be absolute paths or relative to the
747 given root. If no root directory is provided, the repository root will be
748 used.
749 The diff will always use relative paths.
750 """
751 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000752 root = root or SVN.GetCheckoutRoot(previous_cwd)
753 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000754 def RelativePath(path, root):
755 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000756 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000757 return path[len(root):]
758 return path
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000759 # If the user specified a custom diff command in their svn config file,
760 # then it'll be used when we do svn diff, which we don't want to happen
761 # since we want the unified diff. Using --diff-cmd=diff doesn't always
762 # work, since they can have another diff executable in their path that
763 # gives different line endings. So we use a bogus temp directory as the
764 # config directory, which gets around these problems.
765 bogus_dir = tempfile.mkdtemp()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000766 try:
767 os.chdir(root)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000768 # Cleanup filenames
769 filenames = [RelativePath(f, root) for f in filenames]
770 # Get information about the modified items (files and directories)
771 data = dict([(f, SVN.CaptureInfo(f)) for f in filenames])
772 if full_move:
773 # Eliminate modified files inside moved/copied directory.
774 for (filename, info) in data.iteritems():
775 if SVN.IsMovedInfo(info) and info.get("Node Kind") == "directory":
776 # Remove files inside the directory.
777 filenames = [f for f in filenames
778 if not f.startswith(filename + os.path.sep)]
779 for filename in data.keys():
780 if not filename in filenames:
781 # Remove filtered out items.
782 del data[filename]
783 # Now ready to do the actual diff.
784 diffs = []
785 for filename in sorted(data.iterkeys()):
786 diffs.append(SVN._DiffItemInternal(filename, data[filename], bogus_dir,
787 full_move=full_move,
788 revision=revision))
789 # Use StringIO since it can be messy when diffing a directory move with
790 # full_move=True.
791 buf = cStringIO.StringIO()
792 for d in filter(None, diffs):
793 buf.write(d)
794 result = buf.getvalue()
795 buf.close()
796 return result
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000797 finally:
798 os.chdir(previous_cwd)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000799 shutil.rmtree(bogus_dir)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000800
801 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000802 def GetEmail(repo_root):
803 """Retrieves the svn account which we assume is an email address."""
804 infos = SVN.CaptureInfo(repo_root)
805 uuid = infos.get('UUID')
806 root = infos.get('Repository Root')
807 if not root:
808 return None
809
810 # Should check for uuid but it is incorrectly saved for https creds.
811 realm = root.rsplit('/', 1)[0]
812 if root.startswith('https') or not uuid:
813 regexp = re.compile(r'<%s:\d+>.*' % realm)
814 else:
815 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
816 if regexp is None:
817 return None
818 if sys.platform.startswith('win'):
819 if not 'APPDATA' in os.environ:
820 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000821 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
822 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000823 else:
824 if not 'HOME' in os.environ:
825 return None
826 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
827 'svn.simple')
828 for credfile in os.listdir(auth_dir):
829 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
830 if regexp.match(cred_info.get('svn:realmstring')):
831 return cred_info.get('username')
832
833 @staticmethod
834 def ReadSimpleAuth(filename):
835 f = open(filename, 'r')
836 values = {}
837 def ReadOneItem(type):
838 m = re.match(r'%s (\d+)' % type, f.readline())
839 if not m:
840 return None
841 data = f.read(int(m.group(1)))
842 if f.read(1) != '\n':
843 return None
844 return data
845
846 while True:
847 key = ReadOneItem('K')
848 if not key:
849 break
850 value = ReadOneItem('V')
851 if not value:
852 break
853 values[key] = value
854 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000855
856 @staticmethod
857 def GetCheckoutRoot(directory):
858 """Returns the top level directory of the current repository.
859
860 The directory is returned as an absolute path.
861 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000862 directory = os.path.abspath(directory)
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000863 infos = SVN.CaptureInfo(directory, print_error=False)
864 cur_dir_repo_root = infos.get("Repository Root")
865 if not cur_dir_repo_root:
866 return None
867
868 while True:
869 parent = os.path.dirname(directory)
870 if (SVN.CaptureInfo(parent, print_error=False).get(
871 "Repository Root") != cur_dir_repo_root):
872 break
873 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000874 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000875
876 @staticmethod
877 def AssertVersion(min_version):
878 """Asserts svn's version is at least min_version."""
879 def only_int(val):
880 if val.isdigit():
881 return int(val)
882 else:
883 return 0
884 if not SVN.current_version:
885 SVN.current_version = SVN.Capture(['--version']).split()[2]
886 current_version_list = map(only_int, SVN.current_version.split('.'))
887 for min_ver in map(int, min_version.split('.')):
888 ver = current_version_list.pop(0)
889 if ver < min_ver:
890 return (False, SVN.current_version)
891 elif ver > min_ver:
892 return (True, SVN.current_version)
893 return (True, SVN.current_version)