blob: a9b1ecf84b4759b84c43125a6010cc1876342c8d [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00007import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00008import os
9import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000010import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000011import subprocess
12import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000013import tempfile
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000014import xml.dom.minidom
15
16import gclient_utils
17
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000018def ValidateEmail(email):
19 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
20 is not None)
21
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000022
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000023def GetCasedPath(path):
24 """Elcheapos way to get the real path case on Windows."""
25 if sys.platform.startswith('win') and os.path.exists(path):
26 # Reconstruct the path.
27 path = os.path.abspath(path)
28 paths = path.split('\\')
29 for i in range(len(paths)):
30 if i == 0:
31 # Skip drive letter.
32 continue
33 subpath = '\\'.join(paths[:i+1])
34 prev = len('\\'.join(paths[:i]))
35 # glob.glob will return the cased path for the last item only. This is why
36 # we are calling it in a loop. Extract the data we want and put it back
37 # into the list.
38 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
39 path = '\\'.join(paths)
40 return path
41
42
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000043class GIT(object):
44 COMMAND = "git"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000045
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000046 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000047 def Capture(args, in_directory=None, print_error=True, error_ok=False):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000048 """Runs git, capturing output sent to stdout as a string.
49
50 Args:
51 args: A sequence of command line parameters to be passed to git.
52 in_directory: The directory where git is to be run.
53
54 Returns:
55 The output sent to stdout as a string.
56 """
57 c = [GIT.COMMAND]
58 c.extend(args)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000059 try:
60 return gclient_utils.CheckCall(c, in_directory, print_error)
61 except gclient_utils.CheckCallError:
62 if error_ok:
nasser@codeaurora.orgcd968c12010-02-01 06:05:00 +000063 return ('', '')
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000064 raise
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000065
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000066 @staticmethod
67 def CaptureStatus(files, upstream_branch='origin'):
68 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000069
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000070 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000071
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000072 Returns an array of (status, file) tuples."""
bauerb@chromium.org14ec5042010-03-30 18:19:09 +000073 command = ["diff", "--name-status", "-r", "%s..." % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000074 if not files:
75 pass
76 elif isinstance(files, basestring):
77 command.append(files)
78 else:
79 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000080
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000081 status = GIT.Capture(command)[0].rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000082 results = []
83 if status:
84 for statusline in status.split('\n'):
85 m = re.match('^(\w)\t(.+)$', statusline)
86 if not m:
87 raise Exception("status currently unsupported: %s" % statusline)
88 results.append(('%s ' % m.group(1), m.group(2)))
89 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000090
maruel@chromium.orgc78f2462009-11-21 01:20:57 +000091 @staticmethod
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000092 def RunAndFilterOutput(args,
93 in_directory,
94 print_messages,
95 print_stdout,
96 filter):
97 """Runs a command, optionally outputting to stdout.
98
99 stdout is passed line-by-line to the given filter function. If
100 print_stdout is true, it is also printed to sys.stdout as in Run.
101
102 Args:
103 args: A sequence of command line parameters to be passed.
msb@chromium.orgd6504212010-01-13 17:34:31 +0000104 in_directory: The directory where git is to be run.
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000105 print_messages: Whether to print status messages to stdout about
106 which commands are being run.
107 print_stdout: Whether to forward program's output to stdout.
108 filter: A function taking one argument (a string) which will be
109 passed each line (with the ending newline character removed) of
110 program's output for filtering.
111
112 Raises:
113 gclient_utils.Error: An error occurred while running the command.
114 """
115 command = [GIT.COMMAND]
116 command.extend(args)
117 gclient_utils.SubprocessCallAndFilter(command,
118 in_directory,
119 print_messages,
120 print_stdout,
121 filter=filter)
122
123 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000124 def GetEmail(repo_root):
125 """Retrieves the user email address if known."""
126 # We could want to look at the svn cred when it has a svn remote but it
127 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000128 return GIT.Capture(['config', 'user.email'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000129 repo_root, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000130
131 @staticmethod
132 def ShortBranchName(branch):
133 """Converts a name like 'refs/heads/foo' to just 'foo'."""
134 return branch.replace('refs/heads/', '')
135
136 @staticmethod
137 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000138 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000139 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000140
141 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000142 def GetBranch(cwd):
143 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000144 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000145
146 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000147 def IsGitSvn(cwd):
148 """Returns true if this repo looks like it's using git-svn."""
149 # If you have any "svn-remote.*" config keys, we think you're using svn.
150 try:
151 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd)
152 return True
153 except gclient_utils.CheckCallError:
154 return False
155
156 @staticmethod
157 def GetSVNBranch(cwd):
158 """Returns the svn branch name if found."""
159 # Try to figure out which remote branch we're based on.
160 # Strategy:
161 # 1) find all git-svn branches and note their svn URLs.
162 # 2) iterate through our branch history and match up the URLs.
163
164 # regexp matching the git-svn line that contains the URL.
165 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
166
167 # Get the refname and svn url for all refs/remotes/*.
168 remotes = GIT.Capture(
169 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000170 cwd)[0].splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000171 svn_refs = {}
172 for ref in remotes:
173 match = git_svn_re.search(
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000174 GIT.Capture(['cat-file', '-p', ref], cwd)[0])
sky@chromium.org42d8da52010-04-23 18:25:07 +0000175 # Prefer origin/HEAD over all others.
176 if match and (match.group(1) not in svn_refs or
177 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000178 svn_refs[match.group(1)] = ref
179
180 svn_branch = ''
181 if len(svn_refs) == 1:
182 # Only one svn branch exists -- seems like a good candidate.
183 svn_branch = svn_refs.values()[0]
184 elif len(svn_refs) > 1:
185 # We have more than one remote branch available. We don't
186 # want to go through all of history, so read a line from the
187 # pipe at a time.
188 # The -100 is an arbitrary limit so we don't search forever.
189 cmd = ['git', 'log', '-100', '--pretty=medium']
190 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
191 for line in proc.stdout:
192 match = git_svn_re.match(line)
193 if match:
194 url = match.group(1)
195 if url in svn_refs:
196 svn_branch = svn_refs[url]
197 proc.stdout.close() # Cut pipe.
198 break
199 return svn_branch
200
201 @staticmethod
202 def FetchUpstreamTuple(cwd):
203 """Returns a tuple containg remote and remote ref,
204 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000205 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000206 """
207 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000208 branch = GIT.GetBranch(cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000209 upstream_branch = None
210 upstream_branch = GIT.Capture(
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000211 ['config', 'branch.%s.merge' % branch], in_directory=cwd,
212 error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000213 if upstream_branch:
214 remote = GIT.Capture(
215 ['config', 'branch.%s.remote' % branch],
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000216 in_directory=cwd, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000217 else:
218 # Fall back on trying a git-svn upstream branch.
219 if GIT.IsGitSvn(cwd):
220 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000221 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000222 # Else, try to guess the origin remote.
223 remote_branches = GIT.Capture(
224 ['branch', '-r'], in_directory=cwd)[0].split()
225 if 'origin/master' in remote_branches:
226 # Fall back on origin/master if it exits.
227 remote = 'origin'
228 upstream_branch = 'refs/heads/master'
229 elif 'origin/trunk' in remote_branches:
230 # Fall back on origin/trunk if it exists. Generally a shared
231 # git-svn clone
232 remote = 'origin'
233 upstream_branch = 'refs/heads/trunk'
234 else:
235 # Give up.
236 remote = None
237 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000238 return remote, upstream_branch
239
240 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000241 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000242 """Gets the current branch's upstream branch."""
243 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000244 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000245 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
246 return upstream_branch
247
248 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000249 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
250 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000251 """Diffs against the upstream branch or optionally another branch.
252
253 full_move means that move or copy operations should completely recreate the
254 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000255 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000256 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000257 command = ['diff', '-p', '--no-prefix', branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000258 if not full_move:
259 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000260 # TODO(maruel): --binary support.
261 if files:
262 command.append('--')
263 command.extend(files)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000264 diff = GIT.Capture(command, cwd)[0].splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000265 for i in range(len(diff)):
266 # In the case of added files, replace /dev/null with the path to the
267 # file being added.
268 if diff[i].startswith('--- /dev/null'):
269 diff[i] = '--- %s' % diff[i+1][4:]
270 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000271
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000272 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000273 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
274 """Returns the list of modified files between two branches."""
275 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000276 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000277 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000278 return GIT.Capture(command, cwd)[0].splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000279
280 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000281 def GetPatchName(cwd):
282 """Constructs a name for this patch."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000283 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000284 return "%s-%s" % (GIT.GetBranch(cwd), short_sha)
285
286 @staticmethod
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000287 def GetCheckoutRoot(path):
288 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000289 """
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000290 root = GIT.Capture(['rev-parse', '--show-cdup'], path)[0].strip()
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000291 return os.path.abspath(os.path.join(path, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000292
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000293 @staticmethod
294 def AssertVersion(min_version):
295 """Asserts git's version is at least min_version."""
296 def only_int(val):
297 if val.isdigit():
298 return int(val)
299 else:
300 return 0
301 current_version = GIT.Capture(['--version'])[0].split()[-1]
302 current_version_list = map(only_int, current_version.split('.'))
303 for min_ver in map(int, min_version.split('.')):
304 ver = current_version_list.pop(0)
305 if ver < min_ver:
306 return (False, current_version)
307 elif ver > min_ver:
308 return (True, current_version)
309 return (True, current_version)
310
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000311
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000312class SVN(object):
313 COMMAND = "svn"
tony@chromium.org57564662010-04-14 02:35:12 +0000314 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000315
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000316 @staticmethod
317 def Run(args, in_directory):
318 """Runs svn, sending output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000319
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000320 Args:
321 args: A sequence of command line parameters to be passed to svn.
322 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000323
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000324 Raises:
325 Error: An error occurred while running the svn command.
326 """
327 c = [SVN.COMMAND]
328 c.extend(args)
maruel@chromium.org2185f002009-12-18 21:03:47 +0000329 # TODO(maruel): This is very gclient-specific.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000330 gclient_utils.SubprocessCall(c, in_directory)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000331
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000332 @staticmethod
333 def Capture(args, in_directory=None, print_error=True):
334 """Runs svn, capturing output sent to stdout as a string.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000335
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000336 Args:
337 args: A sequence of command line parameters to be passed to svn.
338 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000339
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000340 Returns:
341 The output sent to stdout as a string.
342 """
343 c = [SVN.COMMAND]
344 c.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000345
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000346 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
347 # the svn.exe executable, but shell=True makes subprocess on Linux fail
348 # when it's called with a list because it only tries to execute the
349 # first string ("svn").
350 stderr = None
351 if not print_error:
352 stderr = subprocess.PIPE
353 return subprocess.Popen(c,
354 cwd=in_directory,
355 shell=(sys.platform == 'win32'),
356 stdout=subprocess.PIPE,
357 stderr=stderr).communicate()[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000358
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000359 @staticmethod
360 def RunAndGetFileList(options, args, in_directory, file_list):
361 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000362
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000363 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000364
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000365 svn's stdout is parsed to collect a list of files checked out or updated.
366 These files are appended to file_list. svn's stdout is also printed to
367 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000368
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000369 Args:
370 options: command line options to gclient
371 args: A sequence of command line parameters to be passed to svn.
372 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000373
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000374 Raises:
375 Error: An error occurred while running the svn command.
376 """
377 command = [SVN.COMMAND]
378 command.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000379
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000380 # svn update and svn checkout use the same pattern: the first three columns
381 # are for file status, property status, and lock status. This is followed
382 # by two spaces, and then the path to the file.
383 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000384
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000385 # The first three columns of svn status are the same as for svn update and
386 # svn checkout. The next three columns indicate addition-with-history,
387 # switch, and remote lock status. This is followed by one space, and then
388 # the path to the file.
389 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000390
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000391 # args[0] must be a supported command. This will blow up if it's something
392 # else, which is good. Note that the patterns are only effective when
393 # these commands are used in their ordinary forms, the patterns are invalid
394 # for "svn status --show-updates", for example.
395 pattern = {
396 'checkout': update_pattern,
397 'status': status_pattern,
398 'update': update_pattern,
399 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000400 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000401 # Place an upper limit.
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000402 for _ in range(1, 10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000403 previous_list_len = len(file_list)
404 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000405
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000406 def CaptureMatchingLines(line):
407 match = compiled_pattern.search(line)
408 if match:
409 file_list.append(match.group(1))
410 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000411 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000412
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000413 try:
414 SVN.RunAndFilterOutput(args,
415 in_directory,
416 options.verbose,
417 True,
418 CaptureMatchingLines)
419 except gclient_utils.Error:
maruel@chromium.org2de10252010-02-08 01:10:39 +0000420 # We enforce that some progress has been made or HTTP 502.
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000421 if (filter(lambda x: '502 Bad Gateway' in x, failure) or
maruel@chromium.org2de10252010-02-08 01:10:39 +0000422 (len(failure) and len(file_list) > previous_list_len)):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000423 if args[0] == 'checkout':
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000424 # An aborted checkout is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000425 args = ['update'] + args[1:]
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000426 continue
maruel@chromium.org2de10252010-02-08 01:10:39 +0000427 # No progress was made or an unknown error we aren't sure, bail out.
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000428 raise
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000429 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000430
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000431 @staticmethod
432 def RunAndFilterOutput(args,
433 in_directory,
434 print_messages,
435 print_stdout,
436 filter):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000437 """Runs a command, optionally outputting to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000438
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000439 stdout is passed line-by-line to the given filter function. If
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000440 print_stdout is true, it is also printed to sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000441
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000442 Args:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000443 args: A sequence of command line parameters to be passed.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000444 in_directory: The directory where svn is to be run.
445 print_messages: Whether to print status messages to stdout about
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000446 which commands are being run.
447 print_stdout: Whether to forward program's output to stdout.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000448 filter: A function taking one argument (a string) which will be
449 passed each line (with the ending newline character removed) of
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000450 program's output for filtering.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000451
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000452 Raises:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000453 gclient_utils.Error: An error occurred while running the command.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000454 """
455 command = [SVN.COMMAND]
456 command.extend(args)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000457 gclient_utils.SubprocessCallAndFilter(command,
458 in_directory,
459 print_messages,
460 print_stdout,
461 filter=filter)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000462
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000463 @staticmethod
464 def CaptureInfo(relpath, in_directory=None, print_error=True):
465 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000466
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000467 Args:
468 relpath: The directory where the working copy resides relative to
469 the directory given by in_directory.
470 in_directory: The directory where svn is to be run.
471 """
472 output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
473 dom = gclient_utils.ParseXML(output)
474 result = {}
475 if dom:
476 GetNamedNodeText = gclient_utils.GetNamedNodeText
477 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
478 def C(item, f):
479 if item is not None: return f(item)
480 # /info/entry/
481 # url
482 # reposityory/(root|uuid)
483 # wc-info/(schedule|depth)
484 # commit/(author|date)
485 # str() the results because they may be returned as Unicode, which
486 # interferes with the higher layers matching up things in the deps
487 # dictionary.
488 # TODO(maruel): Fix at higher level instead (!)
489 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
490 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
491 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
492 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
493 'revision'),
494 int)
495 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
496 str)
497 # Differs across versions.
498 if result['Node Kind'] == 'dir':
499 result['Node Kind'] = 'directory'
500 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
501 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
502 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
503 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
504 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000505
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000506 @staticmethod
507 def CaptureHeadRevision(url):
508 """Get the head revision of a SVN repository.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000509
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000510 Returns:
511 Int head revision
512 """
513 info = SVN.Capture(["info", "--xml", url], os.getcwd())
514 dom = xml.dom.minidom.parseString(info)
515 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000516
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000517 @staticmethod
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000518 def CaptureBaseRevision(cwd):
519 """Get the base revision of a SVN repository.
520
521 Returns:
522 Int base revision
523 """
524 info = SVN.Capture(["info", "--xml"], cwd)
525 dom = xml.dom.minidom.parseString(info)
526 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
527
528 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000529 def CaptureStatus(files):
530 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000531
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000532 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000533
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000534 Returns an array of (status, file) tuples."""
535 command = ["status", "--xml"]
536 if not files:
537 pass
538 elif isinstance(files, basestring):
539 command.append(files)
540 else:
541 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000542
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000543 status_letter = {
544 None: ' ',
545 '': ' ',
546 'added': 'A',
547 'conflicted': 'C',
548 'deleted': 'D',
549 'external': 'X',
550 'ignored': 'I',
551 'incomplete': '!',
552 'merged': 'G',
553 'missing': '!',
554 'modified': 'M',
555 'none': ' ',
556 'normal': ' ',
557 'obstructed': '~',
558 'replaced': 'R',
559 'unversioned': '?',
560 }
561 dom = gclient_utils.ParseXML(SVN.Capture(command))
562 results = []
563 if dom:
564 # /status/target/entry/(wc-status|commit|author|date)
565 for target in dom.getElementsByTagName('target'):
566 #base_path = target.getAttribute('path')
567 for entry in target.getElementsByTagName('entry'):
568 file_path = entry.getAttribute('path')
569 wc_status = entry.getElementsByTagName('wc-status')
570 assert len(wc_status) == 1
571 # Emulate svn 1.5 status ouput...
572 statuses = [' '] * 7
573 # Col 0
574 xml_item_status = wc_status[0].getAttribute('item')
575 if xml_item_status in status_letter:
576 statuses[0] = status_letter[xml_item_status]
577 else:
578 raise Exception('Unknown item status "%s"; please implement me!' %
579 xml_item_status)
580 # Col 1
581 xml_props_status = wc_status[0].getAttribute('props')
582 if xml_props_status == 'modified':
583 statuses[1] = 'M'
584 elif xml_props_status == 'conflicted':
585 statuses[1] = 'C'
586 elif (not xml_props_status or xml_props_status == 'none' or
587 xml_props_status == 'normal'):
588 pass
589 else:
590 raise Exception('Unknown props status "%s"; please implement me!' %
591 xml_props_status)
592 # Col 2
593 if wc_status[0].getAttribute('wc-locked') == 'true':
594 statuses[2] = 'L'
595 # Col 3
596 if wc_status[0].getAttribute('copied') == 'true':
597 statuses[3] = '+'
598 # Col 4
599 if wc_status[0].getAttribute('switched') == 'true':
600 statuses[4] = 'S'
601 # TODO(maruel): Col 5 and 6
602 item = (''.join(statuses), file_path)
603 results.append(item)
604 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000605
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000606 @staticmethod
607 def IsMoved(filename):
608 """Determine if a file has been added through svn mv"""
609 info = SVN.CaptureInfo(filename)
610 return (info.get('Copied From URL') and
611 info.get('Copied From Rev') and
612 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000613
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000614 @staticmethod
615 def GetFileProperty(file, property_name):
616 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000617
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000618 Args:
619 file: The file to check
620 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000621
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000622 Returns:
623 The value of the property, which will be the empty string if the property
624 is not set on the file. If the file is not under version control, the
625 empty string is also returned.
626 """
627 output = SVN.Capture(["propget", property_name, file])
628 if (output.startswith("svn: ") and
629 output.endswith("is not under version control")):
630 return ""
631 else:
632 return output
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000633
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000634 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000635 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000636 """Diffs a single file.
637
638 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000639 expected relative path.
640 full_move means that move or copy operations should completely recreate the
641 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000642 # Use svn info output instead of os.path.isdir because the latter fails
643 # when the file is deleted.
644 if SVN.CaptureInfo(filename).get("Node Kind") == "directory":
645 return None
646 # If the user specified a custom diff command in their svn config file,
647 # then it'll be used when we do svn diff, which we don't want to happen
648 # since we want the unified diff. Using --diff-cmd=diff doesn't always
649 # work, since they can have another diff executable in their path that
650 # gives different line endings. So we use a bogus temp directory as the
651 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000652 bogus_dir = tempfile.mkdtemp()
653 try:
654 # Grabs the diff data.
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000655 command = ["diff", "--config-dir", bogus_dir, filename]
maruel@chromium.orgce3c8622010-01-07 02:18:16 +0000656 if revision:
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000657 command.extend(['--revision', revision])
maruel@chromium.org0836c562010-01-22 01:10:06 +0000658 if SVN.IsMoved(filename):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000659 if full_move:
660 file_content = gclient_utils.FileRead(filename, 'rb')
661 # Prepend '+' to every lines.
662 file_content = ['+' + i for i in file_content.splitlines(True)]
663 nb_lines = len(file_content)
664 # We need to use / since patch on unix will fail otherwise.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000665 data = "Index: %s\n" % filename
666 data += '=' * 67 + '\n'
667 # Note: Should we use /dev/null instead?
668 data += "--- %s\n" % filename
669 data += "+++ %s\n" % filename
670 data += "@@ -0,0 +1,%d @@\n" % nb_lines
671 data += ''.join(file_content)
672 else:
maruel@chromium.org0836c562010-01-22 01:10:06 +0000673 # svn diff on a mv/cp'd file outputs nothing if there was no change.
674 data = SVN.Capture(command, None)
675 if not data:
676 # We put in an empty Index entry so upload.py knows about them.
677 data = "Index: %s\n" % filename
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000678 else:
maruel@chromium.org0836c562010-01-22 01:10:06 +0000679 data = SVN.Capture(command, None)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000680 finally:
681 shutil.rmtree(bogus_dir)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000682 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000683
684 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000685 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000686 """Returns a string containing the diff for the given file list.
687
688 The files in the list should either be absolute paths or relative to the
689 given root. If no root directory is provided, the repository root will be
690 used.
691 The diff will always use relative paths.
692 """
693 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000694 root = root or SVN.GetCheckoutRoot(previous_cwd)
695 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000696 def RelativePath(path, root):
697 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000698 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000699 return path[len(root):]
700 return path
701 try:
702 os.chdir(root)
703 diff = "".join(filter(None,
maruel@chromium.org0c401692010-01-07 02:44:20 +0000704 [SVN.DiffItem(RelativePath(f, root),
705 full_move=full_move,
706 revision=revision)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000707 for f in filenames]))
708 finally:
709 os.chdir(previous_cwd)
710 return diff
711
712
713 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000714 def GetEmail(repo_root):
715 """Retrieves the svn account which we assume is an email address."""
716 infos = SVN.CaptureInfo(repo_root)
717 uuid = infos.get('UUID')
718 root = infos.get('Repository Root')
719 if not root:
720 return None
721
722 # Should check for uuid but it is incorrectly saved for https creds.
723 realm = root.rsplit('/', 1)[0]
724 if root.startswith('https') or not uuid:
725 regexp = re.compile(r'<%s:\d+>.*' % realm)
726 else:
727 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
728 if regexp is None:
729 return None
730 if sys.platform.startswith('win'):
731 if not 'APPDATA' in os.environ:
732 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000733 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
734 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000735 else:
736 if not 'HOME' in os.environ:
737 return None
738 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
739 'svn.simple')
740 for credfile in os.listdir(auth_dir):
741 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
742 if regexp.match(cred_info.get('svn:realmstring')):
743 return cred_info.get('username')
744
745 @staticmethod
746 def ReadSimpleAuth(filename):
747 f = open(filename, 'r')
748 values = {}
749 def ReadOneItem(type):
750 m = re.match(r'%s (\d+)' % type, f.readline())
751 if not m:
752 return None
753 data = f.read(int(m.group(1)))
754 if f.read(1) != '\n':
755 return None
756 return data
757
758 while True:
759 key = ReadOneItem('K')
760 if not key:
761 break
762 value = ReadOneItem('V')
763 if not value:
764 break
765 values[key] = value
766 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000767
768 @staticmethod
769 def GetCheckoutRoot(directory):
770 """Returns the top level directory of the current repository.
771
772 The directory is returned as an absolute path.
773 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000774 directory = os.path.abspath(directory)
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000775 infos = SVN.CaptureInfo(directory, print_error=False)
776 cur_dir_repo_root = infos.get("Repository Root")
777 if not cur_dir_repo_root:
778 return None
779
780 while True:
781 parent = os.path.dirname(directory)
782 if (SVN.CaptureInfo(parent, print_error=False).get(
783 "Repository Root") != cur_dir_repo_root):
784 break
785 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000786 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000787
788 @staticmethod
789 def AssertVersion(min_version):
790 """Asserts svn's version is at least min_version."""
791 def only_int(val):
792 if val.isdigit():
793 return int(val)
794 else:
795 return 0
796 if not SVN.current_version:
797 SVN.current_version = SVN.Capture(['--version']).split()[2]
798 current_version_list = map(only_int, SVN.current_version.split('.'))
799 for min_ver in map(int, min_version.split('.')):
800 ver = current_version_list.pop(0)
801 if ver < min_ver:
802 return (False, SVN.current_version)
803 elif ver > min_ver:
804 return (True, SVN.current_version)
805 return (True, SVN.current_version)