blob: 038d2dd4719a5ad777c383e6a83fc346eae9a9da [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00007import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00008import os
9import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000010import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000011import subprocess
12import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000013import tempfile
maruel@chromium.orgfd876172010-04-30 14:01:05 +000014import time
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000015import xml.dom.minidom
16
17import gclient_utils
18
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000019def ValidateEmail(email):
20 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
21 is not None)
22
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000023
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000024def GetCasedPath(path):
25 """Elcheapos way to get the real path case on Windows."""
26 if sys.platform.startswith('win') and os.path.exists(path):
27 # Reconstruct the path.
28 path = os.path.abspath(path)
29 paths = path.split('\\')
30 for i in range(len(paths)):
31 if i == 0:
32 # Skip drive letter.
33 continue
34 subpath = '\\'.join(paths[:i+1])
35 prev = len('\\'.join(paths[:i]))
36 # glob.glob will return the cased path for the last item only. This is why
37 # we are calling it in a loop. Extract the data we want and put it back
38 # into the list.
39 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
40 path = '\\'.join(paths)
41 return path
42
43
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000044class GIT(object):
45 COMMAND = "git"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000046
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000047 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000048 def Capture(args, in_directory=None, print_error=True, error_ok=False):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000049 """Runs git, capturing output sent to stdout as a string.
50
51 Args:
52 args: A sequence of command line parameters to be passed to git.
53 in_directory: The directory where git is to be run.
54
55 Returns:
56 The output sent to stdout as a string.
57 """
58 c = [GIT.COMMAND]
59 c.extend(args)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000060 try:
61 return gclient_utils.CheckCall(c, in_directory, print_error)
62 except gclient_utils.CheckCallError:
63 if error_ok:
nasser@codeaurora.orgcd968c12010-02-01 06:05:00 +000064 return ('', '')
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000065 raise
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000066
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000067 @staticmethod
68 def CaptureStatus(files, upstream_branch='origin'):
69 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000070
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000071 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000072
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000073 Returns an array of (status, file) tuples."""
bauerb@chromium.org14ec5042010-03-30 18:19:09 +000074 command = ["diff", "--name-status", "-r", "%s..." % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000075 if not files:
76 pass
77 elif isinstance(files, basestring):
78 command.append(files)
79 else:
80 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000081
maruel@chromium.org7be5ef22010-01-30 22:31:50 +000082 status = GIT.Capture(command)[0].rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000083 results = []
84 if status:
85 for statusline in status.split('\n'):
86 m = re.match('^(\w)\t(.+)$', statusline)
87 if not m:
88 raise Exception("status currently unsupported: %s" % statusline)
89 results.append(('%s ' % m.group(1), m.group(2)))
90 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000091
maruel@chromium.orgc78f2462009-11-21 01:20:57 +000092 @staticmethod
maruel@chromium.orgee4071d2009-12-22 22:25:37 +000093 def RunAndFilterOutput(args,
94 in_directory,
95 print_messages,
96 print_stdout,
97 filter):
98 """Runs a command, optionally outputting to stdout.
99
100 stdout is passed line-by-line to the given filter function. If
101 print_stdout is true, it is also printed to sys.stdout as in Run.
102
103 Args:
104 args: A sequence of command line parameters to be passed.
msb@chromium.orgd6504212010-01-13 17:34:31 +0000105 in_directory: The directory where git is to be run.
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000106 print_messages: Whether to print status messages to stdout about
107 which commands are being run.
108 print_stdout: Whether to forward program's output to stdout.
109 filter: A function taking one argument (a string) which will be
110 passed each line (with the ending newline character removed) of
111 program's output for filtering.
112
113 Raises:
114 gclient_utils.Error: An error occurred while running the command.
115 """
116 command = [GIT.COMMAND]
117 command.extend(args)
118 gclient_utils.SubprocessCallAndFilter(command,
119 in_directory,
120 print_messages,
121 print_stdout,
122 filter=filter)
123
124 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000125 def GetEmail(repo_root):
126 """Retrieves the user email address if known."""
127 # We could want to look at the svn cred when it has a svn remote but it
128 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000129 return GIT.Capture(['config', 'user.email'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000130 repo_root, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000131
132 @staticmethod
133 def ShortBranchName(branch):
134 """Converts a name like 'refs/heads/foo' to just 'foo'."""
135 return branch.replace('refs/heads/', '')
136
137 @staticmethod
138 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000139 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000140 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000141
142 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000143 def GetBranch(cwd):
144 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000145 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000146
147 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000148 def IsGitSvn(cwd):
149 """Returns true if this repo looks like it's using git-svn."""
150 # If you have any "svn-remote.*" config keys, we think you're using svn.
151 try:
152 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd)
153 return True
154 except gclient_utils.CheckCallError:
155 return False
156
157 @staticmethod
158 def GetSVNBranch(cwd):
159 """Returns the svn branch name if found."""
160 # Try to figure out which remote branch we're based on.
161 # Strategy:
162 # 1) find all git-svn branches and note their svn URLs.
163 # 2) iterate through our branch history and match up the URLs.
164
165 # regexp matching the git-svn line that contains the URL.
166 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
167
168 # Get the refname and svn url for all refs/remotes/*.
169 remotes = GIT.Capture(
170 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000171 cwd)[0].splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000172 svn_refs = {}
173 for ref in remotes:
174 match = git_svn_re.search(
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000175 GIT.Capture(['cat-file', '-p', ref], cwd)[0])
sky@chromium.org42d8da52010-04-23 18:25:07 +0000176 # Prefer origin/HEAD over all others.
177 if match and (match.group(1) not in svn_refs or
178 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000179 svn_refs[match.group(1)] = ref
180
181 svn_branch = ''
182 if len(svn_refs) == 1:
183 # Only one svn branch exists -- seems like a good candidate.
184 svn_branch = svn_refs.values()[0]
185 elif len(svn_refs) > 1:
186 # We have more than one remote branch available. We don't
187 # want to go through all of history, so read a line from the
188 # pipe at a time.
189 # The -100 is an arbitrary limit so we don't search forever.
190 cmd = ['git', 'log', '-100', '--pretty=medium']
191 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
192 for line in proc.stdout:
193 match = git_svn_re.match(line)
194 if match:
195 url = match.group(1)
196 if url in svn_refs:
197 svn_branch = svn_refs[url]
198 proc.stdout.close() # Cut pipe.
199 break
200 return svn_branch
201
202 @staticmethod
203 def FetchUpstreamTuple(cwd):
204 """Returns a tuple containg remote and remote ref,
205 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000206 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000207 """
208 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000209 branch = GIT.GetBranch(cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000210 upstream_branch = None
211 upstream_branch = GIT.Capture(
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000212 ['config', 'branch.%s.merge' % branch], in_directory=cwd,
213 error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000214 if upstream_branch:
215 remote = GIT.Capture(
216 ['config', 'branch.%s.remote' % branch],
nasser@codeaurora.orgb65040a2010-02-01 16:29:14 +0000217 in_directory=cwd, error_ok=True)[0].strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000218 else:
219 # Fall back on trying a git-svn upstream branch.
220 if GIT.IsGitSvn(cwd):
221 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000222 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000223 # Else, try to guess the origin remote.
224 remote_branches = GIT.Capture(
225 ['branch', '-r'], in_directory=cwd)[0].split()
226 if 'origin/master' in remote_branches:
227 # Fall back on origin/master if it exits.
228 remote = 'origin'
229 upstream_branch = 'refs/heads/master'
230 elif 'origin/trunk' in remote_branches:
231 # Fall back on origin/trunk if it exists. Generally a shared
232 # git-svn clone
233 remote = 'origin'
234 upstream_branch = 'refs/heads/trunk'
235 else:
236 # Give up.
237 remote = None
238 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000239 return remote, upstream_branch
240
241 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000242 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000243 """Gets the current branch's upstream branch."""
244 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000245 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000246 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
247 return upstream_branch
248
249 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000250 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
251 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000252 """Diffs against the upstream branch or optionally another branch.
253
254 full_move means that move or copy operations should completely recreate the
255 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000256 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000257 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000258 command = ['diff', '-p', '--no-prefix', branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000259 if not full_move:
260 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000261 # TODO(maruel): --binary support.
262 if files:
263 command.append('--')
264 command.extend(files)
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000265 diff = GIT.Capture(command, cwd)[0].splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000266 for i in range(len(diff)):
267 # In the case of added files, replace /dev/null with the path to the
268 # file being added.
269 if diff[i].startswith('--- /dev/null'):
270 diff[i] = '--- %s' % diff[i+1][4:]
271 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000272
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000273 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000274 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
275 """Returns the list of modified files between two branches."""
276 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000277 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000278 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000279 return GIT.Capture(command, cwd)[0].splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000280
281 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000282 def GetPatchName(cwd):
283 """Constructs a name for this patch."""
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000284 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd)[0].strip()
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000285 return "%s-%s" % (GIT.GetBranch(cwd), short_sha)
286
287 @staticmethod
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000288 def GetCheckoutRoot(path):
289 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000290 """
maruel@chromium.org7be5ef22010-01-30 22:31:50 +0000291 root = GIT.Capture(['rev-parse', '--show-cdup'], path)[0].strip()
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000292 return os.path.abspath(os.path.join(path, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000293
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000294 @staticmethod
295 def AssertVersion(min_version):
296 """Asserts git's version is at least min_version."""
297 def only_int(val):
298 if val.isdigit():
299 return int(val)
300 else:
301 return 0
302 current_version = GIT.Capture(['--version'])[0].split()[-1]
303 current_version_list = map(only_int, current_version.split('.'))
304 for min_ver in map(int, min_version.split('.')):
305 ver = current_version_list.pop(0)
306 if ver < min_ver:
307 return (False, current_version)
308 elif ver > min_ver:
309 return (True, current_version)
310 return (True, current_version)
311
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000312
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000313class SVN(object):
314 COMMAND = "svn"
tony@chromium.org57564662010-04-14 02:35:12 +0000315 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000316
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000317 @staticmethod
318 def Run(args, in_directory):
319 """Runs svn, sending output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000320
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000321 Args:
322 args: A sequence of command line parameters to be passed to svn.
323 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000324
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000325 Raises:
326 Error: An error occurred while running the svn command.
327 """
328 c = [SVN.COMMAND]
329 c.extend(args)
maruel@chromium.org2185f002009-12-18 21:03:47 +0000330 # TODO(maruel): This is very gclient-specific.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000331 gclient_utils.SubprocessCall(c, in_directory)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000332
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000333 @staticmethod
334 def Capture(args, in_directory=None, print_error=True):
335 """Runs svn, capturing output sent to stdout as a string.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000336
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000337 Args:
338 args: A sequence of command line parameters to be passed to svn.
339 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000340
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000341 Returns:
342 The output sent to stdout as a string.
343 """
344 c = [SVN.COMMAND]
345 c.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000346
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000347 # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
348 # the svn.exe executable, but shell=True makes subprocess on Linux fail
349 # when it's called with a list because it only tries to execute the
350 # first string ("svn").
351 stderr = None
352 if not print_error:
353 stderr = subprocess.PIPE
354 return subprocess.Popen(c,
355 cwd=in_directory,
356 shell=(sys.platform == 'win32'),
357 stdout=subprocess.PIPE,
358 stderr=stderr).communicate()[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000359
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000360 @staticmethod
361 def RunAndGetFileList(options, args, in_directory, file_list):
362 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000363
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000364 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000365
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000366 svn's stdout is parsed to collect a list of files checked out or updated.
367 These files are appended to file_list. svn's stdout is also printed to
368 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000369
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000370 Args:
371 options: command line options to gclient
372 args: A sequence of command line parameters to be passed to svn.
373 in_directory: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000374
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000375 Raises:
376 Error: An error occurred while running the svn command.
377 """
378 command = [SVN.COMMAND]
379 command.extend(args)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000380
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000381 # svn update and svn checkout use the same pattern: the first three columns
382 # are for file status, property status, and lock status. This is followed
383 # by two spaces, and then the path to the file.
384 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000385
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000386 # The first three columns of svn status are the same as for svn update and
387 # svn checkout. The next three columns indicate addition-with-history,
388 # switch, and remote lock status. This is followed by one space, and then
389 # the path to the file.
390 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000391
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000392 # args[0] must be a supported command. This will blow up if it's something
393 # else, which is good. Note that the patterns are only effective when
394 # these commands are used in their ordinary forms, the patterns are invalid
395 # for "svn status --show-updates", for example.
396 pattern = {
397 'checkout': update_pattern,
398 'status': status_pattern,
399 'update': update_pattern,
400 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000401 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000402 # Place an upper limit.
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000403 for _ in range(10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000404 previous_list_len = len(file_list)
405 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000406
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000407 def CaptureMatchingLines(line):
408 match = compiled_pattern.search(line)
409 if match:
410 file_list.append(match.group(1))
411 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000412 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000413
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000414 try:
415 SVN.RunAndFilterOutput(args,
416 in_directory,
417 options.verbose,
418 True,
419 CaptureMatchingLines)
420 except gclient_utils.Error:
maruel@chromium.org2de10252010-02-08 01:10:39 +0000421 # We enforce that some progress has been made or HTTP 502.
maruel@chromium.org55e724e2010-03-11 19:36:49 +0000422 if (filter(lambda x: '502 Bad Gateway' in x, failure) or
maruel@chromium.org2de10252010-02-08 01:10:39 +0000423 (len(failure) and len(file_list) > previous_list_len)):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000424 if args[0] == 'checkout':
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000425 # An aborted checkout is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000426 args = ['update'] + args[1:]
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000427 print "Sleeping 15 seconds and retrying...."
428 time.sleep(15)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000429 continue
maruel@chromium.org2de10252010-02-08 01:10:39 +0000430 # No progress was made or an unknown error we aren't sure, bail out.
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000431 raise
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000432 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000433
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000434 @staticmethod
435 def RunAndFilterOutput(args,
436 in_directory,
437 print_messages,
438 print_stdout,
439 filter):
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000440 """Runs a command, optionally outputting to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000441
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000442 stdout is passed line-by-line to the given filter function. If
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000443 print_stdout is true, it is also printed to sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000444
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000445 Args:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000446 args: A sequence of command line parameters to be passed.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000447 in_directory: The directory where svn is to be run.
448 print_messages: Whether to print status messages to stdout about
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000449 which commands are being run.
450 print_stdout: Whether to forward program's output to stdout.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000451 filter: A function taking one argument (a string) which will be
452 passed each line (with the ending newline character removed) of
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000453 program's output for filtering.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000454
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000455 Raises:
maruel@chromium.orgee4071d2009-12-22 22:25:37 +0000456 gclient_utils.Error: An error occurred while running the command.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000457 """
458 command = [SVN.COMMAND]
459 command.extend(args)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000460 gclient_utils.SubprocessCallAndFilter(command,
461 in_directory,
462 print_messages,
463 print_stdout,
464 filter=filter)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000465
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000466 @staticmethod
467 def CaptureInfo(relpath, in_directory=None, print_error=True):
468 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000469
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000470 Args:
471 relpath: The directory where the working copy resides relative to
472 the directory given by in_directory.
473 in_directory: The directory where svn is to be run.
474 """
475 output = SVN.Capture(["info", "--xml", relpath], in_directory, print_error)
476 dom = gclient_utils.ParseXML(output)
477 result = {}
478 if dom:
479 GetNamedNodeText = gclient_utils.GetNamedNodeText
480 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
481 def C(item, f):
482 if item is not None: return f(item)
483 # /info/entry/
484 # url
485 # reposityory/(root|uuid)
486 # wc-info/(schedule|depth)
487 # commit/(author|date)
488 # str() the results because they may be returned as Unicode, which
489 # interferes with the higher layers matching up things in the deps
490 # dictionary.
491 # TODO(maruel): Fix at higher level instead (!)
492 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
493 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
494 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
495 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
496 'revision'),
497 int)
498 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
499 str)
500 # Differs across versions.
501 if result['Node Kind'] == 'dir':
502 result['Node Kind'] = 'directory'
503 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
504 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
505 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
506 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
507 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000508
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000509 @staticmethod
510 def CaptureHeadRevision(url):
511 """Get the head revision of a SVN repository.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000512
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000513 Returns:
514 Int head revision
515 """
516 info = SVN.Capture(["info", "--xml", url], os.getcwd())
517 dom = xml.dom.minidom.parseString(info)
518 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000519
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000520 @staticmethod
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000521 def CaptureBaseRevision(cwd):
522 """Get the base revision of a SVN repository.
523
524 Returns:
525 Int base revision
526 """
527 info = SVN.Capture(["info", "--xml"], cwd)
528 dom = xml.dom.minidom.parseString(info)
529 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
530
531 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000532 def CaptureStatus(files):
533 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000534
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000535 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000536
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000537 Returns an array of (status, file) tuples."""
538 command = ["status", "--xml"]
539 if not files:
540 pass
541 elif isinstance(files, basestring):
542 command.append(files)
543 else:
544 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000545
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000546 status_letter = {
547 None: ' ',
548 '': ' ',
549 'added': 'A',
550 'conflicted': 'C',
551 'deleted': 'D',
552 'external': 'X',
553 'ignored': 'I',
554 'incomplete': '!',
555 'merged': 'G',
556 'missing': '!',
557 'modified': 'M',
558 'none': ' ',
559 'normal': ' ',
560 'obstructed': '~',
561 'replaced': 'R',
562 'unversioned': '?',
563 }
564 dom = gclient_utils.ParseXML(SVN.Capture(command))
565 results = []
566 if dom:
567 # /status/target/entry/(wc-status|commit|author|date)
568 for target in dom.getElementsByTagName('target'):
569 #base_path = target.getAttribute('path')
570 for entry in target.getElementsByTagName('entry'):
571 file_path = entry.getAttribute('path')
572 wc_status = entry.getElementsByTagName('wc-status')
573 assert len(wc_status) == 1
574 # Emulate svn 1.5 status ouput...
575 statuses = [' '] * 7
576 # Col 0
577 xml_item_status = wc_status[0].getAttribute('item')
578 if xml_item_status in status_letter:
579 statuses[0] = status_letter[xml_item_status]
580 else:
581 raise Exception('Unknown item status "%s"; please implement me!' %
582 xml_item_status)
583 # Col 1
584 xml_props_status = wc_status[0].getAttribute('props')
585 if xml_props_status == 'modified':
586 statuses[1] = 'M'
587 elif xml_props_status == 'conflicted':
588 statuses[1] = 'C'
589 elif (not xml_props_status or xml_props_status == 'none' or
590 xml_props_status == 'normal'):
591 pass
592 else:
593 raise Exception('Unknown props status "%s"; please implement me!' %
594 xml_props_status)
595 # Col 2
596 if wc_status[0].getAttribute('wc-locked') == 'true':
597 statuses[2] = 'L'
598 # Col 3
599 if wc_status[0].getAttribute('copied') == 'true':
600 statuses[3] = '+'
601 # Col 4
602 if wc_status[0].getAttribute('switched') == 'true':
603 statuses[4] = 'S'
604 # TODO(maruel): Col 5 and 6
605 item = (''.join(statuses), file_path)
606 results.append(item)
607 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000608
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000609 @staticmethod
610 def IsMoved(filename):
611 """Determine if a file has been added through svn mv"""
612 info = SVN.CaptureInfo(filename)
613 return (info.get('Copied From URL') and
614 info.get('Copied From Rev') and
615 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000616
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000617 @staticmethod
618 def GetFileProperty(file, property_name):
619 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000620
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000621 Args:
622 file: The file to check
623 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000624
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000625 Returns:
626 The value of the property, which will be the empty string if the property
627 is not set on the file. If the file is not under version control, the
628 empty string is also returned.
629 """
630 output = SVN.Capture(["propget", property_name, file])
631 if (output.startswith("svn: ") and
632 output.endswith("is not under version control")):
633 return ""
634 else:
635 return output
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000636
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000637 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000638 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000639 """Diffs a single file.
640
641 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000642 expected relative path.
643 full_move means that move or copy operations should completely recreate the
644 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000645 # Use svn info output instead of os.path.isdir because the latter fails
646 # when the file is deleted.
647 if SVN.CaptureInfo(filename).get("Node Kind") == "directory":
648 return None
649 # If the user specified a custom diff command in their svn config file,
650 # then it'll be used when we do svn diff, which we don't want to happen
651 # since we want the unified diff. Using --diff-cmd=diff doesn't always
652 # work, since they can have another diff executable in their path that
653 # gives different line endings. So we use a bogus temp directory as the
654 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000655 bogus_dir = tempfile.mkdtemp()
656 try:
657 # Grabs the diff data.
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000658 command = ["diff", "--config-dir", bogus_dir, filename]
maruel@chromium.orgce3c8622010-01-07 02:18:16 +0000659 if revision:
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000660 command.extend(['--revision', revision])
maruel@chromium.org0836c562010-01-22 01:10:06 +0000661 if SVN.IsMoved(filename):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000662 if full_move:
663 file_content = gclient_utils.FileRead(filename, 'rb')
664 # Prepend '+' to every lines.
665 file_content = ['+' + i for i in file_content.splitlines(True)]
666 nb_lines = len(file_content)
667 # We need to use / since patch on unix will fail otherwise.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000668 data = "Index: %s\n" % filename
669 data += '=' * 67 + '\n'
670 # Note: Should we use /dev/null instead?
671 data += "--- %s\n" % filename
672 data += "+++ %s\n" % filename
673 data += "@@ -0,0 +1,%d @@\n" % nb_lines
674 data += ''.join(file_content)
675 else:
maruel@chromium.org0836c562010-01-22 01:10:06 +0000676 # svn diff on a mv/cp'd file outputs nothing if there was no change.
677 data = SVN.Capture(command, None)
678 if not data:
679 # We put in an empty Index entry so upload.py knows about them.
680 data = "Index: %s\n" % filename
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000681 else:
maruel@chromium.org0836c562010-01-22 01:10:06 +0000682 data = SVN.Capture(command, None)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000683 finally:
684 shutil.rmtree(bogus_dir)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000685 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000686
687 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000688 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000689 """Returns a string containing the diff for the given file list.
690
691 The files in the list should either be absolute paths or relative to the
692 given root. If no root directory is provided, the repository root will be
693 used.
694 The diff will always use relative paths.
695 """
696 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000697 root = root or SVN.GetCheckoutRoot(previous_cwd)
698 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000699 def RelativePath(path, root):
700 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000701 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000702 return path[len(root):]
703 return path
704 try:
705 os.chdir(root)
706 diff = "".join(filter(None,
maruel@chromium.org0c401692010-01-07 02:44:20 +0000707 [SVN.DiffItem(RelativePath(f, root),
708 full_move=full_move,
709 revision=revision)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000710 for f in filenames]))
711 finally:
712 os.chdir(previous_cwd)
713 return diff
714
715
716 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000717 def GetEmail(repo_root):
718 """Retrieves the svn account which we assume is an email address."""
719 infos = SVN.CaptureInfo(repo_root)
720 uuid = infos.get('UUID')
721 root = infos.get('Repository Root')
722 if not root:
723 return None
724
725 # Should check for uuid but it is incorrectly saved for https creds.
726 realm = root.rsplit('/', 1)[0]
727 if root.startswith('https') or not uuid:
728 regexp = re.compile(r'<%s:\d+>.*' % realm)
729 else:
730 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
731 if regexp is None:
732 return None
733 if sys.platform.startswith('win'):
734 if not 'APPDATA' in os.environ:
735 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000736 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
737 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000738 else:
739 if not 'HOME' in os.environ:
740 return None
741 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
742 'svn.simple')
743 for credfile in os.listdir(auth_dir):
744 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
745 if regexp.match(cred_info.get('svn:realmstring')):
746 return cred_info.get('username')
747
748 @staticmethod
749 def ReadSimpleAuth(filename):
750 f = open(filename, 'r')
751 values = {}
752 def ReadOneItem(type):
753 m = re.match(r'%s (\d+)' % type, f.readline())
754 if not m:
755 return None
756 data = f.read(int(m.group(1)))
757 if f.read(1) != '\n':
758 return None
759 return data
760
761 while True:
762 key = ReadOneItem('K')
763 if not key:
764 break
765 value = ReadOneItem('V')
766 if not value:
767 break
768 values[key] = value
769 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000770
771 @staticmethod
772 def GetCheckoutRoot(directory):
773 """Returns the top level directory of the current repository.
774
775 The directory is returned as an absolute path.
776 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000777 directory = os.path.abspath(directory)
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000778 infos = SVN.CaptureInfo(directory, print_error=False)
779 cur_dir_repo_root = infos.get("Repository Root")
780 if not cur_dir_repo_root:
781 return None
782
783 while True:
784 parent = os.path.dirname(directory)
785 if (SVN.CaptureInfo(parent, print_error=False).get(
786 "Repository Root") != cur_dir_repo_root):
787 break
788 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000789 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000790
791 @staticmethod
792 def AssertVersion(min_version):
793 """Asserts svn's version is at least min_version."""
794 def only_int(val):
795 if val.isdigit():
796 return int(val)
797 else:
798 return 0
799 if not SVN.current_version:
800 SVN.current_version = SVN.Capture(['--version']).split()[2]
801 current_version_list = map(only_int, SVN.current_version.split('.'))
802 for min_ver in map(int, min_version.split('.')):
803 ver = current_version_list.pop(0)
804 if ver < min_ver:
805 return (False, SVN.current_version)
806 elif ver > min_ver:
807 return (True, SVN.current_version)
808 return (True, SVN.current_version)