blob: 5b2c8eafa10277241d8fd41eeb54db3bfbff8c77 [file] [log] [blame]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00001# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00005"""SCM-specific utility classes."""
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00006
maruel@chromium.org3c55d982010-05-06 14:25:44 +00007import cStringIO
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +00008import glob
maruel@chromium.orgd5800f12009-11-12 20:03:43 +00009import os
10import re
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +000011import shutil
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000012import subprocess
13import sys
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000014import tempfile
maruel@chromium.orgfd876172010-04-30 14:01:05 +000015import time
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000016import xml.dom.minidom
17
18import gclient_utils
19
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000020def ValidateEmail(email):
maruel@chromium.org6e29d572010-06-04 17:32:20 +000021 return (re.match(r"^[a-zA-Z0-9._%-+]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email)
22 is not None)
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +000023
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000024
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +000025def GetCasedPath(path):
26 """Elcheapos way to get the real path case on Windows."""
27 if sys.platform.startswith('win') and os.path.exists(path):
28 # Reconstruct the path.
29 path = os.path.abspath(path)
30 paths = path.split('\\')
31 for i in range(len(paths)):
32 if i == 0:
33 # Skip drive letter.
34 continue
35 subpath = '\\'.join(paths[:i+1])
36 prev = len('\\'.join(paths[:i]))
37 # glob.glob will return the cased path for the last item only. This is why
38 # we are calling it in a loop. Extract the data we want and put it back
39 # into the list.
40 paths[i] = glob.glob(subpath + '*')[0][prev+1:len(subpath)]
41 path = '\\'.join(paths)
42 return path
43
44
maruel@chromium.org3c55d982010-05-06 14:25:44 +000045def GenFakeDiff(filename):
46 """Generates a fake diff from a file."""
47 file_content = gclient_utils.FileRead(filename, 'rb').splitlines(True)
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +000048 filename = filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +000049 nb_lines = len(file_content)
50 # We need to use / since patch on unix will fail otherwise.
51 data = cStringIO.StringIO()
52 data.write("Index: %s\n" % filename)
53 data.write('=' * 67 + '\n')
54 # Note: Should we use /dev/null instead?
55 data.write("--- %s\n" % filename)
56 data.write("+++ %s\n" % filename)
57 data.write("@@ -0,0 +1,%d @@\n" % nb_lines)
58 # Prepend '+' to every lines.
59 for line in file_content:
60 data.write('+')
61 data.write(line)
62 result = data.getvalue()
63 data.close()
64 return result
65
66
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000067class GIT(object):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000068 @staticmethod
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +000069 def Capture(args, **kwargs):
70 return gclient_utils.CheckCall(['git'] + args, print_error=False,
71 **kwargs)[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000072
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000073 @staticmethod
msb@chromium.org786fb682010-06-02 15:16:23 +000074 def CaptureStatus(files, upstream_branch=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000075 """Returns git status.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000076
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000077 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +000078
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000079 Returns an array of (status, file) tuples."""
msb@chromium.org786fb682010-06-02 15:16:23 +000080 if upstream_branch is None:
81 upstream_branch = GIT.GetUpstreamBranch(os.getcwd())
82 if upstream_branch is None:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +000083 raise gclient_utils.Error('Cannot determine upstream branch')
84 command = ['diff', '--name-status', '-r', '%s...' % upstream_branch]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000085 if not files:
86 pass
87 elif isinstance(files, basestring):
88 command.append(files)
89 else:
90 command.extend(files)
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +000091 status = GIT.Capture(command).rstrip()
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000092 results = []
93 if status:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +000094 for statusline in status.splitlines():
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000095 m = re.match('^(\w)\t(.+)$', statusline)
96 if not m:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +000097 raise gclient_utils.Error(
98 'status currently unsupported: %s' % statusline)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000099 results.append(('%s ' % m.group(1), m.group(2)))
100 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000101
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000102 @staticmethod
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000103 def GetEmail(cwd):
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000104 """Retrieves the user email address if known."""
105 # We could want to look at the svn cred when it has a svn remote but it
106 # should be fine for now, users should simply configure their git settings.
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000107 try:
108 return GIT.Capture(['config', 'user.email'], cwd=cwd).strip()
109 except gclient_utils.CheckCallError:
110 return ''
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000111
112 @staticmethod
113 def ShortBranchName(branch):
114 """Converts a name like 'refs/heads/foo' to just 'foo'."""
115 return branch.replace('refs/heads/', '')
116
117 @staticmethod
118 def GetBranchRef(cwd):
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000119 """Returns the full branch reference, e.g. 'refs/heads/master'."""
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000120 return GIT.Capture(['symbolic-ref', 'HEAD'], cwd=cwd).strip()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000121
122 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000123 def GetBranch(cwd):
124 """Returns the short branch name, e.g. 'master'."""
maruel@chromium.orgc308a742009-12-22 18:29:33 +0000125 return GIT.ShortBranchName(GIT.GetBranchRef(cwd))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000126
127 @staticmethod
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000128 def IsGitSvn(cwd):
129 """Returns true if this repo looks like it's using git-svn."""
130 # If you have any "svn-remote.*" config keys, we think you're using svn.
131 try:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000132 GIT.Capture(['config', '--get-regexp', r'^svn-remote\.'], cwd=cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000133 return True
134 except gclient_utils.CheckCallError:
135 return False
136
137 @staticmethod
138 def GetSVNBranch(cwd):
139 """Returns the svn branch name if found."""
140 # Try to figure out which remote branch we're based on.
141 # Strategy:
142 # 1) find all git-svn branches and note their svn URLs.
143 # 2) iterate through our branch history and match up the URLs.
144
145 # regexp matching the git-svn line that contains the URL.
146 git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
147
148 # Get the refname and svn url for all refs/remotes/*.
149 remotes = GIT.Capture(
150 ['for-each-ref', '--format=%(refname)', 'refs/remotes'],
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000151 cwd=cwd).splitlines()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000152 svn_refs = {}
153 for ref in remotes:
154 match = git_svn_re.search(
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000155 GIT.Capture(['cat-file', '-p', ref], cwd=cwd))
sky@chromium.org42d8da52010-04-23 18:25:07 +0000156 # Prefer origin/HEAD over all others.
157 if match and (match.group(1) not in svn_refs or
158 ref == "refs/remotes/origin/HEAD"):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000159 svn_refs[match.group(1)] = ref
160
161 svn_branch = ''
162 if len(svn_refs) == 1:
163 # Only one svn branch exists -- seems like a good candidate.
164 svn_branch = svn_refs.values()[0]
165 elif len(svn_refs) > 1:
166 # We have more than one remote branch available. We don't
167 # want to go through all of history, so read a line from the
168 # pipe at a time.
169 # The -100 is an arbitrary limit so we don't search forever.
170 cmd = ['git', 'log', '-100', '--pretty=medium']
maruel@chromium.org3a292682010-08-23 18:54:55 +0000171 proc = gclient_utils.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000172 for line in proc.stdout:
173 match = git_svn_re.match(line)
174 if match:
175 url = match.group(1)
176 if url in svn_refs:
177 svn_branch = svn_refs[url]
178 proc.stdout.close() # Cut pipe.
179 break
180 return svn_branch
181
182 @staticmethod
183 def FetchUpstreamTuple(cwd):
184 """Returns a tuple containg remote and remote ref,
185 e.g. 'origin', 'refs/heads/master'
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000186 Tries to be intelligent and understand git-svn.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000187 """
188 remote = '.'
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000189 branch = GIT.GetBranch(cwd)
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000190 try:
191 upstream_branch = GIT.Capture(
192 ['config', 'branch.%s.merge' % branch], cwd=cwd).strip()
193 except gclient_utils.Error:
194 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000195 if upstream_branch:
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000196 try:
197 remote = GIT.Capture(
198 ['config', 'branch.%s.remote' % branch], cwd=cwd).strip()
199 except gclient_utils.Error:
200 pass
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000201 else:
202 # Fall back on trying a git-svn upstream branch.
203 if GIT.IsGitSvn(cwd):
204 upstream_branch = GIT.GetSVNBranch(cwd)
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000205 else:
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000206 # Else, try to guess the origin remote.
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000207 remote_branches = GIT.Capture(['branch', '-r'], cwd=cwd).split()
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000208 if 'origin/master' in remote_branches:
209 # Fall back on origin/master if it exits.
210 remote = 'origin'
211 upstream_branch = 'refs/heads/master'
212 elif 'origin/trunk' in remote_branches:
213 # Fall back on origin/trunk if it exists. Generally a shared
214 # git-svn clone
215 remote = 'origin'
216 upstream_branch = 'refs/heads/trunk'
217 else:
218 # Give up.
219 remote = None
220 upstream_branch = None
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000221 return remote, upstream_branch
222
223 @staticmethod
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000224 def GetUpstreamBranch(cwd):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000225 """Gets the current branch's upstream branch."""
226 remote, upstream_branch = GIT.FetchUpstreamTuple(cwd)
maruel@chromium.orga630bd72010-04-29 23:32:34 +0000227 if remote != '.' and upstream_branch:
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000228 upstream_branch = upstream_branch.replace('heads', 'remotes/' + remote)
229 return upstream_branch
230
231 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000232 def GenerateDiff(cwd, branch=None, branch_head='HEAD', full_move=False,
233 files=None):
maruel@chromium.orga9371762009-12-22 18:27:38 +0000234 """Diffs against the upstream branch or optionally another branch.
235
236 full_move means that move or copy operations should completely recreate the
237 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000238 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000239 branch = GIT.GetUpstreamBranch(cwd)
evan@chromium.org400f3e72010-05-19 14:23:36 +0000240 command = ['diff', '-p', '--no-prefix', '--no-ext-diff',
241 branch + "..." + branch_head]
maruel@chromium.orga9371762009-12-22 18:27:38 +0000242 if not full_move:
243 command.append('-C')
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000244 # TODO(maruel): --binary support.
245 if files:
246 command.append('--')
247 command.extend(files)
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000248 diff = GIT.Capture(command, cwd=cwd).splitlines(True)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000249 for i in range(len(diff)):
250 # In the case of added files, replace /dev/null with the path to the
251 # file being added.
252 if diff[i].startswith('--- /dev/null'):
253 diff[i] = '--- %s' % diff[i+1][4:]
254 return ''.join(diff)
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000255
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000256 @staticmethod
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000257 def GetDifferentFiles(cwd, branch=None, branch_head='HEAD'):
258 """Returns the list of modified files between two branches."""
259 if not branch:
maruel@chromium.org81e012c2010-04-29 16:07:24 +0000260 branch = GIT.GetUpstreamBranch(cwd)
bauerb@chromium.org838f0f22010-04-09 17:02:50 +0000261 command = ['diff', '--name-only', branch + "..." + branch_head]
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000262 return GIT.Capture(command, cwd=cwd).splitlines(False)
maruel@chromium.org8ede00e2010-01-12 14:35:28 +0000263
264 @staticmethod
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000265 def GetPatchName(cwd):
266 """Constructs a name for this patch."""
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000267 short_sha = GIT.Capture(['rev-parse', '--short=4', 'HEAD'], cwd=cwd).strip()
maruel@chromium.org862ff8e2010-08-06 15:29:16 +0000268 return "%s#%s" % (GIT.GetBranch(cwd), short_sha)
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000269
270 @staticmethod
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000271 def GetCheckoutRoot(cwd):
maruel@chromium.org01d8c1d2010-01-07 01:56:59 +0000272 """Returns the top level directory of a git checkout as an absolute path.
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000273 """
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000274 root = GIT.Capture(['rev-parse', '--show-cdup'], cwd=cwd).strip()
275 return os.path.abspath(os.path.join(cwd, root))
maruel@chromium.orgb24a8e12009-12-22 13:45:48 +0000276
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000277 @staticmethod
278 def AssertVersion(min_version):
279 """Asserts git's version is at least min_version."""
280 def only_int(val):
281 if val.isdigit():
282 return int(val)
283 else:
284 return 0
maruel@chromium.orgad80e3b2010-09-09 14:18:28 +0000285 current_version = GIT.Capture(['--version']).split()[-1]
maruel@chromium.orgd0f854a2010-03-11 19:35:53 +0000286 current_version_list = map(only_int, current_version.split('.'))
287 for min_ver in map(int, min_version.split('.')):
288 ver = current_version_list.pop(0)
289 if ver < min_ver:
290 return (False, current_version)
291 elif ver > min_ver:
292 return (True, current_version)
293 return (True, current_version)
294
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000295
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000296class SVN(object):
tony@chromium.org57564662010-04-14 02:35:12 +0000297 current_version = None
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000298
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000299 @staticmethod
maruel@chromium.org54019f32010-09-09 13:50:11 +0000300 def Capture(args, **kwargs):
301 """Always redirect stderr.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000302
maruel@chromium.org54019f32010-09-09 13:50:11 +0000303 Throws an exception if non-0 is returned."""
304 return gclient_utils.CheckCall(['svn'] + args, print_error=False,
305 **kwargs)[0]
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000306
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000307 @staticmethod
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000308 def RunAndGetFileList(verbose, args, cwd, file_list, stdout=None):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000309 """Runs svn checkout, update, or status, output to stdout.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000310
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000311 The first item in args must be either "checkout", "update", or "status".
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000312
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000313 svn's stdout is parsed to collect a list of files checked out or updated.
314 These files are appended to file_list. svn's stdout is also printed to
315 sys.stdout as in Run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000316
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000317 Args:
maruel@chromium.org03807072010-08-16 17:18:44 +0000318 verbose: If True, uses verbose output
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000319 args: A sequence of command line parameters to be passed to svn.
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000320 cwd: The directory where svn is to be run.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000321
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000322 Raises:
323 Error: An error occurred while running the svn command.
324 """
maruel@chromium.org2b9aa8e2010-08-25 20:01:42 +0000325 stdout = stdout or sys.stdout
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000326
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000327 # svn update and svn checkout use the same pattern: the first three columns
328 # are for file status, property status, and lock status. This is followed
329 # by two spaces, and then the path to the file.
330 update_pattern = '^... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000331
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000332 # The first three columns of svn status are the same as for svn update and
333 # svn checkout. The next three columns indicate addition-with-history,
334 # switch, and remote lock status. This is followed by one space, and then
335 # the path to the file.
336 status_pattern = '^...... (.*)$'
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000337
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000338 # args[0] must be a supported command. This will blow up if it's something
339 # else, which is good. Note that the patterns are only effective when
340 # these commands are used in their ordinary forms, the patterns are invalid
341 # for "svn status --show-updates", for example.
342 pattern = {
343 'checkout': update_pattern,
344 'status': status_pattern,
345 'update': update_pattern,
346 }[args[0]]
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000347 compiled_pattern = re.compile(pattern)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000348 # Place an upper limit.
cbentzel@chromium.org2aee2292010-09-03 14:15:25 +0000349 backoff_time = 5
maruel@chromium.orgfd876172010-04-30 14:01:05 +0000350 for _ in range(10):
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000351 previous_list_len = len(file_list)
352 failure = []
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000353
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000354 def CaptureMatchingLines(line):
355 match = compiled_pattern.search(line)
356 if match:
357 file_list.append(match.group(1))
358 if line.startswith('svn: '):
maruel@chromium.org8599aa72010-02-08 20:27:14 +0000359 failure.append(line)
maruel@chromium.org54d1f1a2010-01-08 19:53:47 +0000360
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000361 try:
maruel@chromium.org17d01792010-09-01 18:07:10 +0000362 gclient_utils.CheckCallAndFilterAndHeader(
363 ['svn'] + args,
364 cwd=cwd,
365 always=verbose,
366 filter_fn=CaptureMatchingLines,
367 stdout=stdout)
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000368 except gclient_utils.Error:
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000369 def IsKnownFailure():
370 for x in failure:
371 if (x.startswith('svn: OPTIONS of') or
372 x.startswith('svn: PROPFIND of') or
373 x.startswith('svn: REPORT of') or
maruel@chromium.orgf61fc932010-08-19 13:05:24 +0000374 x.startswith('svn: Unknown hostname') or
375 x.startswith('svn: Server sent unexpected return value')):
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000376 return True
377 return False
378
maruel@chromium.org953586a2010-06-15 14:22:24 +0000379 # Subversion client is really misbehaving with Google Code.
380 if args[0] == 'checkout':
381 # Ensure at least one file was checked out, otherwise *delete* the
382 # directory.
383 if len(file_list) == previous_list_len:
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000384 if not IsKnownFailure():
maruel@chromium.org953586a2010-06-15 14:22:24 +0000385 # No known svn error was found, bail out.
386 raise
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000387 # No file were checked out, so make sure the directory is
388 # deleted in case it's messed up and try again.
389 # Warning: It's bad, it assumes args[2] is the directory
390 # argument.
391 if os.path.isdir(args[2]):
392 gclient_utils.RemoveDirectory(args[2])
maruel@chromium.org953586a2010-06-15 14:22:24 +0000393 else:
394 # Progress was made, convert to update since an aborted checkout
395 # is now an update.
maruel@chromium.org2de10252010-02-08 01:10:39 +0000396 args = ['update'] + args[1:]
maruel@chromium.org953586a2010-06-15 14:22:24 +0000397 else:
398 # It was an update or export.
maruel@chromium.org6133c5b2010-08-18 18:34:48 +0000399 # We enforce that some progress has been made or a known failure.
400 if len(file_list) == previous_list_len and not IsKnownFailure():
401 # No known svn error was found and no progress, bail out.
402 raise
cbentzel@chromium.org2aee2292010-09-03 14:15:25 +0000403 print "Sleeping %.1f seconds and retrying...." % backoff_time
404 time.sleep(backoff_time)
405 backoff_time *= 1.3
maruel@chromium.org953586a2010-06-15 14:22:24 +0000406 continue
maruel@chromium.orgb71b67e2009-11-24 20:48:19 +0000407 break
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000408
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000409 @staticmethod
maruel@chromium.org54019f32010-09-09 13:50:11 +0000410 def CaptureInfo(cwd):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000411 """Returns a dictionary from the svn info output for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000412
maruel@chromium.org54019f32010-09-09 13:50:11 +0000413 Throws an exception if svn info fails."""
414 output = SVN.Capture(['info', '--xml', cwd])
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000415 dom = gclient_utils.ParseXML(output)
416 result = {}
417 if dom:
418 GetNamedNodeText = gclient_utils.GetNamedNodeText
419 GetNodeNamedAttributeText = gclient_utils.GetNodeNamedAttributeText
420 def C(item, f):
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000421 if item is not None:
422 return f(item)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000423 # /info/entry/
424 # url
425 # reposityory/(root|uuid)
426 # wc-info/(schedule|depth)
427 # commit/(author|date)
428 # str() the results because they may be returned as Unicode, which
429 # interferes with the higher layers matching up things in the deps
430 # dictionary.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000431 result['Repository Root'] = C(GetNamedNodeText(dom, 'root'), str)
432 result['URL'] = C(GetNamedNodeText(dom, 'url'), str)
433 result['UUID'] = C(GetNamedNodeText(dom, 'uuid'), str)
434 result['Revision'] = C(GetNodeNamedAttributeText(dom, 'entry',
435 'revision'),
436 int)
437 result['Node Kind'] = C(GetNodeNamedAttributeText(dom, 'entry', 'kind'),
438 str)
439 # Differs across versions.
440 if result['Node Kind'] == 'dir':
441 result['Node Kind'] = 'directory'
442 result['Schedule'] = C(GetNamedNodeText(dom, 'schedule'), str)
443 result['Path'] = C(GetNodeNamedAttributeText(dom, 'entry', 'path'), str)
444 result['Copied From URL'] = C(GetNamedNodeText(dom, 'copy-from-url'), str)
445 result['Copied From Rev'] = C(GetNamedNodeText(dom, 'copy-from-rev'), str)
446 return result
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000447
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000448 @staticmethod
maruel@chromium.org54019f32010-09-09 13:50:11 +0000449 def CaptureRevision(cwd):
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000450 """Get the base revision of a SVN repository.
451
452 Returns:
453 Int base revision
454 """
maruel@chromium.org54019f32010-09-09 13:50:11 +0000455 info = SVN.Capture(['info', '--xml'], cwd=cwd)
nasser@codeaurora.org5d63eb82010-03-24 23:22:09 +0000456 dom = xml.dom.minidom.parseString(info)
457 return dom.getElementsByTagName('entry')[0].getAttribute('revision')
458
459 @staticmethod
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000460 def CaptureStatus(files):
461 """Returns the svn 1.5 svn status emulated output.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000462
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000463 @files can be a string (one file) or a list of files.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000464
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000465 Returns an array of (status, file) tuples."""
466 command = ["status", "--xml"]
467 if not files:
468 pass
469 elif isinstance(files, basestring):
470 command.append(files)
471 else:
472 command.extend(files)
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000473
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000474 status_letter = {
475 None: ' ',
476 '': ' ',
477 'added': 'A',
478 'conflicted': 'C',
479 'deleted': 'D',
480 'external': 'X',
481 'ignored': 'I',
482 'incomplete': '!',
483 'merged': 'G',
484 'missing': '!',
485 'modified': 'M',
486 'none': ' ',
487 'normal': ' ',
488 'obstructed': '~',
489 'replaced': 'R',
490 'unversioned': '?',
491 }
492 dom = gclient_utils.ParseXML(SVN.Capture(command))
493 results = []
494 if dom:
495 # /status/target/entry/(wc-status|commit|author|date)
496 for target in dom.getElementsByTagName('target'):
497 #base_path = target.getAttribute('path')
498 for entry in target.getElementsByTagName('entry'):
499 file_path = entry.getAttribute('path')
500 wc_status = entry.getElementsByTagName('wc-status')
501 assert len(wc_status) == 1
502 # Emulate svn 1.5 status ouput...
503 statuses = [' '] * 7
504 # Col 0
505 xml_item_status = wc_status[0].getAttribute('item')
506 if xml_item_status in status_letter:
507 statuses[0] = status_letter[xml_item_status]
508 else:
maruel@chromium.org54019f32010-09-09 13:50:11 +0000509 raise gclient_utils.Error(
510 'Unknown item status "%s"; please implement me!' %
511 xml_item_status)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000512 # Col 1
513 xml_props_status = wc_status[0].getAttribute('props')
514 if xml_props_status == 'modified':
515 statuses[1] = 'M'
516 elif xml_props_status == 'conflicted':
517 statuses[1] = 'C'
518 elif (not xml_props_status or xml_props_status == 'none' or
519 xml_props_status == 'normal'):
520 pass
521 else:
maruel@chromium.org54019f32010-09-09 13:50:11 +0000522 raise gclient_utils.Error(
523 'Unknown props status "%s"; please implement me!' %
524 xml_props_status)
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000525 # Col 2
526 if wc_status[0].getAttribute('wc-locked') == 'true':
527 statuses[2] = 'L'
528 # Col 3
529 if wc_status[0].getAttribute('copied') == 'true':
530 statuses[3] = '+'
531 # Col 4
532 if wc_status[0].getAttribute('switched') == 'true':
533 statuses[4] = 'S'
534 # TODO(maruel): Col 5 and 6
535 item = (''.join(statuses), file_path)
536 results.append(item)
537 return results
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000538
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000539 @staticmethod
540 def IsMoved(filename):
541 """Determine if a file has been added through svn mv"""
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000542 return SVN.IsMovedInfo(SVN.CaptureInfo(filename))
543
544 @staticmethod
545 def IsMovedInfo(info):
546 """Determine if a file has been added through svn mv"""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000547 return (info.get('Copied From URL') and
548 info.get('Copied From Rev') and
549 info.get('Schedule') == 'add')
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000550
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000551 @staticmethod
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000552 def GetFileProperty(filename, property_name):
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000553 """Returns the value of an SVN property for the given file.
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000554
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000555 Args:
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000556 filename: The file to check
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000557 property_name: The name of the SVN property, e.g. "svn:mime-type"
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000558
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000559 Returns:
560 The value of the property, which will be the empty string if the property
561 is not set on the file. If the file is not under version control, the
562 empty string is also returned.
563 """
maruel@chromium.org54019f32010-09-09 13:50:11 +0000564 try:
565 return SVN.Capture(['propget', property_name, filename])
566 except gclient_utils.Error:
567 return ''
maruel@chromium.orgd5800f12009-11-12 20:03:43 +0000568
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000569 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000570 def DiffItem(filename, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000571 """Diffs a single file.
572
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000573 Should be simple, eh? No it isn't.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000574 Be sure to be in the appropriate directory before calling to have the
maruel@chromium.orga9371762009-12-22 18:27:38 +0000575 expected relative path.
576 full_move means that move or copy operations should completely recreate the
577 files, usually in the prospect to apply the patch for a try job."""
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000578 # If the user specified a custom diff command in their svn config file,
579 # then it'll be used when we do svn diff, which we don't want to happen
580 # since we want the unified diff. Using --diff-cmd=diff doesn't always
581 # work, since they can have another diff executable in their path that
582 # gives different line endings. So we use a bogus temp directory as the
583 # config directory, which gets around these problems.
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000584 bogus_dir = tempfile.mkdtemp()
585 try:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000586 # Use "svn info" output instead of os.path.isdir because the latter fails
587 # when the file is deleted.
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000588 return SVN._DiffItemInternal(filename, SVN.CaptureInfo(filename),
589 bogus_dir,
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000590 full_move=full_move, revision=revision)
591 finally:
592 shutil.rmtree(bogus_dir)
593
594 @staticmethod
595 def _DiffItemInternal(filename, info, bogus_dir, full_move=False,
596 revision=None):
597 """Grabs the diff data."""
598 command = ["diff", "--config-dir", bogus_dir, filename]
599 if revision:
600 command.extend(['--revision', revision])
601 data = None
602 if SVN.IsMovedInfo(info):
603 if full_move:
604 if info.get("Node Kind") == "directory":
605 # Things become tricky here. It's a directory copy/move. We need to
606 # diff all the files inside it.
607 # This will put a lot of pressure on the heap. This is why StringIO
608 # is used and converted back into a string at the end. The reason to
609 # return a string instead of a StringIO is that StringIO.write()
610 # doesn't accept a StringIO object. *sigh*.
611 for (dirpath, dirnames, filenames) in os.walk(filename):
612 # Cleanup all files starting with a '.'.
613 for d in dirnames:
614 if d.startswith('.'):
615 dirnames.remove(d)
616 for f in filenames:
617 if f.startswith('.'):
618 filenames.remove(f)
619 for f in filenames:
620 if data is None:
621 data = cStringIO.StringIO()
622 data.write(GenFakeDiff(os.path.join(dirpath, f)))
623 if data:
624 tmp = data.getvalue()
625 data.close()
626 data = tmp
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000627 else:
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000628 data = GenFakeDiff(filename)
629 else:
630 if info.get("Node Kind") != "directory":
maruel@chromium.org0836c562010-01-22 01:10:06 +0000631 # svn diff on a mv/cp'd file outputs nothing if there was no change.
maruel@chromium.org54019f32010-09-09 13:50:11 +0000632 data = SVN.Capture(command)
maruel@chromium.org0836c562010-01-22 01:10:06 +0000633 if not data:
634 # We put in an empty Index entry so upload.py knows about them.
maruel@chromium.orgc6d170e2010-06-03 00:06:00 +0000635 data = "Index: %s\n" % filename.replace(os.sep, '/')
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000636 # Otherwise silently ignore directories.
637 else:
638 if info.get("Node Kind") != "directory":
639 # Normal simple case.
maruel@chromium.org54019f32010-09-09 13:50:11 +0000640 data = SVN.Capture(command)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000641 # Otherwise silently ignore directories.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000642 return data
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000643
644 @staticmethod
maruel@chromium.org1c7db8e2010-01-07 02:00:19 +0000645 def GenerateDiff(filenames, root=None, full_move=False, revision=None):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000646 """Returns a string containing the diff for the given file list.
647
648 The files in the list should either be absolute paths or relative to the
649 given root. If no root directory is provided, the repository root will be
650 used.
651 The diff will always use relative paths.
652 """
653 previous_cwd = os.getcwd()
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000654 root = root or SVN.GetCheckoutRoot(previous_cwd)
655 root = os.path.normcase(os.path.join(root, ''))
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000656 def RelativePath(path, root):
657 """We must use relative paths."""
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000658 if os.path.normcase(path).startswith(root):
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000659 return path[len(root):]
660 return path
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000661 # If the user specified a custom diff command in their svn config file,
662 # then it'll be used when we do svn diff, which we don't want to happen
663 # since we want the unified diff. Using --diff-cmd=diff doesn't always
664 # work, since they can have another diff executable in their path that
665 # gives different line endings. So we use a bogus temp directory as the
666 # config directory, which gets around these problems.
667 bogus_dir = tempfile.mkdtemp()
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000668 try:
669 os.chdir(root)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000670 # Cleanup filenames
671 filenames = [RelativePath(f, root) for f in filenames]
672 # Get information about the modified items (files and directories)
673 data = dict([(f, SVN.CaptureInfo(f)) for f in filenames])
gavinp@google.com3fda4cc2010-06-29 13:29:27 +0000674 diffs = []
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000675 if full_move:
676 # Eliminate modified files inside moved/copied directory.
677 for (filename, info) in data.iteritems():
678 if SVN.IsMovedInfo(info) and info.get("Node Kind") == "directory":
679 # Remove files inside the directory.
680 filenames = [f for f in filenames
681 if not f.startswith(filename + os.path.sep)]
682 for filename in data.keys():
683 if not filename in filenames:
684 # Remove filtered out items.
685 del data[filename]
gavinp@google.com3fda4cc2010-06-29 13:29:27 +0000686 else:
687 metaheaders = []
688 for (filename, info) in data.iteritems():
689 if SVN.IsMovedInfo(info):
690 # for now, the most common case is a head copy,
691 # so let's just encode that as a straight up cp.
692 srcurl = info.get('Copied From URL')
693 root = info.get('Repository Root')
694 rev = int(info.get('Copied From Rev'))
695 assert srcurl.startswith(root)
696 src = srcurl[len(root)+1:]
697 srcinfo = SVN.CaptureInfo(srcurl)
698 if (srcinfo.get('Revision') != rev and
699 SVN.Capture(['diff', '-r', '%d:head' % rev, srcurl])):
700 metaheaders.append("#$ svn cp -r %d %s %s "
701 "### WARNING: note non-trunk copy\n" %
702 (rev, src, filename))
703 else:
704 metaheaders.append("#$ cp %s %s\n" % (src,
705 filename))
706
707 if metaheaders:
708 diffs.append("### BEGIN SVN COPY METADATA\n")
709 diffs.extend(metaheaders)
710 diffs.append("### END SVN COPY METADATA\n")
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000711 # Now ready to do the actual diff.
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000712 for filename in sorted(data.iterkeys()):
713 diffs.append(SVN._DiffItemInternal(filename, data[filename], bogus_dir,
714 full_move=full_move,
715 revision=revision))
716 # Use StringIO since it can be messy when diffing a directory move with
717 # full_move=True.
718 buf = cStringIO.StringIO()
719 for d in filter(None, diffs):
720 buf.write(d)
721 result = buf.getvalue()
722 buf.close()
723 return result
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000724 finally:
725 os.chdir(previous_cwd)
maruel@chromium.org3c55d982010-05-06 14:25:44 +0000726 shutil.rmtree(bogus_dir)
maruel@chromium.orgf2f9d552009-12-22 00:12:57 +0000727
728 @staticmethod
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000729 def GetEmail(repo_root):
730 """Retrieves the svn account which we assume is an email address."""
maruel@chromium.org54019f32010-09-09 13:50:11 +0000731 try:
732 infos = SVN.CaptureInfo(repo_root)
733 except gclient_utils.Error:
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000734 return None
735
736 # Should check for uuid but it is incorrectly saved for https creds.
maruel@chromium.org54019f32010-09-09 13:50:11 +0000737 root = infos['Repository Root']
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000738 realm = root.rsplit('/', 1)[0]
maruel@chromium.org54019f32010-09-09 13:50:11 +0000739 uuid = infos['UUID']
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000740 if root.startswith('https') or not uuid:
741 regexp = re.compile(r'<%s:\d+>.*' % realm)
742 else:
743 regexp = re.compile(r'<%s:\d+> %s' % (realm, uuid))
744 if regexp is None:
745 return None
746 if sys.platform.startswith('win'):
747 if not 'APPDATA' in os.environ:
748 return None
maruel@chromium.org720d9f32009-11-21 17:38:57 +0000749 auth_dir = os.path.join(os.environ['APPDATA'], 'Subversion', 'auth',
750 'svn.simple')
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000751 else:
752 if not 'HOME' in os.environ:
753 return None
754 auth_dir = os.path.join(os.environ['HOME'], '.subversion', 'auth',
755 'svn.simple')
756 for credfile in os.listdir(auth_dir):
757 cred_info = SVN.ReadSimpleAuth(os.path.join(auth_dir, credfile))
758 if regexp.match(cred_info.get('svn:realmstring')):
759 return cred_info.get('username')
760
761 @staticmethod
762 def ReadSimpleAuth(filename):
763 f = open(filename, 'r')
764 values = {}
maruel@chromium.org6e29d572010-06-04 17:32:20 +0000765 def ReadOneItem(item_type):
766 m = re.match(r'%s (\d+)' % item_type, f.readline())
maruel@chromium.orgc78f2462009-11-21 01:20:57 +0000767 if not m:
768 return None
769 data = f.read(int(m.group(1)))
770 if f.read(1) != '\n':
771 return None
772 return data
773
774 while True:
775 key = ReadOneItem('K')
776 if not key:
777 break
778 value = ReadOneItem('V')
779 if not value:
780 break
781 values[key] = value
782 return values
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000783
784 @staticmethod
785 def GetCheckoutRoot(directory):
786 """Returns the top level directory of the current repository.
787
788 The directory is returned as an absolute path.
789 """
maruel@chromium.orgf7ae6d52009-12-22 20:49:04 +0000790 directory = os.path.abspath(directory)
maruel@chromium.org54019f32010-09-09 13:50:11 +0000791 try:
792 cur_dir_repo_root = SVN.CaptureInfo(directory)['Repository Root']
793 except gclient_utils.Error:
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000794 return None
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000795 while True:
796 parent = os.path.dirname(directory)
maruel@chromium.org54019f32010-09-09 13:50:11 +0000797 try:
798 if SVN.CaptureInfo(parent)['Repository Root'] != cur_dir_repo_root:
799 break
800 except gclient_utils.Error:
maruel@chromium.org94b1ee92009-12-19 20:27:20 +0000801 break
802 directory = parent
maruel@chromium.orgfd9cbbb2010-01-08 23:04:03 +0000803 return GetCasedPath(directory)
tony@chromium.org57564662010-04-14 02:35:12 +0000804
805 @staticmethod
806 def AssertVersion(min_version):
807 """Asserts svn's version is at least min_version."""
808 def only_int(val):
809 if val.isdigit():
810 return int(val)
811 else:
812 return 0
813 if not SVN.current_version:
814 SVN.current_version = SVN.Capture(['--version']).split()[2]
815 current_version_list = map(only_int, SVN.current_version.split('.'))
816 for min_ver in map(int, min_version.split('.')):
817 ver = current_version_list.pop(0)
818 if ver < min_ver:
819 return (False, SVN.current_version)
820 elif ver > min_ver:
821 return (True, SVN.current_version)
822 return (True, SVN.current_version)