blob: 6fb32d5107971c0d8970331ca2b9565d451b31fc [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
maruel@chromium.org9799a072012-01-11 00:26:25 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000010import ConfigParser
11import fnmatch
12import logging
13import os
14import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000015import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000016import subprocess
17import sys
18import tempfile
19
20import patch
21import scm
22import subprocess2
23
24
25def get_code_review_setting(path, key,
26 codereview_settings_file='codereview.settings'):
27 """Parses codereview.settings and return the value for the key if present.
28
29 Don't cache the values in case the file is changed."""
30 # TODO(maruel): Do not duplicate code.
31 settings = {}
32 try:
33 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
34 try:
35 for line in settings_file.readlines():
36 if not line or line.startswith('#'):
37 continue
38 if not ':' in line:
39 # Invalid file.
40 return None
41 k, v = line.split(':', 1)
42 settings[k.strip()] = v.strip()
43 finally:
44 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000045 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000046 return None
47 return settings.get(key, None)
48
49
50class PatchApplicationFailed(Exception):
51 """Patch failed to be applied."""
52 def __init__(self, filename, status):
53 super(PatchApplicationFailed, self).__init__(filename, status)
54 self.filename = filename
55 self.status = status
56
57
58class CheckoutBase(object):
59 # Set to None to have verbose output.
60 VOID = subprocess2.VOID
61
maruel@chromium.org6ed8b502011-06-12 01:05:35 +000062 def __init__(self, root_dir, project_name, post_processors):
63 """
64 Args:
65 post_processor: list of lambda(checkout, patches) to call on each of the
66 modified files.
67 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000068 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000069 self.root_dir = root_dir
70 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +000071 if self.project_name is None:
72 self.project_path = self.root_dir
73 else:
74 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000075 # Only used for logging purposes.
76 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000077 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000078 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000079 assert self.project_path
80
81 def get_settings(self, key):
82 return get_code_review_setting(self.project_path, key)
83
maruel@chromium.org51919772011-06-12 01:27:42 +000084 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000085 """Checks out a clean copy of the tree and removes any local modification.
86
87 This function shouldn't throw unless the remote repository is inaccessible,
88 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +000089
90 Args:
91 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000092 """
93 raise NotImplementedError()
94
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +000095 def apply_patch(self, patches, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000096 """Applies a patch and returns the list of modified files.
97
98 This function should throw patch.UnsupportedPatchFormat or
99 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000100
101 Args:
102 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000103 """
104 raise NotImplementedError()
105
106 def commit(self, commit_message, user):
107 """Commits the patch upstream, while impersonating 'user'."""
108 raise NotImplementedError()
109
110
111class RawCheckout(CheckoutBase):
112 """Used to apply a patch locally without any intent to commit it.
113
114 To be used by the try server.
115 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000116 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000117 """Stubbed out."""
118 pass
119
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000120 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000121 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000122 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000123 for p in patches:
124 try:
125 stdout = ''
126 filename = os.path.join(self.project_path, p.filename)
127 if p.is_delete:
128 os.remove(filename)
129 else:
130 dirname = os.path.dirname(p.filename)
131 full_dir = os.path.join(self.project_path, dirname)
132 if dirname and not os.path.isdir(full_dir):
133 os.makedirs(full_dir)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000134
135 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000136 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000137 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000138 f.write(p.get())
139 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000140 if p.source_filename:
141 if not p.is_new:
142 raise PatchApplicationFailed(
143 p.filename,
144 'File has a source filename specified but is not new')
145 # Copy the file first.
146 if os.path.isfile(filepath):
147 raise PatchApplicationFailed(
148 p.filename, 'File exist but was about to be overwriten')
149 shutil.copy2(
150 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000151 if p.diff_hunks:
152 stdout = subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000153 ['patch', '-u', '--binary', '-p%s' % p.patchlevel],
154 stdin=p.get(False),
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000155 stderr=subprocess2.STDOUT,
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000156 cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000157 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000158 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000159 open(filepath, 'w').close()
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000160 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000161 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000162 except OSError, e:
163 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
164 except subprocess.CalledProcessError, e:
165 raise PatchApplicationFailed(
166 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
167
168 def commit(self, commit_message, user):
169 """Stubbed out."""
170 raise NotImplementedError('RawCheckout can\'t commit')
171
172
173class SvnConfig(object):
174 """Parses a svn configuration file."""
175 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000176 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000177 self.svn_config_dir = svn_config_dir
178 self.default = not bool(self.svn_config_dir)
179 if not self.svn_config_dir:
180 if sys.platform == 'win32':
181 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
182 else:
183 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
184 svn_config_file = os.path.join(self.svn_config_dir, 'config')
185 parser = ConfigParser.SafeConfigParser()
186 if os.path.isfile(svn_config_file):
187 parser.read(svn_config_file)
188 else:
189 parser.add_section('auto-props')
190 self.auto_props = dict(parser.items('auto-props'))
191
192
193class SvnMixIn(object):
194 """MixIn class to add svn commands common to both svn and git-svn clients."""
195 # These members need to be set by the subclass.
196 commit_user = None
197 commit_pwd = None
198 svn_url = None
199 project_path = None
200 # Override at class level when necessary. If used, --non-interactive is
201 # implied.
202 svn_config = SvnConfig()
203 # Set to True when non-interactivity is necessary but a custom subversion
204 # configuration directory is not necessary.
205 non_interactive = False
206
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000207 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000208 args = ['svn'] + args
209 if not self.svn_config.default:
210 args.extend(['--config-dir', self.svn_config.svn_config_dir])
211 if not self.svn_config.default or self.non_interactive or non_interactive:
212 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000213 if credentials:
214 if self.commit_user:
215 args.extend(['--username', self.commit_user])
216 if self.commit_pwd:
217 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000218 return args
219
220 def _check_call_svn(self, args, **kwargs):
221 """Runs svn and throws an exception if the command failed."""
222 kwargs.setdefault('cwd', self.project_path)
223 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000224 return subprocess2.check_call_out(
225 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000226
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000227 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000228 """Runs svn and throws an exception if the command failed.
229
230 Returns the output.
231 """
232 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000233 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000234 self._add_svn_flags(args, True, credentials),
235 stderr=subprocess2.STDOUT,
236 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000237
238 @staticmethod
239 def _parse_svn_info(output, key):
240 """Returns value for key from svn info output.
241
242 Case insensitive.
243 """
244 values = {}
245 key = key.lower()
246 for line in output.splitlines(False):
247 if not line:
248 continue
249 k, v = line.split(':', 1)
250 k = k.strip().lower()
251 v = v.strip()
252 assert not k in values
253 values[k] = v
254 return values.get(key, None)
255
256
257class SvnCheckout(CheckoutBase, SvnMixIn):
258 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000259 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
260 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000261 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
262 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000263 self.commit_user = commit_user
264 self.commit_pwd = commit_pwd
265 self.svn_url = svn_url
266 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000267
maruel@chromium.org51919772011-06-12 01:27:42 +0000268 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000269 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000270 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000271 if not os.path.isdir(self.project_path):
272 logging.info('Checking out %s in %s' %
273 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000274 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000275
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000276 def apply_patch(self, patches, post_processors=None):
277 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000278 for p in patches:
279 try:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000280 # It is important to use credentials=False otherwise credentials could
281 # leak in the error message. Credentials are not necessary here for the
282 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000283 stdout = ''
284 if p.is_delete:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000285 stdout += self._check_output_svn(
286 ['delete', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000287 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000288 # svn add while creating directories otherwise svn add on the
289 # contained files will silently fail.
290 # First, find the root directory that exists.
291 dirname = os.path.dirname(p.filename)
292 dirs_to_create = []
293 while (dirname and
294 not os.path.isdir(os.path.join(self.project_path, dirname))):
295 dirs_to_create.append(dirname)
296 dirname = os.path.dirname(dirname)
297 for dir_to_create in reversed(dirs_to_create):
298 os.mkdir(os.path.join(self.project_path, dir_to_create))
299 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000300 ['add', dir_to_create, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000301
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000302 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000303 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000304 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000305 f.write(p.get())
306 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000307 if p.source_filename:
308 if not p.is_new:
309 raise PatchApplicationFailed(
310 p.filename,
311 'File has a source filename specified but is not new')
312 # Copy the file first.
313 if os.path.isfile(filepath):
314 raise PatchApplicationFailed(
315 p.filename, 'File exist but was about to be overwriten')
316 shutil.copy2(
317 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000318 if p.diff_hunks:
319 cmd = ['patch', '-p%s' % p.patchlevel, '--forward', '--force']
320 stdout += subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000321 cmd, stdin=p.get(False), cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000322 elif p.is_new and not os.path.exists(filepath):
323 # There is only a header. Just create the file if it doesn't
324 # exist.
325 open(filepath, 'w').close()
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000326 if p.is_new:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000327 stdout += self._check_output_svn(
328 ['add', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000329 for prop in p.svn_properties:
330 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000331 ['propset', prop[0], prop[1], p.filename], credentials=False)
332 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000333 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000334 for value in values.split(';'):
335 if '=' not in value:
336 params = [value, '*']
337 else:
338 params = value.split('=', 1)
339 stdout += self._check_output_svn(
340 ['propset'] + params + [p.filename], credentials=False)
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000341 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000342 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000343 except OSError, e:
344 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
345 except subprocess.CalledProcessError, e:
346 raise PatchApplicationFailed(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000347 p.filename,
348 'While running %s;\n%s%s' % (
349 ' '.join(e.cmd), stdout, getattr(e, 'stdout', '')))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000350
351 def commit(self, commit_message, user):
352 logging.info('Committing patch for %s' % user)
353 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000354 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000355 handle, commit_filename = tempfile.mkstemp(text=True)
356 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000357 # Shouldn't assume default encoding is UTF-8. But really, if you are using
358 # anything else, you are living in another world.
359 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000360 os.close(handle)
361 # When committing, svn won't update the Revision metadata of the checkout,
362 # so if svn commit returns "Committed revision 3.", svn info will still
363 # return "Revision: 2". Since running svn update right after svn commit
364 # creates a race condition with other committers, this code _must_ parse
365 # the output of svn commit and use a regexp to grab the revision number.
366 # Note that "Committed revision N." is localized but subprocess2 forces
367 # LANGUAGE=en.
368 args = ['commit', '--file', commit_filename]
369 # realauthor is parsed by a server-side hook.
370 if user and user != self.commit_user:
371 args.extend(['--with-revprop', 'realauthor=%s' % user])
372 out = self._check_output_svn(args)
373 finally:
374 os.remove(commit_filename)
375 lines = filter(None, out.splitlines())
376 match = re.match(r'^Committed revision (\d+).$', lines[-1])
377 if not match:
378 raise PatchApplicationFailed(
379 None,
380 'Couldn\'t make sense out of svn commit message:\n' + out)
381 return int(match.group(1))
382
maruel@chromium.org51919772011-06-12 01:27:42 +0000383 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000384 """Reverts local modifications or checks out if the directory is not
385 present. Use depot_tools's functionality to do this.
386 """
387 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000388 if revision:
389 flags.extend(['--revision', str(revision)])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000390 if not os.path.isdir(self.project_path):
391 logging.info(
392 'Directory %s is not present, checking it out.' % self.project_path)
393 self._check_call_svn(
394 ['checkout', self.svn_url, self.project_path] + flags, cwd=None)
395 else:
maruel@chromium.orgea15cb72012-05-04 14:16:31 +0000396 scm.SVN.Revert(self.project_path, no_ignore=True)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000397 # Revive files that were deleted in scm.SVN.Revert().
398 self._check_call_svn(['update', '--force'] + flags)
maruel@chromium.org51919772011-06-12 01:27:42 +0000399 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000400
maruel@chromium.org51919772011-06-12 01:27:42 +0000401 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000402 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000403 revision = int(self._parse_svn_info(out, 'revision'))
404 if revision != self._last_seen_revision:
405 logging.info('Updated to revision %d' % revision)
406 self._last_seen_revision = revision
407 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000408
409
410class GitCheckoutBase(CheckoutBase):
411 """Base class for git checkout. Not to be used as-is."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000412 def __init__(self, root_dir, project_name, remote_branch,
413 post_processors=None):
414 super(GitCheckoutBase, self).__init__(
415 root_dir, project_name, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000416 # There is no reason to not hardcode it.
417 self.remote = 'origin'
418 self.remote_branch = remote_branch
419 self.working_branch = 'working_branch'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000420
maruel@chromium.org51919772011-06-12 01:27:42 +0000421 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000422 """Resets the git repository in a clean state.
423
424 Checks it out if not present and deletes the working branch.
425 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000426 assert self.remote_branch
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000427 assert os.path.isdir(self.project_path)
428 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000429 if revision:
430 try:
431 revision = self._check_output_git(['rev-parse', revision])
432 except subprocess.CalledProcessError:
433 self._check_call_git(
434 ['fetch', self.remote, self.remote_branch, '--quiet'])
435 revision = self._check_output_git(['rev-parse', revision])
436 self._check_call_git(['checkout', '--force', '--quiet', revision])
437 else:
438 branches, active = self._branches()
439 if active != 'master':
440 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
441 self._check_call_git(['pull', self.remote, self.remote_branch, '--quiet'])
442 if self.working_branch in branches:
443 self._call_git(['branch', '-D', self.working_branch])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000444
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000445 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000446 """Applies a patch on 'working_branch' and switch to it.
447
448 Also commits the changes on the local branch.
449
450 Ignores svn properties and raise an exception on unexpected ones.
451 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000452 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000453 # It this throws, the checkout is corrupted. Maybe worth deleting it and
454 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000455 if self.remote_branch:
456 self._check_call_git(
457 ['checkout', '-b', self.working_branch,
458 '%s/%s' % (self.remote, self.remote_branch), '--quiet'])
maruel@chromium.org5e975632011-09-29 18:07:06 +0000459 for index, p in enumerate(patches):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000460 try:
461 stdout = ''
462 if p.is_delete:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000463 if (not os.path.exists(p.filename) and
464 any(p1.source_filename == p.filename for p1 in patches[0:index])):
465 # The file could already be deleted if a prior patch with file
466 # rename was already processed. To be sure, look at all the previous
467 # patches to see if they were a file rename.
468 pass
469 else:
470 stdout += self._check_output_git(['rm', p.filename])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000471 else:
472 dirname = os.path.dirname(p.filename)
473 full_dir = os.path.join(self.project_path, dirname)
474 if dirname and not os.path.isdir(full_dir):
475 os.makedirs(full_dir)
476 if p.is_binary:
477 with open(os.path.join(self.project_path, p.filename), 'wb') as f:
478 f.write(p.get())
479 stdout += self._check_output_git(['add', p.filename])
480 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000481 # No need to do anything special with p.is_new or if not
482 # p.diff_hunks. git apply manages all that already.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000483 stdout += self._check_output_git(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000484 ['apply', '--index', '-p%s' % p.patchlevel], stdin=p.get(True))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000485 for prop in p.svn_properties:
486 # Ignore some known auto-props flags through .subversion/config,
487 # bails out on the other ones.
488 # TODO(maruel): Read ~/.subversion/config and detect the rules that
489 # applies here to figure out if the property will be correctly
490 # handled.
maruel@chromium.org9799a072012-01-11 00:26:25 +0000491 if not prop[0] in (
492 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000493 raise patch.UnsupportedPatchFormat(
494 p.filename,
495 'Cannot apply svn property %s to file %s.' % (
496 prop[0], p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000497 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000498 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000499 except OSError, e:
500 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
501 except subprocess.CalledProcessError, e:
502 raise PatchApplicationFailed(
503 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
504 # Once all the patches are processed and added to the index, commit the
505 # index.
506 self._check_call_git(['commit', '-m', 'Committed patch'])
507 # TODO(maruel): Weirdly enough they don't match, need to investigate.
508 #found_files = self._check_output_git(
509 # ['diff', 'master', '--name-only']).splitlines(False)
510 #assert sorted(patches.filenames) == sorted(found_files), (
511 # sorted(out), sorted(found_files))
512
513 def commit(self, commit_message, user):
514 """Updates the commit message.
515
516 Subclass needs to dcommit or push.
517 """
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000518 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000519 self._check_call_git(['commit', '--amend', '-m', commit_message])
520 return self._check_output_git(['rev-parse', 'HEAD']).strip()
521
522 def _check_call_git(self, args, **kwargs):
523 kwargs.setdefault('cwd', self.project_path)
524 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000525 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000526
527 def _call_git(self, args, **kwargs):
528 """Like check_call but doesn't throw on failure."""
529 kwargs.setdefault('cwd', self.project_path)
530 kwargs.setdefault('stdout', self.VOID)
531 return subprocess2.call(['git'] + args, **kwargs)
532
533 def _check_output_git(self, args, **kwargs):
534 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000535 return subprocess2.check_output(
536 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000537
538 def _branches(self):
539 """Returns the list of branches and the active one."""
540 out = self._check_output_git(['branch']).splitlines(False)
541 branches = [l[2:] for l in out]
542 active = None
543 for l in out:
544 if l.startswith('*'):
545 active = l[2:]
546 break
547 return branches, active
548
549
550class GitSvnCheckoutBase(GitCheckoutBase, SvnMixIn):
551 """Base class for git-svn checkout. Not to be used as-is."""
552 def __init__(self,
553 root_dir, project_name, remote_branch,
554 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000555 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000556 """trunk is optional."""
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000557 GitCheckoutBase.__init__(
558 self, root_dir, project_name + '.git', remote_branch, post_processors)
559 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000560 self.commit_user = commit_user
561 self.commit_pwd = commit_pwd
562 # svn_url in this case is the root of the svn repository.
563 self.svn_url = svn_url
564 self.trunk = trunk
565 assert bool(self.commit_user) >= bool(self.commit_pwd)
566 assert self.svn_url
567 assert self.trunk
568 self._cache_svn_auth()
569
maruel@chromium.org51919772011-06-12 01:27:42 +0000570 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000571 """Resets the git repository in a clean state."""
572 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000573 if revision:
574 try:
575 revision = self._check_output_git(
576 ['svn', 'find-rev', 'r%d' % revision])
577 except subprocess.CalledProcessError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000578 self._check_call_git(
maruel@chromium.org51919772011-06-12 01:27:42 +0000579 ['fetch', self.remote, self.remote_branch, '--quiet'])
580 revision = self._check_output_git(
581 ['svn', 'find-rev', 'r%d' % revision])
582 super(GitSvnCheckoutBase, self).prepare(revision)
583 else:
584 branches, active = self._branches()
585 if active != 'master':
586 if not 'master' in branches:
587 self._check_call_git(
588 ['checkout', '--quiet', '-b', 'master',
589 '%s/%s' % (self.remote, self.remote_branch)])
590 else:
591 self._check_call_git(['checkout', 'master', '--force', '--quiet'])
592 # git svn rebase --quiet --quiet doesn't work, use two steps to silence
593 # it.
594 self._check_call_git_svn(['fetch', '--quiet', '--quiet'])
595 self._check_call_git(
596 ['rebase', '--quiet', '--quiet',
597 '%s/%s' % (self.remote, self.remote_branch)])
598 if self.working_branch in branches:
599 self._call_git(['branch', '-D', self.working_branch])
600 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000601
602 def _git_svn_info(self, key):
603 """Calls git svn info. This doesn't support nor need --config-dir."""
604 return self._parse_svn_info(self._check_output_git(['svn', 'info']), key)
605
606 def commit(self, commit_message, user):
607 """Commits a patch."""
608 logging.info('Committing patch for %s' % user)
609 # Fix the commit message and author. It returns the git hash, which we
610 # ignore unless it's None.
611 if not super(GitSvnCheckoutBase, self).commit(commit_message, user):
612 return None
613 # TODO(maruel): git-svn ignores --config-dir as of git-svn version 1.7.4 and
614 # doesn't support --with-revprop.
615 # Either learn perl and upstream or suck it.
616 kwargs = {}
617 if self.commit_pwd:
618 kwargs['stdin'] = self.commit_pwd + '\n'
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000619 kwargs['stderr'] = subprocess2.STDOUT
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000620 self._check_call_git_svn(
621 ['dcommit', '--rmdir', '--find-copies-harder',
622 '--username', self.commit_user],
623 **kwargs)
624 revision = int(self._git_svn_info('revision'))
625 return revision
626
627 def _cache_svn_auth(self):
628 """Caches the svn credentials. It is necessary since git-svn doesn't prompt
629 for it."""
630 if not self.commit_user or not self.commit_pwd:
631 return
632 # Use capture to lower noise in logs.
633 self._check_output_svn(['ls', self.svn_url], cwd=None)
634
635 def _check_call_git_svn(self, args, **kwargs):
636 """Handles svn authentication while calling git svn."""
637 args = ['svn'] + args
638 if not self.svn_config.default:
639 args.extend(['--config-dir', self.svn_config.svn_config_dir])
640 return self._check_call_git(args, **kwargs)
641
642 def _get_revision(self):
643 revision = int(self._git_svn_info('revision'))
644 if revision != self._last_seen_revision:
maruel@chromium.org51919772011-06-12 01:27:42 +0000645 logging.info('Updated to revision %d' % revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000646 self._last_seen_revision = revision
647 return revision
648
649
650class GitSvnPremadeCheckout(GitSvnCheckoutBase):
651 """Manages a git-svn clone made out from an initial git-svn seed.
652
653 This class is very similar to GitSvnCheckout but is faster to bootstrap
654 because it starts right off with an existing git-svn clone.
655 """
656 def __init__(self,
657 root_dir, project_name, remote_branch,
658 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000659 svn_url, trunk, git_url, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000660 super(GitSvnPremadeCheckout, self).__init__(
661 root_dir, project_name, remote_branch,
662 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000663 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000664 self.git_url = git_url
665 assert self.git_url
666
maruel@chromium.org51919772011-06-12 01:27:42 +0000667 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000668 """Creates the initial checkout for the repo."""
669 if not os.path.isdir(self.project_path):
670 logging.info('Checking out %s in %s' %
671 (self.project_name, self.project_path))
672 assert self.remote == 'origin'
673 # self.project_path doesn't exist yet.
674 self._check_call_git(
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000675 ['clone', self.git_url, self.project_name, '--quiet'],
676 cwd=self.root_dir,
677 stderr=subprocess2.STDOUT)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000678 try:
679 configured_svn_url = self._check_output_git(
680 ['config', 'svn-remote.svn.url']).strip()
681 except subprocess.CalledProcessError:
682 configured_svn_url = ''
683
684 if configured_svn_url.strip() != self.svn_url:
685 self._check_call_git_svn(
686 ['init',
687 '--prefix', self.remote + '/',
688 '-T', self.trunk,
689 self.svn_url])
690 self._check_call_git_svn(['fetch'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000691 return super(GitSvnPremadeCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000692
693
694class GitSvnCheckout(GitSvnCheckoutBase):
695 """Manages a git-svn clone.
696
697 Using git-svn hides some of the complexity of using a svn checkout.
698 """
699 def __init__(self,
700 root_dir, project_name,
701 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000702 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000703 super(GitSvnCheckout, self).__init__(
704 root_dir, project_name, 'trunk',
705 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000706 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000707
maruel@chromium.org51919772011-06-12 01:27:42 +0000708 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000709 """Creates the initial checkout for the repo."""
maruel@chromium.org51919772011-06-12 01:27:42 +0000710 assert not revision, 'Implement revision if necessary'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000711 if not os.path.isdir(self.project_path):
712 logging.info('Checking out %s in %s' %
713 (self.project_name, self.project_path))
714 # TODO: Create a shallow clone.
715 # self.project_path doesn't exist yet.
716 self._check_call_git_svn(
717 ['clone',
718 '--prefix', self.remote + '/',
719 '-T', self.trunk,
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000720 self.svn_url, self.project_path,
721 '--quiet'],
722 cwd=self.root_dir,
723 stderr=subprocess2.STDOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000724 return super(GitSvnCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000725
726
727class ReadOnlyCheckout(object):
728 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000729 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000730 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000731 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000732 self.post_processors = (post_processors or []) + (
733 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000734
maruel@chromium.org51919772011-06-12 01:27:42 +0000735 def prepare(self, revision):
736 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000737
738 def get_settings(self, key):
739 return self.checkout.get_settings(key)
740
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000741 def apply_patch(self, patches, post_processors=None):
742 return self.checkout.apply_patch(
743 patches, post_processors or self.post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000744
745 def commit(self, message, user): # pylint: disable=R0201
746 logging.info('Would have committed for %s with message: %s' % (
747 user, message))
748 return 'FAKE'
749
750 @property
751 def project_name(self):
752 return self.checkout.project_name
753
754 @property
755 def project_path(self):
756 return self.checkout.project_path