blob: fa732e8c77fc19d4313e9c713dba3bf55f540cb8 [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
maruel@chromium.org9799a072012-01-11 00:26:25 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000010import ConfigParser
11import fnmatch
12import logging
13import os
14import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000015import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000016import subprocess
17import sys
18import tempfile
19
20import patch
21import scm
22import subprocess2
23
24
25def get_code_review_setting(path, key,
26 codereview_settings_file='codereview.settings'):
27 """Parses codereview.settings and return the value for the key if present.
28
29 Don't cache the values in case the file is changed."""
30 # TODO(maruel): Do not duplicate code.
31 settings = {}
32 try:
33 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
34 try:
35 for line in settings_file.readlines():
36 if not line or line.startswith('#'):
37 continue
38 if not ':' in line:
39 # Invalid file.
40 return None
41 k, v = line.split(':', 1)
42 settings[k.strip()] = v.strip()
43 finally:
44 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000045 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000046 return None
47 return settings.get(key, None)
48
49
50class PatchApplicationFailed(Exception):
51 """Patch failed to be applied."""
52 def __init__(self, filename, status):
53 super(PatchApplicationFailed, self).__init__(filename, status)
54 self.filename = filename
55 self.status = status
56
57
58class CheckoutBase(object):
59 # Set to None to have verbose output.
60 VOID = subprocess2.VOID
61
maruel@chromium.org6ed8b502011-06-12 01:05:35 +000062 def __init__(self, root_dir, project_name, post_processors):
63 """
64 Args:
65 post_processor: list of lambda(checkout, patches) to call on each of the
66 modified files.
67 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000068 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000069 self.root_dir = root_dir
70 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +000071 if self.project_name is None:
72 self.project_path = self.root_dir
73 else:
74 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000075 # Only used for logging purposes.
76 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000077 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000078 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000079 assert self.project_path
80
81 def get_settings(self, key):
82 return get_code_review_setting(self.project_path, key)
83
maruel@chromium.org51919772011-06-12 01:27:42 +000084 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000085 """Checks out a clean copy of the tree and removes any local modification.
86
87 This function shouldn't throw unless the remote repository is inaccessible,
88 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +000089
90 Args:
91 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000092 """
93 raise NotImplementedError()
94
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +000095 def apply_patch(self, patches, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000096 """Applies a patch and returns the list of modified files.
97
98 This function should throw patch.UnsupportedPatchFormat or
99 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000100
101 Args:
102 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000103 """
104 raise NotImplementedError()
105
106 def commit(self, commit_message, user):
107 """Commits the patch upstream, while impersonating 'user'."""
108 raise NotImplementedError()
109
110
111class RawCheckout(CheckoutBase):
112 """Used to apply a patch locally without any intent to commit it.
113
114 To be used by the try server.
115 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000116 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000117 """Stubbed out."""
118 pass
119
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000120 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000121 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000122 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000123 for p in patches:
124 try:
125 stdout = ''
126 filename = os.path.join(self.project_path, p.filename)
127 if p.is_delete:
128 os.remove(filename)
129 else:
130 dirname = os.path.dirname(p.filename)
131 full_dir = os.path.join(self.project_path, dirname)
132 if dirname and not os.path.isdir(full_dir):
133 os.makedirs(full_dir)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000134
135 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000136 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000137 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000138 f.write(p.get())
139 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000140 if p.source_filename:
141 if not p.is_new:
142 raise PatchApplicationFailed(
143 p.filename,
144 'File has a source filename specified but is not new')
145 # Copy the file first.
146 if os.path.isfile(filepath):
147 raise PatchApplicationFailed(
148 p.filename, 'File exist but was about to be overwriten')
149 shutil.copy2(
150 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000151 if p.diff_hunks:
152 stdout = subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000153 ['patch', '-u', '--binary', '-p%s' % p.patchlevel],
154 stdin=p.get(False),
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000155 stderr=subprocess2.STDOUT,
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000156 cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000157 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000158 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000159 open(filepath, 'w').close()
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000160 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000161 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000162 except OSError, e:
163 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
164 except subprocess.CalledProcessError, e:
165 raise PatchApplicationFailed(
166 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
167
168 def commit(self, commit_message, user):
169 """Stubbed out."""
170 raise NotImplementedError('RawCheckout can\'t commit')
171
172
173class SvnConfig(object):
174 """Parses a svn configuration file."""
175 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000176 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000177 self.svn_config_dir = svn_config_dir
178 self.default = not bool(self.svn_config_dir)
179 if not self.svn_config_dir:
180 if sys.platform == 'win32':
181 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
182 else:
183 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
184 svn_config_file = os.path.join(self.svn_config_dir, 'config')
185 parser = ConfigParser.SafeConfigParser()
186 if os.path.isfile(svn_config_file):
187 parser.read(svn_config_file)
188 else:
189 parser.add_section('auto-props')
190 self.auto_props = dict(parser.items('auto-props'))
191
192
193class SvnMixIn(object):
194 """MixIn class to add svn commands common to both svn and git-svn clients."""
195 # These members need to be set by the subclass.
196 commit_user = None
197 commit_pwd = None
198 svn_url = None
199 project_path = None
200 # Override at class level when necessary. If used, --non-interactive is
201 # implied.
202 svn_config = SvnConfig()
203 # Set to True when non-interactivity is necessary but a custom subversion
204 # configuration directory is not necessary.
205 non_interactive = False
206
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000207 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000208 args = ['svn'] + args
209 if not self.svn_config.default:
210 args.extend(['--config-dir', self.svn_config.svn_config_dir])
211 if not self.svn_config.default or self.non_interactive or non_interactive:
212 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000213 if credentials:
214 if self.commit_user:
215 args.extend(['--username', self.commit_user])
216 if self.commit_pwd:
217 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000218 return args
219
220 def _check_call_svn(self, args, **kwargs):
221 """Runs svn and throws an exception if the command failed."""
222 kwargs.setdefault('cwd', self.project_path)
223 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000224 return subprocess2.check_call_out(
225 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000226
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000227 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000228 """Runs svn and throws an exception if the command failed.
229
230 Returns the output.
231 """
232 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000233 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000234 self._add_svn_flags(args, True, credentials),
235 stderr=subprocess2.STDOUT,
236 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000237
238 @staticmethod
239 def _parse_svn_info(output, key):
240 """Returns value for key from svn info output.
241
242 Case insensitive.
243 """
244 values = {}
245 key = key.lower()
246 for line in output.splitlines(False):
247 if not line:
248 continue
249 k, v = line.split(':', 1)
250 k = k.strip().lower()
251 v = v.strip()
252 assert not k in values
253 values[k] = v
254 return values.get(key, None)
255
256
257class SvnCheckout(CheckoutBase, SvnMixIn):
258 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000259 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
260 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000261 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
262 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000263 self.commit_user = commit_user
264 self.commit_pwd = commit_pwd
265 self.svn_url = svn_url
266 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000267
maruel@chromium.org51919772011-06-12 01:27:42 +0000268 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000269 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000270 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000271 if not os.path.isdir(self.project_path):
272 logging.info('Checking out %s in %s' %
273 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000274 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000275
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000276 def apply_patch(self, patches, post_processors=None):
277 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000278 for p in patches:
279 try:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000280 # It is important to use credentials=False otherwise credentials could
281 # leak in the error message. Credentials are not necessary here for the
282 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000283 stdout = ''
284 if p.is_delete:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000285 stdout += self._check_output_svn(
286 ['delete', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000287 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000288 # svn add while creating directories otherwise svn add on the
289 # contained files will silently fail.
290 # First, find the root directory that exists.
291 dirname = os.path.dirname(p.filename)
292 dirs_to_create = []
293 while (dirname and
294 not os.path.isdir(os.path.join(self.project_path, dirname))):
295 dirs_to_create.append(dirname)
296 dirname = os.path.dirname(dirname)
297 for dir_to_create in reversed(dirs_to_create):
298 os.mkdir(os.path.join(self.project_path, dir_to_create))
299 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000300 ['add', dir_to_create, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000301
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000302 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000303 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000304 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000305 f.write(p.get())
306 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000307 if p.source_filename:
308 if not p.is_new:
309 raise PatchApplicationFailed(
310 p.filename,
311 'File has a source filename specified but is not new')
312 # Copy the file first.
313 if os.path.isfile(filepath):
314 raise PatchApplicationFailed(
315 p.filename, 'File exist but was about to be overwriten')
maruel@chromium.org3da83172012-05-07 16:17:20 +0000316 self._check_output_svn(
317 [
318 'copy',
319 os.path.join(self.project_path, p.source_filename),
320 filepath
321 ])
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000322 if p.diff_hunks:
323 cmd = ['patch', '-p%s' % p.patchlevel, '--forward', '--force']
324 stdout += subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000325 cmd, stdin=p.get(False), cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000326 elif p.is_new and not os.path.exists(filepath):
327 # There is only a header. Just create the file if it doesn't
328 # exist.
329 open(filepath, 'w').close()
maruel@chromium.org3da83172012-05-07 16:17:20 +0000330 if p.is_new and not p.source_filename:
331 # Do not run it if p.source_filename is defined, since svn copy was
332 # using above.
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000333 stdout += self._check_output_svn(
334 ['add', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000335 for prop in p.svn_properties:
336 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000337 ['propset', prop[0], prop[1], p.filename], credentials=False)
338 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000339 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000340 for value in values.split(';'):
341 if '=' not in value:
342 params = [value, '*']
343 else:
344 params = value.split('=', 1)
345 stdout += self._check_output_svn(
346 ['propset'] + params + [p.filename], credentials=False)
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000347 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000348 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000349 except OSError, e:
350 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
351 except subprocess.CalledProcessError, e:
352 raise PatchApplicationFailed(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000353 p.filename,
354 'While running %s;\n%s%s' % (
355 ' '.join(e.cmd), stdout, getattr(e, 'stdout', '')))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000356
357 def commit(self, commit_message, user):
358 logging.info('Committing patch for %s' % user)
359 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000360 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000361 handle, commit_filename = tempfile.mkstemp(text=True)
362 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000363 # Shouldn't assume default encoding is UTF-8. But really, if you are using
364 # anything else, you are living in another world.
365 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000366 os.close(handle)
367 # When committing, svn won't update the Revision metadata of the checkout,
368 # so if svn commit returns "Committed revision 3.", svn info will still
369 # return "Revision: 2". Since running svn update right after svn commit
370 # creates a race condition with other committers, this code _must_ parse
371 # the output of svn commit and use a regexp to grab the revision number.
372 # Note that "Committed revision N." is localized but subprocess2 forces
373 # LANGUAGE=en.
374 args = ['commit', '--file', commit_filename]
375 # realauthor is parsed by a server-side hook.
376 if user and user != self.commit_user:
377 args.extend(['--with-revprop', 'realauthor=%s' % user])
378 out = self._check_output_svn(args)
379 finally:
380 os.remove(commit_filename)
381 lines = filter(None, out.splitlines())
382 match = re.match(r'^Committed revision (\d+).$', lines[-1])
383 if not match:
384 raise PatchApplicationFailed(
385 None,
386 'Couldn\'t make sense out of svn commit message:\n' + out)
387 return int(match.group(1))
388
maruel@chromium.org51919772011-06-12 01:27:42 +0000389 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000390 """Reverts local modifications or checks out if the directory is not
391 present. Use depot_tools's functionality to do this.
392 """
393 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000394 if revision:
395 flags.extend(['--revision', str(revision)])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000396 if not os.path.isdir(self.project_path):
397 logging.info(
398 'Directory %s is not present, checking it out.' % self.project_path)
399 self._check_call_svn(
400 ['checkout', self.svn_url, self.project_path] + flags, cwd=None)
401 else:
maruel@chromium.orgea15cb72012-05-04 14:16:31 +0000402 scm.SVN.Revert(self.project_path, no_ignore=True)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000403 # Revive files that were deleted in scm.SVN.Revert().
404 self._check_call_svn(['update', '--force'] + flags)
maruel@chromium.org51919772011-06-12 01:27:42 +0000405 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000406
maruel@chromium.org51919772011-06-12 01:27:42 +0000407 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000408 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000409 revision = int(self._parse_svn_info(out, 'revision'))
410 if revision != self._last_seen_revision:
411 logging.info('Updated to revision %d' % revision)
412 self._last_seen_revision = revision
413 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000414
415
416class GitCheckoutBase(CheckoutBase):
417 """Base class for git checkout. Not to be used as-is."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000418 def __init__(self, root_dir, project_name, remote_branch,
419 post_processors=None):
420 super(GitCheckoutBase, self).__init__(
421 root_dir, project_name, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000422 # There is no reason to not hardcode it.
423 self.remote = 'origin'
424 self.remote_branch = remote_branch
425 self.working_branch = 'working_branch'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000426
maruel@chromium.org51919772011-06-12 01:27:42 +0000427 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000428 """Resets the git repository in a clean state.
429
430 Checks it out if not present and deletes the working branch.
431 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000432 assert self.remote_branch
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000433 assert os.path.isdir(self.project_path)
434 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000435 if revision:
436 try:
437 revision = self._check_output_git(['rev-parse', revision])
438 except subprocess.CalledProcessError:
439 self._check_call_git(
440 ['fetch', self.remote, self.remote_branch, '--quiet'])
441 revision = self._check_output_git(['rev-parse', revision])
442 self._check_call_git(['checkout', '--force', '--quiet', revision])
443 else:
444 branches, active = self._branches()
445 if active != 'master':
446 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
447 self._check_call_git(['pull', self.remote, self.remote_branch, '--quiet'])
448 if self.working_branch in branches:
449 self._call_git(['branch', '-D', self.working_branch])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000450
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000451 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000452 """Applies a patch on 'working_branch' and switch to it.
453
454 Also commits the changes on the local branch.
455
456 Ignores svn properties and raise an exception on unexpected ones.
457 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000458 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000459 # It this throws, the checkout is corrupted. Maybe worth deleting it and
460 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000461 if self.remote_branch:
462 self._check_call_git(
463 ['checkout', '-b', self.working_branch,
464 '%s/%s' % (self.remote, self.remote_branch), '--quiet'])
maruel@chromium.org5e975632011-09-29 18:07:06 +0000465 for index, p in enumerate(patches):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000466 try:
467 stdout = ''
468 if p.is_delete:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000469 if (not os.path.exists(p.filename) and
470 any(p1.source_filename == p.filename for p1 in patches[0:index])):
471 # The file could already be deleted if a prior patch with file
472 # rename was already processed. To be sure, look at all the previous
473 # patches to see if they were a file rename.
474 pass
475 else:
476 stdout += self._check_output_git(['rm', p.filename])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000477 else:
478 dirname = os.path.dirname(p.filename)
479 full_dir = os.path.join(self.project_path, dirname)
480 if dirname and not os.path.isdir(full_dir):
481 os.makedirs(full_dir)
482 if p.is_binary:
483 with open(os.path.join(self.project_path, p.filename), 'wb') as f:
484 f.write(p.get())
485 stdout += self._check_output_git(['add', p.filename])
486 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000487 # No need to do anything special with p.is_new or if not
488 # p.diff_hunks. git apply manages all that already.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000489 stdout += self._check_output_git(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000490 ['apply', '--index', '-p%s' % p.patchlevel], stdin=p.get(True))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000491 for prop in p.svn_properties:
492 # Ignore some known auto-props flags through .subversion/config,
493 # bails out on the other ones.
494 # TODO(maruel): Read ~/.subversion/config and detect the rules that
495 # applies here to figure out if the property will be correctly
496 # handled.
maruel@chromium.org9799a072012-01-11 00:26:25 +0000497 if not prop[0] in (
498 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000499 raise patch.UnsupportedPatchFormat(
500 p.filename,
501 'Cannot apply svn property %s to file %s.' % (
502 prop[0], p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000503 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000504 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000505 except OSError, e:
506 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
507 except subprocess.CalledProcessError, e:
508 raise PatchApplicationFailed(
509 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
510 # Once all the patches are processed and added to the index, commit the
511 # index.
512 self._check_call_git(['commit', '-m', 'Committed patch'])
513 # TODO(maruel): Weirdly enough they don't match, need to investigate.
514 #found_files = self._check_output_git(
515 # ['diff', 'master', '--name-only']).splitlines(False)
516 #assert sorted(patches.filenames) == sorted(found_files), (
517 # sorted(out), sorted(found_files))
518
519 def commit(self, commit_message, user):
520 """Updates the commit message.
521
522 Subclass needs to dcommit or push.
523 """
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000524 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000525 self._check_call_git(['commit', '--amend', '-m', commit_message])
526 return self._check_output_git(['rev-parse', 'HEAD']).strip()
527
528 def _check_call_git(self, args, **kwargs):
529 kwargs.setdefault('cwd', self.project_path)
530 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000531 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000532
533 def _call_git(self, args, **kwargs):
534 """Like check_call but doesn't throw on failure."""
535 kwargs.setdefault('cwd', self.project_path)
536 kwargs.setdefault('stdout', self.VOID)
537 return subprocess2.call(['git'] + args, **kwargs)
538
539 def _check_output_git(self, args, **kwargs):
540 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000541 return subprocess2.check_output(
542 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000543
544 def _branches(self):
545 """Returns the list of branches and the active one."""
546 out = self._check_output_git(['branch']).splitlines(False)
547 branches = [l[2:] for l in out]
548 active = None
549 for l in out:
550 if l.startswith('*'):
551 active = l[2:]
552 break
553 return branches, active
554
555
556class GitSvnCheckoutBase(GitCheckoutBase, SvnMixIn):
557 """Base class for git-svn checkout. Not to be used as-is."""
558 def __init__(self,
559 root_dir, project_name, remote_branch,
560 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000561 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000562 """trunk is optional."""
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000563 GitCheckoutBase.__init__(
564 self, root_dir, project_name + '.git', remote_branch, post_processors)
565 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000566 self.commit_user = commit_user
567 self.commit_pwd = commit_pwd
568 # svn_url in this case is the root of the svn repository.
569 self.svn_url = svn_url
570 self.trunk = trunk
571 assert bool(self.commit_user) >= bool(self.commit_pwd)
572 assert self.svn_url
573 assert self.trunk
574 self._cache_svn_auth()
575
maruel@chromium.org51919772011-06-12 01:27:42 +0000576 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000577 """Resets the git repository in a clean state."""
578 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000579 if revision:
580 try:
581 revision = self._check_output_git(
582 ['svn', 'find-rev', 'r%d' % revision])
583 except subprocess.CalledProcessError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000584 self._check_call_git(
maruel@chromium.org51919772011-06-12 01:27:42 +0000585 ['fetch', self.remote, self.remote_branch, '--quiet'])
586 revision = self._check_output_git(
587 ['svn', 'find-rev', 'r%d' % revision])
588 super(GitSvnCheckoutBase, self).prepare(revision)
589 else:
590 branches, active = self._branches()
591 if active != 'master':
592 if not 'master' in branches:
593 self._check_call_git(
594 ['checkout', '--quiet', '-b', 'master',
595 '%s/%s' % (self.remote, self.remote_branch)])
596 else:
597 self._check_call_git(['checkout', 'master', '--force', '--quiet'])
598 # git svn rebase --quiet --quiet doesn't work, use two steps to silence
599 # it.
600 self._check_call_git_svn(['fetch', '--quiet', '--quiet'])
601 self._check_call_git(
602 ['rebase', '--quiet', '--quiet',
603 '%s/%s' % (self.remote, self.remote_branch)])
604 if self.working_branch in branches:
605 self._call_git(['branch', '-D', self.working_branch])
606 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000607
608 def _git_svn_info(self, key):
609 """Calls git svn info. This doesn't support nor need --config-dir."""
610 return self._parse_svn_info(self._check_output_git(['svn', 'info']), key)
611
612 def commit(self, commit_message, user):
613 """Commits a patch."""
614 logging.info('Committing patch for %s' % user)
615 # Fix the commit message and author. It returns the git hash, which we
616 # ignore unless it's None.
617 if not super(GitSvnCheckoutBase, self).commit(commit_message, user):
618 return None
619 # TODO(maruel): git-svn ignores --config-dir as of git-svn version 1.7.4 and
620 # doesn't support --with-revprop.
621 # Either learn perl and upstream or suck it.
622 kwargs = {}
623 if self.commit_pwd:
624 kwargs['stdin'] = self.commit_pwd + '\n'
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000625 kwargs['stderr'] = subprocess2.STDOUT
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000626 self._check_call_git_svn(
627 ['dcommit', '--rmdir', '--find-copies-harder',
628 '--username', self.commit_user],
629 **kwargs)
630 revision = int(self._git_svn_info('revision'))
631 return revision
632
633 def _cache_svn_auth(self):
634 """Caches the svn credentials. It is necessary since git-svn doesn't prompt
635 for it."""
636 if not self.commit_user or not self.commit_pwd:
637 return
638 # Use capture to lower noise in logs.
639 self._check_output_svn(['ls', self.svn_url], cwd=None)
640
641 def _check_call_git_svn(self, args, **kwargs):
642 """Handles svn authentication while calling git svn."""
643 args = ['svn'] + args
644 if not self.svn_config.default:
645 args.extend(['--config-dir', self.svn_config.svn_config_dir])
646 return self._check_call_git(args, **kwargs)
647
648 def _get_revision(self):
649 revision = int(self._git_svn_info('revision'))
650 if revision != self._last_seen_revision:
maruel@chromium.org51919772011-06-12 01:27:42 +0000651 logging.info('Updated to revision %d' % revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000652 self._last_seen_revision = revision
653 return revision
654
655
656class GitSvnPremadeCheckout(GitSvnCheckoutBase):
657 """Manages a git-svn clone made out from an initial git-svn seed.
658
659 This class is very similar to GitSvnCheckout but is faster to bootstrap
660 because it starts right off with an existing git-svn clone.
661 """
662 def __init__(self,
663 root_dir, project_name, remote_branch,
664 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000665 svn_url, trunk, git_url, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000666 super(GitSvnPremadeCheckout, self).__init__(
667 root_dir, project_name, remote_branch,
668 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000669 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000670 self.git_url = git_url
671 assert self.git_url
672
maruel@chromium.org51919772011-06-12 01:27:42 +0000673 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000674 """Creates the initial checkout for the repo."""
675 if not os.path.isdir(self.project_path):
676 logging.info('Checking out %s in %s' %
677 (self.project_name, self.project_path))
678 assert self.remote == 'origin'
679 # self.project_path doesn't exist yet.
680 self._check_call_git(
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000681 ['clone', self.git_url, self.project_name, '--quiet'],
682 cwd=self.root_dir,
683 stderr=subprocess2.STDOUT)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000684 try:
685 configured_svn_url = self._check_output_git(
686 ['config', 'svn-remote.svn.url']).strip()
687 except subprocess.CalledProcessError:
688 configured_svn_url = ''
689
690 if configured_svn_url.strip() != self.svn_url:
691 self._check_call_git_svn(
692 ['init',
693 '--prefix', self.remote + '/',
694 '-T', self.trunk,
695 self.svn_url])
696 self._check_call_git_svn(['fetch'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000697 return super(GitSvnPremadeCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000698
699
700class GitSvnCheckout(GitSvnCheckoutBase):
701 """Manages a git-svn clone.
702
703 Using git-svn hides some of the complexity of using a svn checkout.
704 """
705 def __init__(self,
706 root_dir, project_name,
707 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000708 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000709 super(GitSvnCheckout, self).__init__(
710 root_dir, project_name, 'trunk',
711 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000712 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000713
maruel@chromium.org51919772011-06-12 01:27:42 +0000714 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000715 """Creates the initial checkout for the repo."""
maruel@chromium.org51919772011-06-12 01:27:42 +0000716 assert not revision, 'Implement revision if necessary'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000717 if not os.path.isdir(self.project_path):
718 logging.info('Checking out %s in %s' %
719 (self.project_name, self.project_path))
720 # TODO: Create a shallow clone.
721 # self.project_path doesn't exist yet.
722 self._check_call_git_svn(
723 ['clone',
724 '--prefix', self.remote + '/',
725 '-T', self.trunk,
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000726 self.svn_url, self.project_path,
727 '--quiet'],
728 cwd=self.root_dir,
729 stderr=subprocess2.STDOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000730 return super(GitSvnCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000731
732
733class ReadOnlyCheckout(object):
734 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000735 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000736 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000737 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000738 self.post_processors = (post_processors or []) + (
739 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000740
maruel@chromium.org51919772011-06-12 01:27:42 +0000741 def prepare(self, revision):
742 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000743
744 def get_settings(self, key):
745 return self.checkout.get_settings(key)
746
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000747 def apply_patch(self, patches, post_processors=None):
748 return self.checkout.apply_patch(
749 patches, post_processors or self.post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000750
751 def commit(self, message, user): # pylint: disable=R0201
752 logging.info('Would have committed for %s with message: %s' % (
753 user, message))
754 return 'FAKE'
755
756 @property
757 def project_name(self):
758 return self.checkout.project_name
759
760 @property
761 def project_path(self):
762 return self.checkout.project_path