blob: 8a3f7125d4ab754f7712c722b062bb43239e0e5c [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
2# Copyright (c) 2011 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
10from __future__ import with_statement
11import ConfigParser
12import fnmatch
13import logging
14import os
15import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000016import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000017import subprocess
18import sys
19import tempfile
20
21import patch
22import scm
23import subprocess2
24
25
26def get_code_review_setting(path, key,
27 codereview_settings_file='codereview.settings'):
28 """Parses codereview.settings and return the value for the key if present.
29
30 Don't cache the values in case the file is changed."""
31 # TODO(maruel): Do not duplicate code.
32 settings = {}
33 try:
34 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
35 try:
36 for line in settings_file.readlines():
37 if not line or line.startswith('#'):
38 continue
39 if not ':' in line:
40 # Invalid file.
41 return None
42 k, v = line.split(':', 1)
43 settings[k.strip()] = v.strip()
44 finally:
45 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000046 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000047 return None
48 return settings.get(key, None)
49
50
51class PatchApplicationFailed(Exception):
52 """Patch failed to be applied."""
53 def __init__(self, filename, status):
54 super(PatchApplicationFailed, self).__init__(filename, status)
55 self.filename = filename
56 self.status = status
57
58
59class CheckoutBase(object):
60 # Set to None to have verbose output.
61 VOID = subprocess2.VOID
62
maruel@chromium.org6ed8b502011-06-12 01:05:35 +000063 def __init__(self, root_dir, project_name, post_processors):
64 """
65 Args:
66 post_processor: list of lambda(checkout, patches) to call on each of the
67 modified files.
68 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000069 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000070 self.root_dir = root_dir
71 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +000072 if self.project_name is None:
73 self.project_path = self.root_dir
74 else:
75 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000076 # Only used for logging purposes.
77 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +000078 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000079 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000080 assert self.project_path
81
82 def get_settings(self, key):
83 return get_code_review_setting(self.project_path, key)
84
maruel@chromium.org51919772011-06-12 01:27:42 +000085 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000086 """Checks out a clean copy of the tree and removes any local modification.
87
88 This function shouldn't throw unless the remote repository is inaccessible,
89 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +000090
91 Args:
92 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000093 """
94 raise NotImplementedError()
95
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +000096 def apply_patch(self, patches, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000097 """Applies a patch and returns the list of modified files.
98
99 This function should throw patch.UnsupportedPatchFormat or
100 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000101
102 Args:
103 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000104 """
105 raise NotImplementedError()
106
107 def commit(self, commit_message, user):
108 """Commits the patch upstream, while impersonating 'user'."""
109 raise NotImplementedError()
110
111
112class RawCheckout(CheckoutBase):
113 """Used to apply a patch locally without any intent to commit it.
114
115 To be used by the try server.
116 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000117 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000118 """Stubbed out."""
119 pass
120
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000121 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000122 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000123 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000124 for p in patches:
125 try:
126 stdout = ''
127 filename = os.path.join(self.project_path, p.filename)
128 if p.is_delete:
129 os.remove(filename)
130 else:
131 dirname = os.path.dirname(p.filename)
132 full_dir = os.path.join(self.project_path, dirname)
133 if dirname and not os.path.isdir(full_dir):
134 os.makedirs(full_dir)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000135
136 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000137 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000138 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000139 f.write(p.get())
140 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000141 if p.source_filename:
142 if not p.is_new:
143 raise PatchApplicationFailed(
144 p.filename,
145 'File has a source filename specified but is not new')
146 # Copy the file first.
147 if os.path.isfile(filepath):
148 raise PatchApplicationFailed(
149 p.filename, 'File exist but was about to be overwriten')
150 shutil.copy2(
151 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000152 if p.diff_hunks:
153 stdout = subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000154 ['patch', '-u', '--binary', '-p%s' % p.patchlevel],
155 stdin=p.get(False),
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000156 stderr=subprocess2.STDOUT,
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000157 cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000158 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000159 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000160 open(filepath, 'w').close()
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000161 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000162 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000163 except OSError, e:
164 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
165 except subprocess.CalledProcessError, e:
166 raise PatchApplicationFailed(
167 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
168
169 def commit(self, commit_message, user):
170 """Stubbed out."""
171 raise NotImplementedError('RawCheckout can\'t commit')
172
173
174class SvnConfig(object):
175 """Parses a svn configuration file."""
176 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000177 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000178 self.svn_config_dir = svn_config_dir
179 self.default = not bool(self.svn_config_dir)
180 if not self.svn_config_dir:
181 if sys.platform == 'win32':
182 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
183 else:
184 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
185 svn_config_file = os.path.join(self.svn_config_dir, 'config')
186 parser = ConfigParser.SafeConfigParser()
187 if os.path.isfile(svn_config_file):
188 parser.read(svn_config_file)
189 else:
190 parser.add_section('auto-props')
191 self.auto_props = dict(parser.items('auto-props'))
192
193
194class SvnMixIn(object):
195 """MixIn class to add svn commands common to both svn and git-svn clients."""
196 # These members need to be set by the subclass.
197 commit_user = None
198 commit_pwd = None
199 svn_url = None
200 project_path = None
201 # Override at class level when necessary. If used, --non-interactive is
202 # implied.
203 svn_config = SvnConfig()
204 # Set to True when non-interactivity is necessary but a custom subversion
205 # configuration directory is not necessary.
206 non_interactive = False
207
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000208 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000209 args = ['svn'] + args
210 if not self.svn_config.default:
211 args.extend(['--config-dir', self.svn_config.svn_config_dir])
212 if not self.svn_config.default or self.non_interactive or non_interactive:
213 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000214 if credentials:
215 if self.commit_user:
216 args.extend(['--username', self.commit_user])
217 if self.commit_pwd:
218 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000219 return args
220
221 def _check_call_svn(self, args, **kwargs):
222 """Runs svn and throws an exception if the command failed."""
223 kwargs.setdefault('cwd', self.project_path)
224 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000225 return subprocess2.check_call_out(
226 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000227
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000228 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000229 """Runs svn and throws an exception if the command failed.
230
231 Returns the output.
232 """
233 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000234 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000235 self._add_svn_flags(args, True, credentials),
236 stderr=subprocess2.STDOUT,
237 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000238
239 @staticmethod
240 def _parse_svn_info(output, key):
241 """Returns value for key from svn info output.
242
243 Case insensitive.
244 """
245 values = {}
246 key = key.lower()
247 for line in output.splitlines(False):
248 if not line:
249 continue
250 k, v = line.split(':', 1)
251 k = k.strip().lower()
252 v = v.strip()
253 assert not k in values
254 values[k] = v
255 return values.get(key, None)
256
257
258class SvnCheckout(CheckoutBase, SvnMixIn):
259 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000260 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
261 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000262 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
263 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000264 self.commit_user = commit_user
265 self.commit_pwd = commit_pwd
266 self.svn_url = svn_url
267 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000268
maruel@chromium.org51919772011-06-12 01:27:42 +0000269 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000270 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000271 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000272 if not os.path.isdir(self.project_path):
273 logging.info('Checking out %s in %s' %
274 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000275 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000276
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000277 def apply_patch(self, patches, post_processors=None):
278 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000279 for p in patches:
280 try:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000281 # It is important to use credentials=False otherwise credentials could
282 # leak in the error message. Credentials are not necessary here for the
283 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000284 stdout = ''
285 if p.is_delete:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000286 stdout += self._check_output_svn(
287 ['delete', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000288 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000289 # svn add while creating directories otherwise svn add on the
290 # contained files will silently fail.
291 # First, find the root directory that exists.
292 dirname = os.path.dirname(p.filename)
293 dirs_to_create = []
294 while (dirname and
295 not os.path.isdir(os.path.join(self.project_path, dirname))):
296 dirs_to_create.append(dirname)
297 dirname = os.path.dirname(dirname)
298 for dir_to_create in reversed(dirs_to_create):
299 os.mkdir(os.path.join(self.project_path, dir_to_create))
300 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000301 ['add', dir_to_create, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000302
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000303 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000304 if p.is_binary:
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000305 with open(filepath, 'wb') as f:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000306 f.write(p.get())
307 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000308 if p.source_filename:
309 if not p.is_new:
310 raise PatchApplicationFailed(
311 p.filename,
312 'File has a source filename specified but is not new')
313 # Copy the file first.
314 if os.path.isfile(filepath):
315 raise PatchApplicationFailed(
316 p.filename, 'File exist but was about to be overwriten')
317 shutil.copy2(
318 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000319 if p.diff_hunks:
320 cmd = ['patch', '-p%s' % p.patchlevel, '--forward', '--force']
321 stdout += subprocess2.check_output(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000322 cmd, stdin=p.get(False), cwd=self.project_path)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000323 elif p.is_new and not os.path.exists(filepath):
324 # There is only a header. Just create the file if it doesn't
325 # exist.
326 open(filepath, 'w').close()
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000327 if p.is_new:
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000328 stdout += self._check_output_svn(
329 ['add', p.filename, '--force'], credentials=False)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000330 for prop in p.svn_properties:
331 stdout += self._check_output_svn(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000332 ['propset', prop[0], prop[1], p.filename], credentials=False)
333 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000334 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000335 for value in values.split(';'):
336 if '=' not in value:
337 params = [value, '*']
338 else:
339 params = value.split('=', 1)
340 stdout += self._check_output_svn(
341 ['propset'] + params + [p.filename], credentials=False)
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000342 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000343 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000344 except OSError, e:
345 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
346 except subprocess.CalledProcessError, e:
347 raise PatchApplicationFailed(
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000348 p.filename,
349 'While running %s;\n%s%s' % (
350 ' '.join(e.cmd), stdout, getattr(e, 'stdout', '')))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000351
352 def commit(self, commit_message, user):
353 logging.info('Committing patch for %s' % user)
354 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000355 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000356 handle, commit_filename = tempfile.mkstemp(text=True)
357 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000358 # Shouldn't assume default encoding is UTF-8. But really, if you are using
359 # anything else, you are living in another world.
360 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000361 os.close(handle)
362 # When committing, svn won't update the Revision metadata of the checkout,
363 # so if svn commit returns "Committed revision 3.", svn info will still
364 # return "Revision: 2". Since running svn update right after svn commit
365 # creates a race condition with other committers, this code _must_ parse
366 # the output of svn commit and use a regexp to grab the revision number.
367 # Note that "Committed revision N." is localized but subprocess2 forces
368 # LANGUAGE=en.
369 args = ['commit', '--file', commit_filename]
370 # realauthor is parsed by a server-side hook.
371 if user and user != self.commit_user:
372 args.extend(['--with-revprop', 'realauthor=%s' % user])
373 out = self._check_output_svn(args)
374 finally:
375 os.remove(commit_filename)
376 lines = filter(None, out.splitlines())
377 match = re.match(r'^Committed revision (\d+).$', lines[-1])
378 if not match:
379 raise PatchApplicationFailed(
380 None,
381 'Couldn\'t make sense out of svn commit message:\n' + out)
382 return int(match.group(1))
383
maruel@chromium.org51919772011-06-12 01:27:42 +0000384 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000385 """Reverts local modifications or checks out if the directory is not
386 present. Use depot_tools's functionality to do this.
387 """
388 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000389 if revision:
390 flags.extend(['--revision', str(revision)])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000391 if not os.path.isdir(self.project_path):
392 logging.info(
393 'Directory %s is not present, checking it out.' % self.project_path)
394 self._check_call_svn(
395 ['checkout', self.svn_url, self.project_path] + flags, cwd=None)
396 else:
397 scm.SVN.Revert(self.project_path)
398 # Revive files that were deleted in scm.SVN.Revert().
399 self._check_call_svn(['update', '--force'] + flags)
maruel@chromium.org51919772011-06-12 01:27:42 +0000400 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000401
maruel@chromium.org51919772011-06-12 01:27:42 +0000402 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000403 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000404 revision = int(self._parse_svn_info(out, 'revision'))
405 if revision != self._last_seen_revision:
406 logging.info('Updated to revision %d' % revision)
407 self._last_seen_revision = revision
408 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000409
410
411class GitCheckoutBase(CheckoutBase):
412 """Base class for git checkout. Not to be used as-is."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000413 def __init__(self, root_dir, project_name, remote_branch,
414 post_processors=None):
415 super(GitCheckoutBase, self).__init__(
416 root_dir, project_name, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000417 # There is no reason to not hardcode it.
418 self.remote = 'origin'
419 self.remote_branch = remote_branch
420 self.working_branch = 'working_branch'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000421
maruel@chromium.org51919772011-06-12 01:27:42 +0000422 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000423 """Resets the git repository in a clean state.
424
425 Checks it out if not present and deletes the working branch.
426 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000427 assert self.remote_branch
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000428 assert os.path.isdir(self.project_path)
429 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000430 if revision:
431 try:
432 revision = self._check_output_git(['rev-parse', revision])
433 except subprocess.CalledProcessError:
434 self._check_call_git(
435 ['fetch', self.remote, self.remote_branch, '--quiet'])
436 revision = self._check_output_git(['rev-parse', revision])
437 self._check_call_git(['checkout', '--force', '--quiet', revision])
438 else:
439 branches, active = self._branches()
440 if active != 'master':
441 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
442 self._check_call_git(['pull', self.remote, self.remote_branch, '--quiet'])
443 if self.working_branch in branches:
444 self._call_git(['branch', '-D', self.working_branch])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000445
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000446 def apply_patch(self, patches, post_processors=None):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000447 """Applies a patch on 'working_branch' and switch to it.
448
449 Also commits the changes on the local branch.
450
451 Ignores svn properties and raise an exception on unexpected ones.
452 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000453 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000454 # It this throws, the checkout is corrupted. Maybe worth deleting it and
455 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000456 if self.remote_branch:
457 self._check_call_git(
458 ['checkout', '-b', self.working_branch,
459 '%s/%s' % (self.remote, self.remote_branch), '--quiet'])
maruel@chromium.org5e975632011-09-29 18:07:06 +0000460 for index, p in enumerate(patches):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000461 try:
462 stdout = ''
463 if p.is_delete:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000464 if (not os.path.exists(p.filename) and
465 any(p1.source_filename == p.filename for p1 in patches[0:index])):
466 # The file could already be deleted if a prior patch with file
467 # rename was already processed. To be sure, look at all the previous
468 # patches to see if they were a file rename.
469 pass
470 else:
471 stdout += self._check_output_git(['rm', p.filename])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000472 else:
473 dirname = os.path.dirname(p.filename)
474 full_dir = os.path.join(self.project_path, dirname)
475 if dirname and not os.path.isdir(full_dir):
476 os.makedirs(full_dir)
477 if p.is_binary:
478 with open(os.path.join(self.project_path, p.filename), 'wb') as f:
479 f.write(p.get())
480 stdout += self._check_output_git(['add', p.filename])
481 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000482 # No need to do anything special with p.is_new or if not
483 # p.diff_hunks. git apply manages all that already.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000484 stdout += self._check_output_git(
maruel@chromium.org5e975632011-09-29 18:07:06 +0000485 ['apply', '--index', '-p%s' % p.patchlevel], stdin=p.get(True))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000486 for prop in p.svn_properties:
487 # Ignore some known auto-props flags through .subversion/config,
488 # bails out on the other ones.
489 # TODO(maruel): Read ~/.subversion/config and detect the rules that
490 # applies here to figure out if the property will be correctly
491 # handled.
492 if not prop[0] in ('svn:eol-style', 'svn:executable'):
493 raise patch.UnsupportedPatchFormat(
494 p.filename,
495 'Cannot apply svn property %s to file %s.' % (
496 prop[0], p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000497 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000498 post(self, p)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000499 except OSError, e:
500 raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
501 except subprocess.CalledProcessError, e:
502 raise PatchApplicationFailed(
503 p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
504 # Once all the patches are processed and added to the index, commit the
505 # index.
506 self._check_call_git(['commit', '-m', 'Committed patch'])
507 # TODO(maruel): Weirdly enough they don't match, need to investigate.
508 #found_files = self._check_output_git(
509 # ['diff', 'master', '--name-only']).splitlines(False)
510 #assert sorted(patches.filenames) == sorted(found_files), (
511 # sorted(out), sorted(found_files))
512
513 def commit(self, commit_message, user):
514 """Updates the commit message.
515
516 Subclass needs to dcommit or push.
517 """
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000518 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000519 self._check_call_git(['commit', '--amend', '-m', commit_message])
520 return self._check_output_git(['rev-parse', 'HEAD']).strip()
521
522 def _check_call_git(self, args, **kwargs):
523 kwargs.setdefault('cwd', self.project_path)
524 kwargs.setdefault('stdout', self.VOID)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000525 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000526
527 def _call_git(self, args, **kwargs):
528 """Like check_call but doesn't throw on failure."""
529 kwargs.setdefault('cwd', self.project_path)
530 kwargs.setdefault('stdout', self.VOID)
531 return subprocess2.call(['git'] + args, **kwargs)
532
533 def _check_output_git(self, args, **kwargs):
534 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000535 return subprocess2.check_output(
536 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000537
538 def _branches(self):
539 """Returns the list of branches and the active one."""
540 out = self._check_output_git(['branch']).splitlines(False)
541 branches = [l[2:] for l in out]
542 active = None
543 for l in out:
544 if l.startswith('*'):
545 active = l[2:]
546 break
547 return branches, active
548
549
550class GitSvnCheckoutBase(GitCheckoutBase, SvnMixIn):
551 """Base class for git-svn checkout. Not to be used as-is."""
552 def __init__(self,
553 root_dir, project_name, remote_branch,
554 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000555 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000556 """trunk is optional."""
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000557 GitCheckoutBase.__init__(
558 self, root_dir, project_name + '.git', remote_branch, post_processors)
559 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000560 self.commit_user = commit_user
561 self.commit_pwd = commit_pwd
562 # svn_url in this case is the root of the svn repository.
563 self.svn_url = svn_url
564 self.trunk = trunk
565 assert bool(self.commit_user) >= bool(self.commit_pwd)
566 assert self.svn_url
567 assert self.trunk
568 self._cache_svn_auth()
569
maruel@chromium.org51919772011-06-12 01:27:42 +0000570 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000571 """Resets the git repository in a clean state."""
572 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000573 if revision:
574 try:
575 revision = self._check_output_git(
576 ['svn', 'find-rev', 'r%d' % revision])
577 except subprocess.CalledProcessError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000578 self._check_call_git(
maruel@chromium.org51919772011-06-12 01:27:42 +0000579 ['fetch', self.remote, self.remote_branch, '--quiet'])
580 revision = self._check_output_git(
581 ['svn', 'find-rev', 'r%d' % revision])
582 super(GitSvnCheckoutBase, self).prepare(revision)
583 else:
584 branches, active = self._branches()
585 if active != 'master':
586 if not 'master' in branches:
587 self._check_call_git(
588 ['checkout', '--quiet', '-b', 'master',
589 '%s/%s' % (self.remote, self.remote_branch)])
590 else:
591 self._check_call_git(['checkout', 'master', '--force', '--quiet'])
592 # git svn rebase --quiet --quiet doesn't work, use two steps to silence
593 # it.
594 self._check_call_git_svn(['fetch', '--quiet', '--quiet'])
595 self._check_call_git(
596 ['rebase', '--quiet', '--quiet',
597 '%s/%s' % (self.remote, self.remote_branch)])
598 if self.working_branch in branches:
599 self._call_git(['branch', '-D', self.working_branch])
600 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000601
602 def _git_svn_info(self, key):
603 """Calls git svn info. This doesn't support nor need --config-dir."""
604 return self._parse_svn_info(self._check_output_git(['svn', 'info']), key)
605
606 def commit(self, commit_message, user):
607 """Commits a patch."""
608 logging.info('Committing patch for %s' % user)
609 # Fix the commit message and author. It returns the git hash, which we
610 # ignore unless it's None.
611 if not super(GitSvnCheckoutBase, self).commit(commit_message, user):
612 return None
613 # TODO(maruel): git-svn ignores --config-dir as of git-svn version 1.7.4 and
614 # doesn't support --with-revprop.
615 # Either learn perl and upstream or suck it.
616 kwargs = {}
617 if self.commit_pwd:
618 kwargs['stdin'] = self.commit_pwd + '\n'
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000619 kwargs['stderr'] = subprocess2.STDOUT
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000620 self._check_call_git_svn(
621 ['dcommit', '--rmdir', '--find-copies-harder',
622 '--username', self.commit_user],
623 **kwargs)
624 revision = int(self._git_svn_info('revision'))
625 return revision
626
627 def _cache_svn_auth(self):
628 """Caches the svn credentials. It is necessary since git-svn doesn't prompt
629 for it."""
630 if not self.commit_user or not self.commit_pwd:
631 return
632 # Use capture to lower noise in logs.
633 self._check_output_svn(['ls', self.svn_url], cwd=None)
634
635 def _check_call_git_svn(self, args, **kwargs):
636 """Handles svn authentication while calling git svn."""
637 args = ['svn'] + args
638 if not self.svn_config.default:
639 args.extend(['--config-dir', self.svn_config.svn_config_dir])
640 return self._check_call_git(args, **kwargs)
641
642 def _get_revision(self):
643 revision = int(self._git_svn_info('revision'))
644 if revision != self._last_seen_revision:
maruel@chromium.org51919772011-06-12 01:27:42 +0000645 logging.info('Updated to revision %d' % revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000646 self._last_seen_revision = revision
647 return revision
648
649
650class GitSvnPremadeCheckout(GitSvnCheckoutBase):
651 """Manages a git-svn clone made out from an initial git-svn seed.
652
653 This class is very similar to GitSvnCheckout but is faster to bootstrap
654 because it starts right off with an existing git-svn clone.
655 """
656 def __init__(self,
657 root_dir, project_name, remote_branch,
658 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000659 svn_url, trunk, git_url, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000660 super(GitSvnPremadeCheckout, self).__init__(
661 root_dir, project_name, remote_branch,
662 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000663 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000664 self.git_url = git_url
665 assert self.git_url
666
maruel@chromium.org51919772011-06-12 01:27:42 +0000667 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000668 """Creates the initial checkout for the repo."""
669 if not os.path.isdir(self.project_path):
670 logging.info('Checking out %s in %s' %
671 (self.project_name, self.project_path))
672 assert self.remote == 'origin'
673 # self.project_path doesn't exist yet.
674 self._check_call_git(
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000675 ['clone', self.git_url, self.project_name, '--quiet'],
676 cwd=self.root_dir,
677 stderr=subprocess2.STDOUT)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000678 try:
679 configured_svn_url = self._check_output_git(
680 ['config', 'svn-remote.svn.url']).strip()
681 except subprocess.CalledProcessError:
682 configured_svn_url = ''
683
684 if configured_svn_url.strip() != self.svn_url:
685 self._check_call_git_svn(
686 ['init',
687 '--prefix', self.remote + '/',
688 '-T', self.trunk,
689 self.svn_url])
690 self._check_call_git_svn(['fetch'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000691 return super(GitSvnPremadeCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000692
693
694class GitSvnCheckout(GitSvnCheckoutBase):
695 """Manages a git-svn clone.
696
697 Using git-svn hides some of the complexity of using a svn checkout.
698 """
699 def __init__(self,
700 root_dir, project_name,
701 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000702 svn_url, trunk, post_processors=None):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000703 super(GitSvnCheckout, self).__init__(
704 root_dir, project_name, 'trunk',
705 commit_user, commit_pwd,
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000706 svn_url, trunk, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000707
maruel@chromium.org51919772011-06-12 01:27:42 +0000708 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000709 """Creates the initial checkout for the repo."""
maruel@chromium.org51919772011-06-12 01:27:42 +0000710 assert not revision, 'Implement revision if necessary'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000711 if not os.path.isdir(self.project_path):
712 logging.info('Checking out %s in %s' %
713 (self.project_name, self.project_path))
714 # TODO: Create a shallow clone.
715 # self.project_path doesn't exist yet.
716 self._check_call_git_svn(
717 ['clone',
718 '--prefix', self.remote + '/',
719 '-T', self.trunk,
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000720 self.svn_url, self.project_path,
721 '--quiet'],
722 cwd=self.root_dir,
723 stderr=subprocess2.STDOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000724 return super(GitSvnCheckout, self).prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000725
726
727class ReadOnlyCheckout(object):
728 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000729 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000730 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000731 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000732 self.post_processors = (post_processors or []) + (
733 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000734
maruel@chromium.org51919772011-06-12 01:27:42 +0000735 def prepare(self, revision):
736 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000737
738 def get_settings(self, key):
739 return self.checkout.get_settings(key)
740
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000741 def apply_patch(self, patches, post_processors=None):
742 return self.checkout.apply_patch(
743 patches, post_processors or self.post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000744
745 def commit(self, message, user): # pylint: disable=R0201
746 logging.info('Would have committed for %s with message: %s' % (
747 user, message))
748 return 'FAKE'
749
750 @property
751 def project_name(self):
752 return self.checkout.project_name
753
754 @property
755 def project_path(self):
756 return self.checkout.project_path