blob: 97a193f9cc59ec0548f207e52463e0acbc6d7a2d [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
maruel@chromium.org9799a072012-01-11 00:26:25 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000010import ConfigParser
11import fnmatch
12import logging
13import os
14import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000015import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000016import subprocess
17import sys
18import tempfile
19
20import patch
21import scm
22import subprocess2
23
24
csharp@chromium.org9af0a112013-03-20 20:21:35 +000025if sys.platform in ('cygwin', 'win32'):
26 # Disable timeouts on Windows since we can't have shells with timeouts.
27 GLOBAL_TIMEOUT = None
28 FETCH_TIMEOUT = None
29else:
30 # Default timeout of 15 minutes.
31 GLOBAL_TIMEOUT = 15*60
32 # Use a larger timeout for checkout since it can be a genuinely slower
33 # operation.
34 FETCH_TIMEOUT = 30*60
35
36
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000037def get_code_review_setting(path, key,
38 codereview_settings_file='codereview.settings'):
39 """Parses codereview.settings and return the value for the key if present.
40
41 Don't cache the values in case the file is changed."""
42 # TODO(maruel): Do not duplicate code.
43 settings = {}
44 try:
45 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
46 try:
47 for line in settings_file.readlines():
48 if not line or line.startswith('#'):
49 continue
50 if not ':' in line:
51 # Invalid file.
52 return None
53 k, v = line.split(':', 1)
54 settings[k.strip()] = v.strip()
55 finally:
56 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000057 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000058 return None
59 return settings.get(key, None)
60
61
maruel@chromium.org4dd9f722012-10-01 16:23:03 +000062def align_stdout(stdout):
63 """Returns the aligned output of multiple stdouts."""
64 output = ''
65 for item in stdout:
66 item = item.strip()
67 if not item:
68 continue
69 output += ''.join(' %s\n' % line for line in item.splitlines())
70 return output
71
72
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000073class PatchApplicationFailed(Exception):
74 """Patch failed to be applied."""
maruel@chromium.org34f68552012-05-09 19:18:36 +000075 def __init__(self, p, status):
76 super(PatchApplicationFailed, self).__init__(p, status)
77 self.patch = p
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000078 self.status = status
79
maruel@chromium.org34f68552012-05-09 19:18:36 +000080 @property
81 def filename(self):
82 if self.patch:
83 return self.patch.filename
84
85 def __str__(self):
86 out = []
87 if self.filename:
88 out.append('Failed to apply patch for %s:' % self.filename)
89 if self.status:
90 out.append(self.status)
maruel@chromium.orgcb5667a2012-10-23 19:42:10 +000091 if self.patch:
92 out.append('Patch: %s' % self.patch.dump())
maruel@chromium.org34f68552012-05-09 19:18:36 +000093 return '\n'.join(out)
94
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000095
96class CheckoutBase(object):
97 # Set to None to have verbose output.
98 VOID = subprocess2.VOID
99
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000100 def __init__(self, root_dir, project_name, post_processors):
101 """
102 Args:
103 post_processor: list of lambda(checkout, patches) to call on each of the
104 modified files.
105 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000106 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000107 self.root_dir = root_dir
108 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000109 if self.project_name is None:
110 self.project_path = self.root_dir
111 else:
112 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000113 # Only used for logging purposes.
114 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000115 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000116 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000117 assert self.project_path
maruel@chromium.org0aca0f92012-10-01 16:39:45 +0000118 assert os.path.isabs(self.project_path)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000119
120 def get_settings(self, key):
121 return get_code_review_setting(self.project_path, key)
122
maruel@chromium.org51919772011-06-12 01:27:42 +0000123 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000124 """Checks out a clean copy of the tree and removes any local modification.
125
126 This function shouldn't throw unless the remote repository is inaccessible,
127 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +0000128
129 Args:
130 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000131 """
132 raise NotImplementedError()
133
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000134 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000135 """Applies a patch and returns the list of modified files.
136
137 This function should throw patch.UnsupportedPatchFormat or
138 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000139
140 Args:
141 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000142 """
143 raise NotImplementedError()
144
145 def commit(self, commit_message, user):
146 """Commits the patch upstream, while impersonating 'user'."""
147 raise NotImplementedError()
148
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000149 def revisions(self, rev1, rev2):
150 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]).
151
152 If rev2 is None, it means 'HEAD'.
153
154 Returns None if there is no link between the two.
155 """
156 raise NotImplementedError()
157
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000158
159class RawCheckout(CheckoutBase):
160 """Used to apply a patch locally without any intent to commit it.
161
162 To be used by the try server.
163 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000164 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000165 """Stubbed out."""
166 pass
167
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000168 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000169 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000170 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000171 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000172 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000173 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000174 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000175 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000176 os.remove(filepath)
177 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000178 else:
179 dirname = os.path.dirname(p.filename)
180 full_dir = os.path.join(self.project_path, dirname)
181 if dirname and not os.path.isdir(full_dir):
182 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000183 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000184
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000185 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000186 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000187 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000188 f.write(content)
189 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000190 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000191 if p.source_filename:
192 if not p.is_new:
193 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000194 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000195 'File has a source filename specified but is not new')
196 # Copy the file first.
197 if os.path.isfile(filepath):
198 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000199 p, 'File exist but was about to be overwriten')
maruel@chromium.org5e975632011-09-29 18:07:06 +0000200 shutil.copy2(
201 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000202 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000203 if p.diff_hunks:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000204 cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel]
205 if verbose:
206 cmd.append('--verbose')
groby@chromium.org23279942013-07-12 19:32:33 +0000207 env = os.environ.copy()
208 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
209 try:
210 stdout.append(
211 subprocess2.check_output(
212 cmd,
213 stdin=p.get(False),
214 stderr=subprocess2.STDOUT,
215 cwd=self.project_path,
216 timeout=GLOBAL_TIMEOUT,
217 env=env))
218 finally:
219 shutil.rmtree(env['TMPDIR'])
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000220 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000221 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000222 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000223 stdout.append('Created an empty file.')
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000224 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000225 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000226 if verbose:
227 print p.filename
228 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000229 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000230 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000231 except subprocess.CalledProcessError, e:
232 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000233 p,
234 'While running %s;\n%s%s' % (
235 ' '.join(e.cmd),
236 align_stdout(stdout),
237 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000238
239 def commit(self, commit_message, user):
240 """Stubbed out."""
241 raise NotImplementedError('RawCheckout can\'t commit')
242
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000243 def revisions(self, _rev1, _rev2):
244 return None
245
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000246
247class SvnConfig(object):
248 """Parses a svn configuration file."""
249 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000250 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000251 self.svn_config_dir = svn_config_dir
252 self.default = not bool(self.svn_config_dir)
253 if not self.svn_config_dir:
254 if sys.platform == 'win32':
255 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
256 else:
257 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
258 svn_config_file = os.path.join(self.svn_config_dir, 'config')
259 parser = ConfigParser.SafeConfigParser()
260 if os.path.isfile(svn_config_file):
261 parser.read(svn_config_file)
262 else:
263 parser.add_section('auto-props')
264 self.auto_props = dict(parser.items('auto-props'))
265
266
267class SvnMixIn(object):
268 """MixIn class to add svn commands common to both svn and git-svn clients."""
269 # These members need to be set by the subclass.
270 commit_user = None
271 commit_pwd = None
272 svn_url = None
273 project_path = None
274 # Override at class level when necessary. If used, --non-interactive is
275 # implied.
276 svn_config = SvnConfig()
277 # Set to True when non-interactivity is necessary but a custom subversion
278 # configuration directory is not necessary.
279 non_interactive = False
280
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000281 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000282 args = ['svn'] + args
283 if not self.svn_config.default:
284 args.extend(['--config-dir', self.svn_config.svn_config_dir])
285 if not self.svn_config.default or self.non_interactive or non_interactive:
286 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000287 if credentials:
288 if self.commit_user:
289 args.extend(['--username', self.commit_user])
290 if self.commit_pwd:
291 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000292 return args
293
294 def _check_call_svn(self, args, **kwargs):
295 """Runs svn and throws an exception if the command failed."""
296 kwargs.setdefault('cwd', self.project_path)
297 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000298 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000299 return subprocess2.check_call_out(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000300 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000301
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000302 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000303 """Runs svn and throws an exception if the command failed.
304
305 Returns the output.
306 """
307 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000308 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000309 self._add_svn_flags(args, True, credentials),
310 stderr=subprocess2.STDOUT,
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000311 timeout=GLOBAL_TIMEOUT,
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000312 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000313
314 @staticmethod
315 def _parse_svn_info(output, key):
316 """Returns value for key from svn info output.
317
318 Case insensitive.
319 """
320 values = {}
321 key = key.lower()
322 for line in output.splitlines(False):
323 if not line:
324 continue
325 k, v = line.split(':', 1)
326 k = k.strip().lower()
327 v = v.strip()
328 assert not k in values
329 values[k] = v
330 return values.get(key, None)
331
332
333class SvnCheckout(CheckoutBase, SvnMixIn):
334 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000335 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
336 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000337 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
338 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000339 self.commit_user = commit_user
340 self.commit_pwd = commit_pwd
341 self.svn_url = svn_url
342 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000343
maruel@chromium.org51919772011-06-12 01:27:42 +0000344 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000345 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000346 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000347 if not os.path.isdir(self.project_path):
348 logging.info('Checking out %s in %s' %
349 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000350 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000351
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000352 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000353 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000354 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000355 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000356 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000357 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000358 # It is important to use credentials=False otherwise credentials could
359 # leak in the error message. Credentials are not necessary here for the
360 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000361 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000362 stdout.append(self._check_output_svn(
363 ['delete', p.filename, '--force'], credentials=False))
364 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000365 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000366 # svn add while creating directories otherwise svn add on the
367 # contained files will silently fail.
368 # First, find the root directory that exists.
369 dirname = os.path.dirname(p.filename)
370 dirs_to_create = []
371 while (dirname and
372 not os.path.isdir(os.path.join(self.project_path, dirname))):
373 dirs_to_create.append(dirname)
374 dirname = os.path.dirname(dirname)
375 for dir_to_create in reversed(dirs_to_create):
376 os.mkdir(os.path.join(self.project_path, dir_to_create))
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000377 stdout.append(
378 self._check_output_svn(
379 ['add', dir_to_create, '--force'], credentials=False))
380 stdout.append('Created missing directory %s.' % dir_to_create)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000381
382 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000383 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000384 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000385 f.write(content)
386 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000387 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000388 if p.source_filename:
389 if not p.is_new:
390 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000391 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000392 'File has a source filename specified but is not new')
393 # Copy the file first.
394 if os.path.isfile(filepath):
395 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000396 p, 'File exist but was about to be overwriten')
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000397 stdout.append(
398 self._check_output_svn(
399 ['copy', p.source_filename, p.filename]))
400 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000401 if p.diff_hunks:
maruel@chromium.orgec4a9182012-09-28 20:39:45 +0000402 cmd = [
403 'patch',
404 '-p%s' % p.patchlevel,
405 '--forward',
406 '--force',
407 '--no-backup-if-mismatch',
408 ]
groby@chromium.org23279942013-07-12 19:32:33 +0000409 env = os.environ.copy()
410 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
411 try:
412 stdout.append(
413 subprocess2.check_output(
414 cmd,
415 stdin=p.get(False),
416 cwd=self.project_path,
417 timeout=GLOBAL_TIMEOUT,
418 env=env))
419 finally:
420 shutil.rmtree(env['TMPDIR'])
421
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000422 elif p.is_new and not os.path.exists(filepath):
423 # There is only a header. Just create the file if it doesn't
424 # exist.
425 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000426 stdout.append('Created an empty file.')
maruel@chromium.org3da83172012-05-07 16:17:20 +0000427 if p.is_new and not p.source_filename:
428 # Do not run it if p.source_filename is defined, since svn copy was
429 # using above.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000430 stdout.append(
431 self._check_output_svn(
432 ['add', p.filename, '--force'], credentials=False))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000433 for name, value in p.svn_properties:
434 if value is None:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000435 stdout.append(
436 self._check_output_svn(
437 ['propdel', '--quiet', name, p.filename],
438 credentials=False))
439 stdout.append('Property %s deleted.' % name)
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000440 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000441 stdout.append(
442 self._check_output_svn(
443 ['propset', name, value, p.filename], credentials=False))
444 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000445 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000446 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000447 for value in values.split(';'):
448 if '=' not in value:
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000449 params = [value, '.']
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000450 else:
451 params = value.split('=', 1)
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000452 if params[1] == '*':
453 # Works around crbug.com/150960 on Windows.
454 params[1] = '.'
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000455 stdout.append(
456 self._check_output_svn(
457 ['propset'] + params + [p.filename], credentials=False))
458 stdout.append('Property (auto) %s' % '='.join(params))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000459 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000460 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000461 if verbose:
462 print p.filename
463 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000464 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000465 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000466 except subprocess.CalledProcessError, e:
467 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000468 p,
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000469 'While running %s;\n%s%s' % (
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000470 ' '.join(e.cmd),
471 align_stdout(stdout),
472 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000473
474 def commit(self, commit_message, user):
475 logging.info('Committing patch for %s' % user)
476 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000477 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000478 handle, commit_filename = tempfile.mkstemp(text=True)
479 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000480 # Shouldn't assume default encoding is UTF-8. But really, if you are using
481 # anything else, you are living in another world.
482 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000483 os.close(handle)
484 # When committing, svn won't update the Revision metadata of the checkout,
485 # so if svn commit returns "Committed revision 3.", svn info will still
486 # return "Revision: 2". Since running svn update right after svn commit
487 # creates a race condition with other committers, this code _must_ parse
488 # the output of svn commit and use a regexp to grab the revision number.
489 # Note that "Committed revision N." is localized but subprocess2 forces
490 # LANGUAGE=en.
491 args = ['commit', '--file', commit_filename]
492 # realauthor is parsed by a server-side hook.
493 if user and user != self.commit_user:
494 args.extend(['--with-revprop', 'realauthor=%s' % user])
495 out = self._check_output_svn(args)
496 finally:
497 os.remove(commit_filename)
498 lines = filter(None, out.splitlines())
499 match = re.match(r'^Committed revision (\d+).$', lines[-1])
500 if not match:
501 raise PatchApplicationFailed(
502 None,
503 'Couldn\'t make sense out of svn commit message:\n' + out)
504 return int(match.group(1))
505
maruel@chromium.org51919772011-06-12 01:27:42 +0000506 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000507 """Reverts local modifications or checks out if the directory is not
508 present. Use depot_tools's functionality to do this.
509 """
510 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000511 if revision:
512 flags.extend(['--revision', str(revision)])
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000513 if os.path.isdir(self.project_path):
514 # This may remove any part (or all) of the checkout.
515 scm.SVN.Revert(self.project_path, no_ignore=True)
516
517 if os.path.isdir(self.project_path):
518 # Revive files that were deleted in scm.SVN.Revert().
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000519 self._check_call_svn(['update', '--force'] + flags,
520 timeout=FETCH_TIMEOUT)
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000521 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000522 logging.info(
523 'Directory %s is not present, checking it out.' % self.project_path)
524 self._check_call_svn(
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000525 ['checkout', self.svn_url, self.project_path] + flags, cwd=None,
526 timeout=FETCH_TIMEOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000527 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000528
maruel@chromium.org51919772011-06-12 01:27:42 +0000529 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000530 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000531 revision = int(self._parse_svn_info(out, 'revision'))
532 if revision != self._last_seen_revision:
533 logging.info('Updated to revision %d' % revision)
534 self._last_seen_revision = revision
535 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000536
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000537 def revisions(self, rev1, rev2):
538 """Returns the number of actual commits, not just the difference between
539 numbers.
540 """
541 rev2 = rev2 or 'HEAD'
542 # Revision range is inclusive and ordering doesn't matter, they'll appear in
543 # the order specified.
544 try:
545 out = self._check_output_svn(
546 ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)])
547 except subprocess.CalledProcessError:
548 return None
549 # Ignore the '----' lines.
550 return len([l for l in out.splitlines() if l.startswith('r')]) - 1
551
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000552
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000553class GitCheckout(CheckoutBase):
554 """Manages a git checkout."""
555 def __init__(self, root_dir, project_name, remote_branch, git_url,
556 commit_user, post_processors=None):
557 assert git_url
558 assert commit_user
559 super(GitCheckout, self).__init__(root_dir, project_name, post_processors)
560 self.git_url = git_url
561 self.commit_user = commit_user
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000562 self.remote_branch = remote_branch
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000563 # The working branch where patches will be applied. It will track the
564 # remote branch.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000565 self.working_branch = 'working_branch'
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000566 # There is no reason to not hardcode origin.
567 self.remote = 'origin'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000568
maruel@chromium.org51919772011-06-12 01:27:42 +0000569 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000570 """Resets the git repository in a clean state.
571
572 Checks it out if not present and deletes the working branch.
573 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000574 assert self.remote_branch
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000575
576 if not os.path.isdir(self.project_path):
577 # Clone the repo if the directory is not present.
578 logging.info(
579 'Checking out %s in %s', self.project_name, self.project_path)
580 self._check_call_git(
581 ['clone', self.git_url, '-b', self.remote_branch, self.project_path],
582 cwd=None, timeout=FETCH_TIMEOUT)
583 else:
584 # Throw away all uncommitted changes in the existing checkout.
585 self._check_call_git(['checkout', self.remote_branch])
586 self._check_call_git(
587 ['reset', '--hard', '--quiet',
588 '%s/%s' % (self.remote, self.remote_branch)])
589
maruel@chromium.org51919772011-06-12 01:27:42 +0000590 if revision:
591 try:
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000592 # Look if the commit hash already exist. If so, we can skip a
593 # 'git fetch' call.
maruel@chromium.org51919772011-06-12 01:27:42 +0000594 revision = self._check_output_git(['rev-parse', revision])
595 except subprocess.CalledProcessError:
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000596 self._check_call_git(
597 ['fetch', self.remote, self.remote_branch, '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000598 revision = self._check_output_git(['rev-parse', revision])
599 self._check_call_git(['checkout', '--force', '--quiet', revision])
600 else:
601 branches, active = self._branches()
602 if active != 'master':
603 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000604 self._sync_remote_branch()
605
maruel@chromium.org51919772011-06-12 01:27:42 +0000606 if self.working_branch in branches:
607 self._call_git(['branch', '-D', self.working_branch])
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000608 return self._get_head_commit_hash()
609
610 def _sync_remote_branch(self):
611 """Syncs the remote branch."""
612 # We do a 'git pull origin master:refs/remotes/origin/master' instead of
613 # 'git pull origin master' because from the manpage for git-pull:
614 # A parameter <ref> without a colon is equivalent to <ref>: when
615 # pulling/fetching, so it merges <ref> into the current branch without
616 # storing the remote branch anywhere locally.
617 remote_tracked_path = 'refs/remotes/%s/%s' % (
618 self.remote, self.remote_branch)
619 self._check_call_git(
620 ['pull', self.remote,
621 '%s:%s' % (self.remote_branch, remote_tracked_path),
622 '--quiet'])
623
624 def _get_head_commit_hash(self):
rmistry@google.com11145db2013-10-03 12:43:40 +0000625 """Gets the current revision (in unicode) from the local branch."""
626 return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip())
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000627
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000628 def apply_patch(self, patches, post_processors=None, verbose=False):
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000629 """Applies a patch on 'working_branch' and switches to it.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000630
631 Also commits the changes on the local branch.
632
633 Ignores svn properties and raise an exception on unexpected ones.
634 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000635 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000636 # It this throws, the checkout is corrupted. Maybe worth deleting it and
637 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000638 if self.remote_branch:
639 self._check_call_git(
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000640 ['checkout', '-b', self.working_branch, '-t', self.remote_branch,
641 '--quiet'])
642
maruel@chromium.org5e975632011-09-29 18:07:06 +0000643 for index, p in enumerate(patches):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000644 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000645 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000646 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000647 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000648 if (not os.path.exists(filepath) and
maruel@chromium.org5e975632011-09-29 18:07:06 +0000649 any(p1.source_filename == p.filename for p1 in patches[0:index])):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000650 # The file was already deleted if a prior patch with file rename
651 # was already processed because 'git apply' did it for us.
maruel@chromium.org5e975632011-09-29 18:07:06 +0000652 pass
653 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000654 stdout.append(self._check_output_git(['rm', p.filename]))
655 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000656 else:
657 dirname = os.path.dirname(p.filename)
658 full_dir = os.path.join(self.project_path, dirname)
659 if dirname and not os.path.isdir(full_dir):
660 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000661 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000662 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000663 content = p.get()
664 with open(filepath, 'wb') as f:
665 f.write(content)
666 stdout.append('Added binary file %d bytes' % len(content))
667 cmd = ['add', p.filename]
668 if verbose:
669 cmd.append('--verbose')
670 stdout.append(self._check_output_git(cmd))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000671 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000672 # No need to do anything special with p.is_new or if not
673 # p.diff_hunks. git apply manages all that already.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000674 cmd = ['apply', '--index', '-p%s' % p.patchlevel]
675 if verbose:
676 cmd.append('--verbose')
677 stdout.append(self._check_output_git(cmd, stdin=p.get(True)))
678 for name, value in p.svn_properties:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000679 # Ignore some known auto-props flags through .subversion/config,
680 # bails out on the other ones.
681 # TODO(maruel): Read ~/.subversion/config and detect the rules that
682 # applies here to figure out if the property will be correctly
683 # handled.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000684 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000685 if not name in (
maruel@chromium.org9799a072012-01-11 00:26:25 +0000686 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000687 raise patch.UnsupportedPatchFormat(
688 p.filename,
689 'Cannot apply svn property %s to file %s.' % (
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000690 name, p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000691 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000692 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000693 if verbose:
694 print p.filename
695 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000696 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000697 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000698 except subprocess.CalledProcessError, e:
699 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000700 p,
701 'While running %s;\n%s%s' % (
702 ' '.join(e.cmd),
703 align_stdout(stdout),
704 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000705 # Once all the patches are processed and added to the index, commit the
706 # index.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000707 cmd = ['commit', '-m', 'Committed patch']
708 if verbose:
709 cmd.append('--verbose')
710 self._check_call_git(cmd)
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000711 found_files = self._check_output_git(
712 ['diff', '%s/%s' % (self.remote, self.remote_branch),
713 '--name-only']).splitlines(False)
714 assert sorted(patches.filenames) == sorted(found_files), (
715 sorted(patches.filenames), sorted(found_files))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000716
717 def commit(self, commit_message, user):
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000718 """Commits, updates the commit message and pushes."""
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000719 assert isinstance(commit_message, unicode)
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000720 current_branch = self._check_output_git(
721 ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
722 assert current_branch == self.working_branch
723
724 commit_cmd = ['commit', '--amend', '-m', commit_message]
725 if user and user != self.commit_user:
726 # We do not have the first or last name of the user, grab the username
727 # from the email and call it the original author's name.
728 # TODO(rmistry): Do not need the below if user is already in
729 # "Name <email>" format.
730 name = user.split('@')[0]
731 commit_cmd.extend(['--author', '%s <%s>' % (name, user)])
732 self._check_call_git(commit_cmd)
733
734 # Push to the remote repository.
735 self._check_call_git(
736 ['push', 'origin', '%s:%s' % (self.working_branch, self.remote_branch),
737 '--force', '--quiet'])
738 # Get the revision after the push.
739 revision = self._get_head_commit_hash()
740 # Switch back to the remote_branch and sync it.
741 self._check_call_git(['checkout', self.remote_branch])
742 self._sync_remote_branch()
743 # Delete the working branch since we are done with it.
744 self._check_call_git(['branch', '-D', self.working_branch])
745
746 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000747
748 def _check_call_git(self, args, **kwargs):
749 kwargs.setdefault('cwd', self.project_path)
750 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000751 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000752 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000753
754 def _call_git(self, args, **kwargs):
755 """Like check_call but doesn't throw on failure."""
756 kwargs.setdefault('cwd', self.project_path)
757 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000758 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000759 return subprocess2.call(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000760
761 def _check_output_git(self, args, **kwargs):
762 kwargs.setdefault('cwd', self.project_path)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000763 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000764 return subprocess2.check_output(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000765 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000766
767 def _branches(self):
768 """Returns the list of branches and the active one."""
769 out = self._check_output_git(['branch']).splitlines(False)
770 branches = [l[2:] for l in out]
771 active = None
772 for l in out:
773 if l.startswith('*'):
774 active = l[2:]
775 break
776 return branches, active
777
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000778 def revisions(self, rev1, rev2):
779 """Returns the number of actual commits between both hash."""
780 self._fetch_remote()
781
782 rev2 = rev2 or '%s/%s' % (self.remote, self.remote_branch)
783 # Revision range is ]rev1, rev2] and ordering matters.
784 try:
785 out = self._check_output_git(
786 ['log', '--format="%H"' , '%s..%s' % (rev1, rev2)])
787 except subprocess.CalledProcessError:
788 return None
789 return len(out.splitlines())
790
791 def _fetch_remote(self):
792 """Fetches the remote without rebasing."""
rmistry@google.com3b5efdf2013-09-05 11:59:40 +0000793 # git fetch is always verbose even with -q, so redirect its output.
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000794 self._check_output_git(['fetch', self.remote, self.remote_branch],
795 timeout=FETCH_TIMEOUT)
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000796
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000797
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000798class ReadOnlyCheckout(object):
799 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000800 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000801 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000802 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000803 self.post_processors = (post_processors or []) + (
804 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000805
maruel@chromium.org51919772011-06-12 01:27:42 +0000806 def prepare(self, revision):
807 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000808
809 def get_settings(self, key):
810 return self.checkout.get_settings(key)
811
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000812 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000813 return self.checkout.apply_patch(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000814 patches, post_processors or self.post_processors, verbose)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000815
816 def commit(self, message, user): # pylint: disable=R0201
817 logging.info('Would have committed for %s with message: %s' % (
818 user, message))
819 return 'FAKE'
820
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000821 def revisions(self, rev1, rev2):
822 return self.checkout.revisions(rev1, rev2)
823
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000824 @property
825 def project_name(self):
826 return self.checkout.project_name
827
828 @property
829 def project_path(self):
830 return self.checkout.project_path