blob: d1a694b453e9debc6d53683829023b2b7da8d3db [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
maruel@chromium.org9799a072012-01-11 00:26:25 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000010import ConfigParser
11import fnmatch
12import logging
13import os
14import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000015import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000016import subprocess
17import sys
18import tempfile
19
20import patch
21import scm
22import subprocess2
23
24
csharp@chromium.org9af0a112013-03-20 20:21:35 +000025if sys.platform in ('cygwin', 'win32'):
26 # Disable timeouts on Windows since we can't have shells with timeouts.
27 GLOBAL_TIMEOUT = None
28 FETCH_TIMEOUT = None
29else:
30 # Default timeout of 15 minutes.
31 GLOBAL_TIMEOUT = 15*60
32 # Use a larger timeout for checkout since it can be a genuinely slower
33 # operation.
34 FETCH_TIMEOUT = 30*60
35
36
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000037def get_code_review_setting(path, key,
38 codereview_settings_file='codereview.settings'):
39 """Parses codereview.settings and return the value for the key if present.
40
41 Don't cache the values in case the file is changed."""
42 # TODO(maruel): Do not duplicate code.
43 settings = {}
44 try:
45 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
46 try:
47 for line in settings_file.readlines():
48 if not line or line.startswith('#'):
49 continue
50 if not ':' in line:
51 # Invalid file.
52 return None
53 k, v = line.split(':', 1)
54 settings[k.strip()] = v.strip()
55 finally:
56 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000057 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000058 return None
59 return settings.get(key, None)
60
61
maruel@chromium.org4dd9f722012-10-01 16:23:03 +000062def align_stdout(stdout):
63 """Returns the aligned output of multiple stdouts."""
64 output = ''
65 for item in stdout:
66 item = item.strip()
67 if not item:
68 continue
69 output += ''.join(' %s\n' % line for line in item.splitlines())
70 return output
71
72
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000073class PatchApplicationFailed(Exception):
74 """Patch failed to be applied."""
maruel@chromium.org34f68552012-05-09 19:18:36 +000075 def __init__(self, p, status):
76 super(PatchApplicationFailed, self).__init__(p, status)
77 self.patch = p
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000078 self.status = status
79
maruel@chromium.org34f68552012-05-09 19:18:36 +000080 @property
81 def filename(self):
82 if self.patch:
83 return self.patch.filename
84
85 def __str__(self):
86 out = []
87 if self.filename:
88 out.append('Failed to apply patch for %s:' % self.filename)
89 if self.status:
90 out.append(self.status)
maruel@chromium.orgcb5667a2012-10-23 19:42:10 +000091 if self.patch:
92 out.append('Patch: %s' % self.patch.dump())
maruel@chromium.org34f68552012-05-09 19:18:36 +000093 return '\n'.join(out)
94
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000095
96class CheckoutBase(object):
97 # Set to None to have verbose output.
98 VOID = subprocess2.VOID
99
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000100 def __init__(self, root_dir, project_name, post_processors):
101 """
102 Args:
103 post_processor: list of lambda(checkout, patches) to call on each of the
104 modified files.
105 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000106 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000107 self.root_dir = root_dir
108 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000109 if self.project_name is None:
110 self.project_path = self.root_dir
111 else:
112 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000113 # Only used for logging purposes.
114 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000115 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000116 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000117 assert self.project_path
maruel@chromium.org0aca0f92012-10-01 16:39:45 +0000118 assert os.path.isabs(self.project_path)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000119
120 def get_settings(self, key):
121 return get_code_review_setting(self.project_path, key)
122
maruel@chromium.org51919772011-06-12 01:27:42 +0000123 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000124 """Checks out a clean copy of the tree and removes any local modification.
125
126 This function shouldn't throw unless the remote repository is inaccessible,
127 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +0000128
129 Args:
130 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000131 """
132 raise NotImplementedError()
133
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000134 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000135 """Applies a patch and returns the list of modified files.
136
137 This function should throw patch.UnsupportedPatchFormat or
138 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000139
140 Args:
141 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000142 """
143 raise NotImplementedError()
144
145 def commit(self, commit_message, user):
146 """Commits the patch upstream, while impersonating 'user'."""
147 raise NotImplementedError()
148
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000149 def revisions(self, rev1, rev2):
150 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]).
151
152 If rev2 is None, it means 'HEAD'.
153
154 Returns None if there is no link between the two.
155 """
156 raise NotImplementedError()
157
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000158
159class RawCheckout(CheckoutBase):
160 """Used to apply a patch locally without any intent to commit it.
161
162 To be used by the try server.
163 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000164 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000165 """Stubbed out."""
166 pass
167
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000168 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000169 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000170 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000171 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000172 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000173 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000174 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000175 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000176 os.remove(filepath)
177 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000178 else:
179 dirname = os.path.dirname(p.filename)
180 full_dir = os.path.join(self.project_path, dirname)
181 if dirname and not os.path.isdir(full_dir):
182 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000183 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000184
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000185 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000186 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000187 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000188 f.write(content)
189 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000190 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000191 if p.source_filename:
192 if not p.is_new:
193 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000194 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000195 'File has a source filename specified but is not new')
196 # Copy the file first.
197 if os.path.isfile(filepath):
198 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000199 p, 'File exist but was about to be overwriten')
maruel@chromium.org5e975632011-09-29 18:07:06 +0000200 shutil.copy2(
201 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000202 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000203 if p.diff_hunks:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000204 cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel]
205 if verbose:
206 cmd.append('--verbose')
groby@chromium.org23279942013-07-12 19:32:33 +0000207 env = os.environ.copy()
208 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
209 try:
210 stdout.append(
211 subprocess2.check_output(
212 cmd,
213 stdin=p.get(False),
214 stderr=subprocess2.STDOUT,
215 cwd=self.project_path,
216 timeout=GLOBAL_TIMEOUT,
217 env=env))
218 finally:
219 shutil.rmtree(env['TMPDIR'])
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000220 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000221 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000222 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000223 stdout.append('Created an empty file.')
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000224 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000225 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000226 if verbose:
227 print p.filename
228 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000229 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000230 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000231 except subprocess.CalledProcessError, e:
232 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000233 p,
234 'While running %s;\n%s%s' % (
235 ' '.join(e.cmd),
236 align_stdout(stdout),
237 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000238
239 def commit(self, commit_message, user):
240 """Stubbed out."""
241 raise NotImplementedError('RawCheckout can\'t commit')
242
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000243 def revisions(self, _rev1, _rev2):
244 return None
245
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000246
247class SvnConfig(object):
248 """Parses a svn configuration file."""
249 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000250 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000251 self.svn_config_dir = svn_config_dir
252 self.default = not bool(self.svn_config_dir)
253 if not self.svn_config_dir:
254 if sys.platform == 'win32':
255 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
256 else:
257 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
258 svn_config_file = os.path.join(self.svn_config_dir, 'config')
259 parser = ConfigParser.SafeConfigParser()
260 if os.path.isfile(svn_config_file):
261 parser.read(svn_config_file)
262 else:
263 parser.add_section('auto-props')
264 self.auto_props = dict(parser.items('auto-props'))
265
266
267class SvnMixIn(object):
268 """MixIn class to add svn commands common to both svn and git-svn clients."""
269 # These members need to be set by the subclass.
270 commit_user = None
271 commit_pwd = None
272 svn_url = None
273 project_path = None
274 # Override at class level when necessary. If used, --non-interactive is
275 # implied.
276 svn_config = SvnConfig()
277 # Set to True when non-interactivity is necessary but a custom subversion
278 # configuration directory is not necessary.
279 non_interactive = False
280
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000281 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000282 args = ['svn'] + args
283 if not self.svn_config.default:
284 args.extend(['--config-dir', self.svn_config.svn_config_dir])
285 if not self.svn_config.default or self.non_interactive or non_interactive:
286 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000287 if credentials:
288 if self.commit_user:
289 args.extend(['--username', self.commit_user])
290 if self.commit_pwd:
291 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000292 return args
293
294 def _check_call_svn(self, args, **kwargs):
295 """Runs svn and throws an exception if the command failed."""
296 kwargs.setdefault('cwd', self.project_path)
297 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000298 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000299 return subprocess2.check_call_out(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000300 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000301
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000302 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000303 """Runs svn and throws an exception if the command failed.
304
305 Returns the output.
306 """
307 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000308 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000309 self._add_svn_flags(args, True, credentials),
310 stderr=subprocess2.STDOUT,
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000311 timeout=GLOBAL_TIMEOUT,
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000312 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000313
314 @staticmethod
315 def _parse_svn_info(output, key):
316 """Returns value for key from svn info output.
317
318 Case insensitive.
319 """
320 values = {}
321 key = key.lower()
322 for line in output.splitlines(False):
323 if not line:
324 continue
325 k, v = line.split(':', 1)
326 k = k.strip().lower()
327 v = v.strip()
328 assert not k in values
329 values[k] = v
330 return values.get(key, None)
331
332
333class SvnCheckout(CheckoutBase, SvnMixIn):
334 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000335 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
336 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000337 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
338 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000339 self.commit_user = commit_user
340 self.commit_pwd = commit_pwd
341 self.svn_url = svn_url
342 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000343
maruel@chromium.org51919772011-06-12 01:27:42 +0000344 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000345 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000346 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000347 if not os.path.isdir(self.project_path):
348 logging.info('Checking out %s in %s' %
349 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000350 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000351
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000352 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000353 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000354 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000355 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000356 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000357 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000358 # It is important to use credentials=False otherwise credentials could
359 # leak in the error message. Credentials are not necessary here for the
360 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000361 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000362 stdout.append(self._check_output_svn(
363 ['delete', p.filename, '--force'], credentials=False))
364 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000365 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000366 # svn add while creating directories otherwise svn add on the
367 # contained files will silently fail.
368 # First, find the root directory that exists.
369 dirname = os.path.dirname(p.filename)
370 dirs_to_create = []
371 while (dirname and
372 not os.path.isdir(os.path.join(self.project_path, dirname))):
373 dirs_to_create.append(dirname)
374 dirname = os.path.dirname(dirname)
375 for dir_to_create in reversed(dirs_to_create):
376 os.mkdir(os.path.join(self.project_path, dir_to_create))
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000377 stdout.append(
378 self._check_output_svn(
379 ['add', dir_to_create, '--force'], credentials=False))
380 stdout.append('Created missing directory %s.' % dir_to_create)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000381
382 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000383 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000384 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000385 f.write(content)
386 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000387 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000388 if p.source_filename:
389 if not p.is_new:
390 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000391 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000392 'File has a source filename specified but is not new')
393 # Copy the file first.
394 if os.path.isfile(filepath):
395 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000396 p, 'File exist but was about to be overwriten')
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000397 stdout.append(
398 self._check_output_svn(
399 ['copy', p.source_filename, p.filename]))
400 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000401 if p.diff_hunks:
maruel@chromium.orgec4a9182012-09-28 20:39:45 +0000402 cmd = [
403 'patch',
404 '-p%s' % p.patchlevel,
405 '--forward',
406 '--force',
407 '--no-backup-if-mismatch',
408 ]
groby@chromium.org23279942013-07-12 19:32:33 +0000409 env = os.environ.copy()
410 env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
411 try:
412 stdout.append(
413 subprocess2.check_output(
414 cmd,
415 stdin=p.get(False),
416 cwd=self.project_path,
417 timeout=GLOBAL_TIMEOUT,
418 env=env))
419 finally:
420 shutil.rmtree(env['TMPDIR'])
421
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000422 elif p.is_new and not os.path.exists(filepath):
423 # There is only a header. Just create the file if it doesn't
424 # exist.
425 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000426 stdout.append('Created an empty file.')
maruel@chromium.org3da83172012-05-07 16:17:20 +0000427 if p.is_new and not p.source_filename:
428 # Do not run it if p.source_filename is defined, since svn copy was
429 # using above.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000430 stdout.append(
431 self._check_output_svn(
432 ['add', p.filename, '--force'], credentials=False))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000433 for name, value in p.svn_properties:
434 if value is None:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000435 stdout.append(
436 self._check_output_svn(
437 ['propdel', '--quiet', name, p.filename],
438 credentials=False))
439 stdout.append('Property %s deleted.' % name)
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000440 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000441 stdout.append(
442 self._check_output_svn(
443 ['propset', name, value, p.filename], credentials=False))
444 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000445 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000446 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000447 for value in values.split(';'):
448 if '=' not in value:
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000449 params = [value, '.']
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000450 else:
451 params = value.split('=', 1)
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000452 if params[1] == '*':
453 # Works around crbug.com/150960 on Windows.
454 params[1] = '.'
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000455 stdout.append(
456 self._check_output_svn(
457 ['propset'] + params + [p.filename], credentials=False))
458 stdout.append('Property (auto) %s' % '='.join(params))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000459 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000460 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000461 if verbose:
462 print p.filename
463 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000464 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000465 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000466 except subprocess.CalledProcessError, e:
467 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000468 p,
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000469 'While running %s;\n%s%s' % (
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000470 ' '.join(e.cmd),
471 align_stdout(stdout),
472 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000473
474 def commit(self, commit_message, user):
475 logging.info('Committing patch for %s' % user)
476 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000477 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000478 handle, commit_filename = tempfile.mkstemp(text=True)
479 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000480 # Shouldn't assume default encoding is UTF-8. But really, if you are using
481 # anything else, you are living in another world.
482 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000483 os.close(handle)
484 # When committing, svn won't update the Revision metadata of the checkout,
485 # so if svn commit returns "Committed revision 3.", svn info will still
486 # return "Revision: 2". Since running svn update right after svn commit
487 # creates a race condition with other committers, this code _must_ parse
488 # the output of svn commit and use a regexp to grab the revision number.
489 # Note that "Committed revision N." is localized but subprocess2 forces
490 # LANGUAGE=en.
491 args = ['commit', '--file', commit_filename]
492 # realauthor is parsed by a server-side hook.
493 if user and user != self.commit_user:
494 args.extend(['--with-revprop', 'realauthor=%s' % user])
495 out = self._check_output_svn(args)
496 finally:
497 os.remove(commit_filename)
498 lines = filter(None, out.splitlines())
499 match = re.match(r'^Committed revision (\d+).$', lines[-1])
500 if not match:
501 raise PatchApplicationFailed(
502 None,
503 'Couldn\'t make sense out of svn commit message:\n' + out)
504 return int(match.group(1))
505
maruel@chromium.org51919772011-06-12 01:27:42 +0000506 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000507 """Reverts local modifications or checks out if the directory is not
508 present. Use depot_tools's functionality to do this.
509 """
510 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000511 if revision:
512 flags.extend(['--revision', str(revision)])
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000513 if os.path.isdir(self.project_path):
514 # This may remove any part (or all) of the checkout.
515 scm.SVN.Revert(self.project_path, no_ignore=True)
516
517 if os.path.isdir(self.project_path):
518 # Revive files that were deleted in scm.SVN.Revert().
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000519 self._check_call_svn(['update', '--force'] + flags,
520 timeout=FETCH_TIMEOUT)
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000521 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000522 logging.info(
523 'Directory %s is not present, checking it out.' % self.project_path)
524 self._check_call_svn(
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000525 ['checkout', self.svn_url, self.project_path] + flags, cwd=None,
526 timeout=FETCH_TIMEOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000527 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000528
maruel@chromium.org51919772011-06-12 01:27:42 +0000529 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000530 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000531 revision = int(self._parse_svn_info(out, 'revision'))
532 if revision != self._last_seen_revision:
533 logging.info('Updated to revision %d' % revision)
534 self._last_seen_revision = revision
535 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000536
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000537 def revisions(self, rev1, rev2):
538 """Returns the number of actual commits, not just the difference between
539 numbers.
540 """
541 rev2 = rev2 or 'HEAD'
542 # Revision range is inclusive and ordering doesn't matter, they'll appear in
543 # the order specified.
544 try:
545 out = self._check_output_svn(
546 ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)])
547 except subprocess.CalledProcessError:
548 return None
549 # Ignore the '----' lines.
550 return len([l for l in out.splitlines() if l.startswith('r')]) - 1
551
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000552
553class GitCheckoutBase(CheckoutBase):
554 """Base class for git checkout. Not to be used as-is."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000555 def __init__(self, root_dir, project_name, remote_branch,
556 post_processors=None):
557 super(GitCheckoutBase, self).__init__(
558 root_dir, project_name, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000559 # There is no reason to not hardcode it.
560 self.remote = 'origin'
561 self.remote_branch = remote_branch
562 self.working_branch = 'working_branch'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000563
maruel@chromium.org51919772011-06-12 01:27:42 +0000564 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000565 """Resets the git repository in a clean state.
566
567 Checks it out if not present and deletes the working branch.
568 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000569 assert self.remote_branch
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000570 assert os.path.isdir(self.project_path)
571 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000572 if revision:
573 try:
574 revision = self._check_output_git(['rev-parse', revision])
575 except subprocess.CalledProcessError:
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000576 self._check_call_git(
577 ['fetch', self.remote, self.remote_branch, '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000578 revision = self._check_output_git(['rev-parse', revision])
579 self._check_call_git(['checkout', '--force', '--quiet', revision])
580 else:
581 branches, active = self._branches()
582 if active != 'master':
583 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
584 self._check_call_git(['pull', self.remote, self.remote_branch, '--quiet'])
585 if self.working_branch in branches:
586 self._call_git(['branch', '-D', self.working_branch])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000587
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000588 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000589 """Applies a patch on 'working_branch' and switch to it.
590
591 Also commits the changes on the local branch.
592
593 Ignores svn properties and raise an exception on unexpected ones.
594 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000595 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000596 # It this throws, the checkout is corrupted. Maybe worth deleting it and
597 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000598 if self.remote_branch:
599 self._check_call_git(
600 ['checkout', '-b', self.working_branch,
601 '%s/%s' % (self.remote, self.remote_branch), '--quiet'])
maruel@chromium.org5e975632011-09-29 18:07:06 +0000602 for index, p in enumerate(patches):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000603 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000604 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000605 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000606 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000607 if (not os.path.exists(filepath) and
maruel@chromium.org5e975632011-09-29 18:07:06 +0000608 any(p1.source_filename == p.filename for p1 in patches[0:index])):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000609 # The file was already deleted if a prior patch with file rename
610 # was already processed because 'git apply' did it for us.
maruel@chromium.org5e975632011-09-29 18:07:06 +0000611 pass
612 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000613 stdout.append(self._check_output_git(['rm', p.filename]))
614 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000615 else:
616 dirname = os.path.dirname(p.filename)
617 full_dir = os.path.join(self.project_path, dirname)
618 if dirname and not os.path.isdir(full_dir):
619 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000620 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000621 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000622 content = p.get()
623 with open(filepath, 'wb') as f:
624 f.write(content)
625 stdout.append('Added binary file %d bytes' % len(content))
626 cmd = ['add', p.filename]
627 if verbose:
628 cmd.append('--verbose')
629 stdout.append(self._check_output_git(cmd))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000630 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000631 # No need to do anything special with p.is_new or if not
632 # p.diff_hunks. git apply manages all that already.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000633 cmd = ['apply', '--index', '-p%s' % p.patchlevel]
634 if verbose:
635 cmd.append('--verbose')
636 stdout.append(self._check_output_git(cmd, stdin=p.get(True)))
637 for name, value in p.svn_properties:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000638 # Ignore some known auto-props flags through .subversion/config,
639 # bails out on the other ones.
640 # TODO(maruel): Read ~/.subversion/config and detect the rules that
641 # applies here to figure out if the property will be correctly
642 # handled.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000643 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000644 if not name in (
maruel@chromium.org9799a072012-01-11 00:26:25 +0000645 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000646 raise patch.UnsupportedPatchFormat(
647 p.filename,
648 'Cannot apply svn property %s to file %s.' % (
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000649 name, p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000650 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000651 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000652 if verbose:
653 print p.filename
654 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000655 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000656 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000657 except subprocess.CalledProcessError, e:
658 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000659 p,
660 'While running %s;\n%s%s' % (
661 ' '.join(e.cmd),
662 align_stdout(stdout),
663 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000664 # Once all the patches are processed and added to the index, commit the
665 # index.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000666 cmd = ['commit', '-m', 'Committed patch']
667 if verbose:
668 cmd.append('--verbose')
669 self._check_call_git(cmd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000670 # TODO(maruel): Weirdly enough they don't match, need to investigate.
671 #found_files = self._check_output_git(
672 # ['diff', 'master', '--name-only']).splitlines(False)
673 #assert sorted(patches.filenames) == sorted(found_files), (
674 # sorted(out), sorted(found_files))
675
676 def commit(self, commit_message, user):
677 """Updates the commit message.
678
679 Subclass needs to dcommit or push.
680 """
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000681 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000682 self._check_call_git(['commit', '--amend', '-m', commit_message])
683 return self._check_output_git(['rev-parse', 'HEAD']).strip()
684
685 def _check_call_git(self, args, **kwargs):
686 kwargs.setdefault('cwd', self.project_path)
687 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000688 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000689 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000690
691 def _call_git(self, args, **kwargs):
692 """Like check_call but doesn't throw on failure."""
693 kwargs.setdefault('cwd', self.project_path)
694 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000695 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000696 return subprocess2.call(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000697
698 def _check_output_git(self, args, **kwargs):
699 kwargs.setdefault('cwd', self.project_path)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000700 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000701 return subprocess2.check_output(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000702 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000703
704 def _branches(self):
705 """Returns the list of branches and the active one."""
706 out = self._check_output_git(['branch']).splitlines(False)
707 branches = [l[2:] for l in out]
708 active = None
709 for l in out:
710 if l.startswith('*'):
711 active = l[2:]
712 break
713 return branches, active
714
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000715 def revisions(self, rev1, rev2):
716 """Returns the number of actual commits between both hash."""
717 self._fetch_remote()
718
719 rev2 = rev2 or '%s/%s' % (self.remote, self.remote_branch)
720 # Revision range is ]rev1, rev2] and ordering matters.
721 try:
722 out = self._check_output_git(
723 ['log', '--format="%H"' , '%s..%s' % (rev1, rev2)])
724 except subprocess.CalledProcessError:
725 return None
726 return len(out.splitlines())
727
728 def _fetch_remote(self):
729 """Fetches the remote without rebasing."""
730 raise NotImplementedError()
731
732
733class GitCheckout(GitCheckoutBase):
734 """Git checkout implementation."""
735 def _fetch_remote(self):
736 # git fetch is always verbose even with -q -q so redirect its output.
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000737 self._check_output_git(['fetch', self.remote, self.remote_branch],
738 timeout=FETCH_TIMEOUT)
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000739
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000740
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000741class ReadOnlyCheckout(object):
742 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000743 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000744 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000745 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000746 self.post_processors = (post_processors or []) + (
747 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000748
maruel@chromium.org51919772011-06-12 01:27:42 +0000749 def prepare(self, revision):
750 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000751
752 def get_settings(self, key):
753 return self.checkout.get_settings(key)
754
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000755 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000756 return self.checkout.apply_patch(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000757 patches, post_processors or self.post_processors, verbose)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000758
759 def commit(self, message, user): # pylint: disable=R0201
760 logging.info('Would have committed for %s with message: %s' % (
761 user, message))
762 return 'FAKE'
763
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000764 def revisions(self, rev1, rev2):
765 return self.checkout.revisions(rev1, rev2)
766
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000767 @property
768 def project_name(self):
769 return self.checkout.project_name
770
771 @property
772 def project_path(self):
773 return self.checkout.project_path