blob: 49799d03df4ed29103fb56894c183b79802b97d9 [file] [log] [blame]
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00001# coding=utf8
maruel@chromium.org9799a072012-01-11 00:26:25 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Manages a project checkout.
6
7Includes support for svn, git-svn and git.
8"""
9
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000010import ConfigParser
11import fnmatch
12import logging
13import os
14import re
maruel@chromium.org5e975632011-09-29 18:07:06 +000015import shutil
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000016import subprocess
17import sys
18import tempfile
19
20import patch
21import scm
22import subprocess2
23
24
csharp@chromium.org9af0a112013-03-20 20:21:35 +000025if sys.platform in ('cygwin', 'win32'):
26 # Disable timeouts on Windows since we can't have shells with timeouts.
27 GLOBAL_TIMEOUT = None
28 FETCH_TIMEOUT = None
29else:
30 # Default timeout of 15 minutes.
31 GLOBAL_TIMEOUT = 15*60
32 # Use a larger timeout for checkout since it can be a genuinely slower
33 # operation.
34 FETCH_TIMEOUT = 30*60
35
36
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000037def get_code_review_setting(path, key,
38 codereview_settings_file='codereview.settings'):
39 """Parses codereview.settings and return the value for the key if present.
40
41 Don't cache the values in case the file is changed."""
42 # TODO(maruel): Do not duplicate code.
43 settings = {}
44 try:
45 settings_file = open(os.path.join(path, codereview_settings_file), 'r')
46 try:
47 for line in settings_file.readlines():
48 if not line or line.startswith('#'):
49 continue
50 if not ':' in line:
51 # Invalid file.
52 return None
53 k, v = line.split(':', 1)
54 settings[k.strip()] = v.strip()
55 finally:
56 settings_file.close()
maruel@chromium.org004fb712011-06-21 20:02:16 +000057 except IOError:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000058 return None
59 return settings.get(key, None)
60
61
maruel@chromium.org4dd9f722012-10-01 16:23:03 +000062def align_stdout(stdout):
63 """Returns the aligned output of multiple stdouts."""
64 output = ''
65 for item in stdout:
66 item = item.strip()
67 if not item:
68 continue
69 output += ''.join(' %s\n' % line for line in item.splitlines())
70 return output
71
72
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000073class PatchApplicationFailed(Exception):
74 """Patch failed to be applied."""
maruel@chromium.org34f68552012-05-09 19:18:36 +000075 def __init__(self, p, status):
76 super(PatchApplicationFailed, self).__init__(p, status)
77 self.patch = p
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000078 self.status = status
79
maruel@chromium.org34f68552012-05-09 19:18:36 +000080 @property
81 def filename(self):
82 if self.patch:
83 return self.patch.filename
84
85 def __str__(self):
86 out = []
87 if self.filename:
88 out.append('Failed to apply patch for %s:' % self.filename)
89 if self.status:
90 out.append(self.status)
maruel@chromium.orgcb5667a2012-10-23 19:42:10 +000091 if self.patch:
92 out.append('Patch: %s' % self.patch.dump())
maruel@chromium.org34f68552012-05-09 19:18:36 +000093 return '\n'.join(out)
94
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +000095
96class CheckoutBase(object):
97 # Set to None to have verbose output.
98 VOID = subprocess2.VOID
99
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000100 def __init__(self, root_dir, project_name, post_processors):
101 """
102 Args:
103 post_processor: list of lambda(checkout, patches) to call on each of the
104 modified files.
105 """
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000106 super(CheckoutBase, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000107 self.root_dir = root_dir
108 self.project_name = project_name
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000109 if self.project_name is None:
110 self.project_path = self.root_dir
111 else:
112 self.project_path = os.path.join(self.root_dir, self.project_name)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000113 # Only used for logging purposes.
114 self._last_seen_revision = None
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000115 self.post_processors = post_processors
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000116 assert self.root_dir
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000117 assert self.project_path
maruel@chromium.org0aca0f92012-10-01 16:39:45 +0000118 assert os.path.isabs(self.project_path)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000119
120 def get_settings(self, key):
121 return get_code_review_setting(self.project_path, key)
122
maruel@chromium.org51919772011-06-12 01:27:42 +0000123 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000124 """Checks out a clean copy of the tree and removes any local modification.
125
126 This function shouldn't throw unless the remote repository is inaccessible,
127 there is no free disk space or hard issues like that.
maruel@chromium.org51919772011-06-12 01:27:42 +0000128
129 Args:
130 revision: The revision it should sync to, SCM specific.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000131 """
132 raise NotImplementedError()
133
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000134 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000135 """Applies a patch and returns the list of modified files.
136
137 This function should throw patch.UnsupportedPatchFormat or
138 PatchApplicationFailed when relevant.
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000139
140 Args:
141 patches: patch.PatchSet object.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000142 """
143 raise NotImplementedError()
144
145 def commit(self, commit_message, user):
146 """Commits the patch upstream, while impersonating 'user'."""
147 raise NotImplementedError()
148
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000149 def revisions(self, rev1, rev2):
150 """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]).
151
152 If rev2 is None, it means 'HEAD'.
153
154 Returns None if there is no link between the two.
155 """
156 raise NotImplementedError()
157
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000158
159class RawCheckout(CheckoutBase):
160 """Used to apply a patch locally without any intent to commit it.
161
162 To be used by the try server.
163 """
maruel@chromium.org51919772011-06-12 01:27:42 +0000164 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000165 """Stubbed out."""
166 pass
167
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000168 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000169 """Ignores svn properties."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000170 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000171 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000172 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000173 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000174 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000175 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000176 os.remove(filepath)
177 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000178 else:
179 dirname = os.path.dirname(p.filename)
180 full_dir = os.path.join(self.project_path, dirname)
181 if dirname and not os.path.isdir(full_dir):
182 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000183 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000184
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000185 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000186 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000187 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000188 f.write(content)
189 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000190 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000191 if p.source_filename:
192 if not p.is_new:
193 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000194 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000195 'File has a source filename specified but is not new')
196 # Copy the file first.
197 if os.path.isfile(filepath):
198 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000199 p, 'File exist but was about to be overwriten')
maruel@chromium.org5e975632011-09-29 18:07:06 +0000200 shutil.copy2(
201 os.path.join(self.project_path, p.source_filename), filepath)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000202 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000203 if p.diff_hunks:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000204 cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel]
205 if verbose:
206 cmd.append('--verbose')
207 stdout.append(
208 subprocess2.check_output(
209 cmd,
210 stdin=p.get(False),
211 stderr=subprocess2.STDOUT,
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000212 cwd=self.project_path,
213 timeout=GLOBAL_TIMEOUT))
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000214 elif p.is_new and not os.path.exists(filepath):
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000215 # There is only a header. Just create the file.
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000216 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000217 stdout.append('Created an empty file.')
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000218 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000219 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000220 if verbose:
221 print p.filename
222 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000223 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000224 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000225 except subprocess.CalledProcessError, e:
226 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000227 p,
228 'While running %s;\n%s%s' % (
229 ' '.join(e.cmd),
230 align_stdout(stdout),
231 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000232
233 def commit(self, commit_message, user):
234 """Stubbed out."""
235 raise NotImplementedError('RawCheckout can\'t commit')
236
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000237 def revisions(self, _rev1, _rev2):
238 return None
239
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000240
241class SvnConfig(object):
242 """Parses a svn configuration file."""
243 def __init__(self, svn_config_dir=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000244 super(SvnConfig, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000245 self.svn_config_dir = svn_config_dir
246 self.default = not bool(self.svn_config_dir)
247 if not self.svn_config_dir:
248 if sys.platform == 'win32':
249 self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
250 else:
251 self.svn_config_dir = os.path.join(os.environ['HOME'], '.subversion')
252 svn_config_file = os.path.join(self.svn_config_dir, 'config')
253 parser = ConfigParser.SafeConfigParser()
254 if os.path.isfile(svn_config_file):
255 parser.read(svn_config_file)
256 else:
257 parser.add_section('auto-props')
258 self.auto_props = dict(parser.items('auto-props'))
259
260
261class SvnMixIn(object):
262 """MixIn class to add svn commands common to both svn and git-svn clients."""
263 # These members need to be set by the subclass.
264 commit_user = None
265 commit_pwd = None
266 svn_url = None
267 project_path = None
268 # Override at class level when necessary. If used, --non-interactive is
269 # implied.
270 svn_config = SvnConfig()
271 # Set to True when non-interactivity is necessary but a custom subversion
272 # configuration directory is not necessary.
273 non_interactive = False
274
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000275 def _add_svn_flags(self, args, non_interactive, credentials=True):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000276 args = ['svn'] + args
277 if not self.svn_config.default:
278 args.extend(['--config-dir', self.svn_config.svn_config_dir])
279 if not self.svn_config.default or self.non_interactive or non_interactive:
280 args.append('--non-interactive')
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000281 if credentials:
282 if self.commit_user:
283 args.extend(['--username', self.commit_user])
284 if self.commit_pwd:
285 args.extend(['--password', self.commit_pwd])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000286 return args
287
288 def _check_call_svn(self, args, **kwargs):
289 """Runs svn and throws an exception if the command failed."""
290 kwargs.setdefault('cwd', self.project_path)
291 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000292 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org0bcd1d32011-04-26 15:55:49 +0000293 return subprocess2.check_call_out(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000294 self._add_svn_flags(args, False), **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000295
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000296 def _check_output_svn(self, args, credentials=True, **kwargs):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000297 """Runs svn and throws an exception if the command failed.
298
299 Returns the output.
300 """
301 kwargs.setdefault('cwd', self.project_path)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000302 return subprocess2.check_output(
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000303 self._add_svn_flags(args, True, credentials),
304 stderr=subprocess2.STDOUT,
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000305 timeout=GLOBAL_TIMEOUT,
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000306 **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000307
308 @staticmethod
309 def _parse_svn_info(output, key):
310 """Returns value for key from svn info output.
311
312 Case insensitive.
313 """
314 values = {}
315 key = key.lower()
316 for line in output.splitlines(False):
317 if not line:
318 continue
319 k, v = line.split(':', 1)
320 k = k.strip().lower()
321 v = v.strip()
322 assert not k in values
323 values[k] = v
324 return values.get(key, None)
325
326
327class SvnCheckout(CheckoutBase, SvnMixIn):
328 """Manages a subversion checkout."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000329 def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
330 post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000331 CheckoutBase.__init__(self, root_dir, project_name, post_processors)
332 SvnMixIn.__init__(self)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000333 self.commit_user = commit_user
334 self.commit_pwd = commit_pwd
335 self.svn_url = svn_url
336 assert bool(self.commit_user) >= bool(self.commit_pwd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000337
maruel@chromium.org51919772011-06-12 01:27:42 +0000338 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000339 # Will checkout if the directory is not present.
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000340 assert self.svn_url
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000341 if not os.path.isdir(self.project_path):
342 logging.info('Checking out %s in %s' %
343 (self.project_name, self.project_path))
maruel@chromium.org51919772011-06-12 01:27:42 +0000344 return self._revert(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000345
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000346 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000347 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000348 for p in patches:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000349 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000350 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000351 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000352 # It is important to use credentials=False otherwise credentials could
353 # leak in the error message. Credentials are not necessary here for the
354 # following commands anyway.
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000355 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000356 stdout.append(self._check_output_svn(
357 ['delete', p.filename, '--force'], credentials=False))
358 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000359 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000360 # svn add while creating directories otherwise svn add on the
361 # contained files will silently fail.
362 # First, find the root directory that exists.
363 dirname = os.path.dirname(p.filename)
364 dirs_to_create = []
365 while (dirname and
366 not os.path.isdir(os.path.join(self.project_path, dirname))):
367 dirs_to_create.append(dirname)
368 dirname = os.path.dirname(dirname)
369 for dir_to_create in reversed(dirs_to_create):
370 os.mkdir(os.path.join(self.project_path, dir_to_create))
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000371 stdout.append(
372 self._check_output_svn(
373 ['add', dir_to_create, '--force'], credentials=False))
374 stdout.append('Created missing directory %s.' % dir_to_create)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000375
376 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000377 content = p.get()
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000378 with open(filepath, 'wb') as f:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000379 f.write(content)
380 stdout.append('Added binary file %d bytes.' % len(content))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000381 else:
maruel@chromium.org5e975632011-09-29 18:07:06 +0000382 if p.source_filename:
383 if not p.is_new:
384 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000385 p,
maruel@chromium.org5e975632011-09-29 18:07:06 +0000386 'File has a source filename specified but is not new')
387 # Copy the file first.
388 if os.path.isfile(filepath):
389 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000390 p, 'File exist but was about to be overwriten')
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000391 stdout.append(
392 self._check_output_svn(
393 ['copy', p.source_filename, p.filename]))
394 stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000395 if p.diff_hunks:
maruel@chromium.orgec4a9182012-09-28 20:39:45 +0000396 cmd = [
397 'patch',
398 '-p%s' % p.patchlevel,
399 '--forward',
400 '--force',
401 '--no-backup-if-mismatch',
402 ]
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000403 stdout.append(
404 subprocess2.check_output(
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000405 cmd,
406 stdin=p.get(False),
407 cwd=self.project_path,
408 timeout=GLOBAL_TIMEOUT))
maruel@chromium.org4869bcf2011-06-04 01:14:32 +0000409 elif p.is_new and not os.path.exists(filepath):
410 # There is only a header. Just create the file if it doesn't
411 # exist.
412 open(filepath, 'w').close()
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000413 stdout.append('Created an empty file.')
maruel@chromium.org3da83172012-05-07 16:17:20 +0000414 if p.is_new and not p.source_filename:
415 # Do not run it if p.source_filename is defined, since svn copy was
416 # using above.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000417 stdout.append(
418 self._check_output_svn(
419 ['add', p.filename, '--force'], credentials=False))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000420 for name, value in p.svn_properties:
421 if value is None:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000422 stdout.append(
423 self._check_output_svn(
424 ['propdel', '--quiet', name, p.filename],
425 credentials=False))
426 stdout.append('Property %s deleted.' % name)
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000427 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000428 stdout.append(
429 self._check_output_svn(
430 ['propset', name, value, p.filename], credentials=False))
431 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000432 for prop, values in self.svn_config.auto_props.iteritems():
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000433 if fnmatch.fnmatch(p.filename, prop):
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000434 for value in values.split(';'):
435 if '=' not in value:
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000436 params = [value, '.']
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000437 else:
438 params = value.split('=', 1)
maruel@chromium.orge1a03762012-09-24 15:28:52 +0000439 if params[1] == '*':
440 # Works around crbug.com/150960 on Windows.
441 params[1] = '.'
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000442 stdout.append(
443 self._check_output_svn(
444 ['propset'] + params + [p.filename], credentials=False))
445 stdout.append('Property (auto) %s' % '='.join(params))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000446 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000447 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000448 if verbose:
449 print p.filename
450 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000451 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000452 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000453 except subprocess.CalledProcessError, e:
454 raise PatchApplicationFailed(
maruel@chromium.org34f68552012-05-09 19:18:36 +0000455 p,
maruel@chromium.org9842a0c2011-05-30 20:41:54 +0000456 'While running %s;\n%s%s' % (
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000457 ' '.join(e.cmd),
458 align_stdout(stdout),
459 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000460
461 def commit(self, commit_message, user):
462 logging.info('Committing patch for %s' % user)
463 assert self.commit_user
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000464 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000465 handle, commit_filename = tempfile.mkstemp(text=True)
466 try:
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000467 # Shouldn't assume default encoding is UTF-8. But really, if you are using
468 # anything else, you are living in another world.
469 os.write(handle, commit_message.encode('utf-8'))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000470 os.close(handle)
471 # When committing, svn won't update the Revision metadata of the checkout,
472 # so if svn commit returns "Committed revision 3.", svn info will still
473 # return "Revision: 2". Since running svn update right after svn commit
474 # creates a race condition with other committers, this code _must_ parse
475 # the output of svn commit and use a regexp to grab the revision number.
476 # Note that "Committed revision N." is localized but subprocess2 forces
477 # LANGUAGE=en.
478 args = ['commit', '--file', commit_filename]
479 # realauthor is parsed by a server-side hook.
480 if user and user != self.commit_user:
481 args.extend(['--with-revprop', 'realauthor=%s' % user])
482 out = self._check_output_svn(args)
483 finally:
484 os.remove(commit_filename)
485 lines = filter(None, out.splitlines())
486 match = re.match(r'^Committed revision (\d+).$', lines[-1])
487 if not match:
488 raise PatchApplicationFailed(
489 None,
490 'Couldn\'t make sense out of svn commit message:\n' + out)
491 return int(match.group(1))
492
maruel@chromium.org51919772011-06-12 01:27:42 +0000493 def _revert(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000494 """Reverts local modifications or checks out if the directory is not
495 present. Use depot_tools's functionality to do this.
496 """
497 flags = ['--ignore-externals']
maruel@chromium.org51919772011-06-12 01:27:42 +0000498 if revision:
499 flags.extend(['--revision', str(revision)])
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000500 if os.path.isdir(self.project_path):
501 # This may remove any part (or all) of the checkout.
502 scm.SVN.Revert(self.project_path, no_ignore=True)
503
504 if os.path.isdir(self.project_path):
505 # Revive files that were deleted in scm.SVN.Revert().
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000506 self._check_call_svn(['update', '--force'] + flags,
507 timeout=FETCH_TIMEOUT)
maruel@chromium.org6e904b42012-12-19 14:21:14 +0000508 else:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000509 logging.info(
510 'Directory %s is not present, checking it out.' % self.project_path)
511 self._check_call_svn(
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000512 ['checkout', self.svn_url, self.project_path] + flags, cwd=None,
513 timeout=FETCH_TIMEOUT)
maruel@chromium.org51919772011-06-12 01:27:42 +0000514 return self._get_revision()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000515
maruel@chromium.org51919772011-06-12 01:27:42 +0000516 def _get_revision(self):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000517 out = self._check_output_svn(['info', '.'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000518 revision = int(self._parse_svn_info(out, 'revision'))
519 if revision != self._last_seen_revision:
520 logging.info('Updated to revision %d' % revision)
521 self._last_seen_revision = revision
522 return revision
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000523
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000524 def revisions(self, rev1, rev2):
525 """Returns the number of actual commits, not just the difference between
526 numbers.
527 """
528 rev2 = rev2 or 'HEAD'
529 # Revision range is inclusive and ordering doesn't matter, they'll appear in
530 # the order specified.
531 try:
532 out = self._check_output_svn(
533 ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)])
534 except subprocess.CalledProcessError:
535 return None
536 # Ignore the '----' lines.
537 return len([l for l in out.splitlines() if l.startswith('r')]) - 1
538
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000539
540class GitCheckoutBase(CheckoutBase):
541 """Base class for git checkout. Not to be used as-is."""
maruel@chromium.org6ed8b502011-06-12 01:05:35 +0000542 def __init__(self, root_dir, project_name, remote_branch,
543 post_processors=None):
544 super(GitCheckoutBase, self).__init__(
545 root_dir, project_name, post_processors)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000546 # There is no reason to not hardcode it.
547 self.remote = 'origin'
548 self.remote_branch = remote_branch
549 self.working_branch = 'working_branch'
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000550
maruel@chromium.org51919772011-06-12 01:27:42 +0000551 def prepare(self, revision):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000552 """Resets the git repository in a clean state.
553
554 Checks it out if not present and deletes the working branch.
555 """
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000556 assert self.remote_branch
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000557 assert os.path.isdir(self.project_path)
558 self._check_call_git(['reset', '--hard', '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000559 if revision:
560 try:
561 revision = self._check_output_git(['rev-parse', revision])
562 except subprocess.CalledProcessError:
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000563 self._check_call_git(
564 ['fetch', self.remote, self.remote_branch, '--quiet'])
maruel@chromium.org51919772011-06-12 01:27:42 +0000565 revision = self._check_output_git(['rev-parse', revision])
566 self._check_call_git(['checkout', '--force', '--quiet', revision])
567 else:
568 branches, active = self._branches()
569 if active != 'master':
570 self._check_call_git(['checkout', '--force', '--quiet', 'master'])
571 self._check_call_git(['pull', self.remote, self.remote_branch, '--quiet'])
572 if self.working_branch in branches:
573 self._call_git(['branch', '-D', self.working_branch])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000574
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000575 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000576 """Applies a patch on 'working_branch' and switch to it.
577
578 Also commits the changes on the local branch.
579
580 Ignores svn properties and raise an exception on unexpected ones.
581 """
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000582 post_processors = post_processors or self.post_processors or []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000583 # It this throws, the checkout is corrupted. Maybe worth deleting it and
584 # trying again?
maruel@chromium.org3cdb7f32011-05-05 16:37:24 +0000585 if self.remote_branch:
586 self._check_call_git(
587 ['checkout', '-b', self.working_branch,
588 '%s/%s' % (self.remote, self.remote_branch), '--quiet'])
maruel@chromium.org5e975632011-09-29 18:07:06 +0000589 for index, p in enumerate(patches):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000590 stdout = []
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000591 try:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000592 filepath = os.path.join(self.project_path, p.filename)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000593 if p.is_delete:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000594 if (not os.path.exists(filepath) and
maruel@chromium.org5e975632011-09-29 18:07:06 +0000595 any(p1.source_filename == p.filename for p1 in patches[0:index])):
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000596 # The file was already deleted if a prior patch with file rename
597 # was already processed because 'git apply' did it for us.
maruel@chromium.org5e975632011-09-29 18:07:06 +0000598 pass
599 else:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000600 stdout.append(self._check_output_git(['rm', p.filename]))
601 stdout.append('Deleted.')
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000602 else:
603 dirname = os.path.dirname(p.filename)
604 full_dir = os.path.join(self.project_path, dirname)
605 if dirname and not os.path.isdir(full_dir):
606 os.makedirs(full_dir)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000607 stdout.append('Created missing directory %s.' % dirname)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000608 if p.is_binary:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000609 content = p.get()
610 with open(filepath, 'wb') as f:
611 f.write(content)
612 stdout.append('Added binary file %d bytes' % len(content))
613 cmd = ['add', p.filename]
614 if verbose:
615 cmd.append('--verbose')
616 stdout.append(self._check_output_git(cmd))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000617 else:
maruel@chromium.org58fe6622011-06-03 20:59:27 +0000618 # No need to do anything special with p.is_new or if not
619 # p.diff_hunks. git apply manages all that already.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000620 cmd = ['apply', '--index', '-p%s' % p.patchlevel]
621 if verbose:
622 cmd.append('--verbose')
623 stdout.append(self._check_output_git(cmd, stdin=p.get(True)))
624 for name, value in p.svn_properties:
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000625 # Ignore some known auto-props flags through .subversion/config,
626 # bails out on the other ones.
627 # TODO(maruel): Read ~/.subversion/config and detect the rules that
628 # applies here to figure out if the property will be correctly
629 # handled.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000630 stdout.append('Property %s=%s' % (name, value))
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000631 if not name in (
maruel@chromium.org9799a072012-01-11 00:26:25 +0000632 'svn:eol-style', 'svn:executable', 'svn:mime-type'):
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000633 raise patch.UnsupportedPatchFormat(
634 p.filename,
635 'Cannot apply svn property %s to file %s.' % (
maruel@chromium.orgd7ca6162012-08-29 17:22:22 +0000636 name, p.filename))
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000637 for post in post_processors:
maruel@chromium.org8a1396c2011-04-22 00:14:24 +0000638 post(self, p)
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000639 if verbose:
640 print p.filename
641 print align_stdout(stdout)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000642 except OSError, e:
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000643 raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000644 except subprocess.CalledProcessError, e:
645 raise PatchApplicationFailed(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000646 p,
647 'While running %s;\n%s%s' % (
648 ' '.join(e.cmd),
649 align_stdout(stdout),
650 align_stdout([getattr(e, 'stdout', '')])))
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000651 # Once all the patches are processed and added to the index, commit the
652 # index.
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000653 cmd = ['commit', '-m', 'Committed patch']
654 if verbose:
655 cmd.append('--verbose')
656 self._check_call_git(cmd)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000657 # TODO(maruel): Weirdly enough they don't match, need to investigate.
658 #found_files = self._check_output_git(
659 # ['diff', 'master', '--name-only']).splitlines(False)
660 #assert sorted(patches.filenames) == sorted(found_files), (
661 # sorted(out), sorted(found_files))
662
663 def commit(self, commit_message, user):
664 """Updates the commit message.
665
666 Subclass needs to dcommit or push.
667 """
maruel@chromium.org1bf50972011-05-05 19:57:21 +0000668 assert isinstance(commit_message, unicode)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000669 self._check_call_git(['commit', '--amend', '-m', commit_message])
670 return self._check_output_git(['rev-parse', 'HEAD']).strip()
671
672 def _check_call_git(self, args, **kwargs):
673 kwargs.setdefault('cwd', self.project_path)
674 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000675 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000676 return subprocess2.check_call_out(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000677
678 def _call_git(self, args, **kwargs):
679 """Like check_call but doesn't throw on failure."""
680 kwargs.setdefault('cwd', self.project_path)
681 kwargs.setdefault('stdout', self.VOID)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000682 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000683 return subprocess2.call(['git'] + args, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000684
685 def _check_output_git(self, args, **kwargs):
686 kwargs.setdefault('cwd', self.project_path)
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000687 kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
maruel@chromium.org87e6d332011-09-09 19:01:28 +0000688 return subprocess2.check_output(
maruel@chromium.org44b21b92012-11-08 19:37:08 +0000689 ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000690
691 def _branches(self):
692 """Returns the list of branches and the active one."""
693 out = self._check_output_git(['branch']).splitlines(False)
694 branches = [l[2:] for l in out]
695 active = None
696 for l in out:
697 if l.startswith('*'):
698 active = l[2:]
699 break
700 return branches, active
701
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000702 def revisions(self, rev1, rev2):
703 """Returns the number of actual commits between both hash."""
704 self._fetch_remote()
705
706 rev2 = rev2 or '%s/%s' % (self.remote, self.remote_branch)
707 # Revision range is ]rev1, rev2] and ordering matters.
708 try:
709 out = self._check_output_git(
710 ['log', '--format="%H"' , '%s..%s' % (rev1, rev2)])
711 except subprocess.CalledProcessError:
712 return None
713 return len(out.splitlines())
714
715 def _fetch_remote(self):
716 """Fetches the remote without rebasing."""
717 raise NotImplementedError()
718
719
720class GitCheckout(GitCheckoutBase):
721 """Git checkout implementation."""
722 def _fetch_remote(self):
723 # git fetch is always verbose even with -q -q so redirect its output.
csharp@chromium.org9af0a112013-03-20 20:21:35 +0000724 self._check_output_git(['fetch', self.remote, self.remote_branch],
725 timeout=FETCH_TIMEOUT)
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000726
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000727
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000728class ReadOnlyCheckout(object):
729 """Converts a checkout into a read-only one."""
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000730 def __init__(self, checkout, post_processors=None):
maruel@chromium.orga5129fb2011-06-20 18:36:25 +0000731 super(ReadOnlyCheckout, self).__init__()
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000732 self.checkout = checkout
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000733 self.post_processors = (post_processors or []) + (
734 self.checkout.post_processors or [])
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000735
maruel@chromium.org51919772011-06-12 01:27:42 +0000736 def prepare(self, revision):
737 return self.checkout.prepare(revision)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000738
739 def get_settings(self, key):
740 return self.checkout.get_settings(key)
741
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000742 def apply_patch(self, patches, post_processors=None, verbose=False):
maruel@chromium.orgb1d1a782011-09-29 14:13:55 +0000743 return self.checkout.apply_patch(
maruel@chromium.org4dd9f722012-10-01 16:23:03 +0000744 patches, post_processors or self.post_processors, verbose)
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000745
746 def commit(self, message, user): # pylint: disable=R0201
747 logging.info('Would have committed for %s with message: %s' % (
748 user, message))
749 return 'FAKE'
750
maruel@chromium.orgbc32ad12012-07-26 13:22:47 +0000751 def revisions(self, rev1, rev2):
752 return self.checkout.revisions(rev1, rev2)
753
maruel@chromium.orgdfaecd22011-04-21 00:33:31 +0000754 @property
755 def project_name(self):
756 return self.checkout.project_name
757
758 @property
759 def project_path(self):
760 return self.checkout.project_path