blob: e58b8bdd949c3a2328f82ca133258d7f2b52393c [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.orgb7d46902009-06-10 14:12:10 +00009__version__ = '1.3.2'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
20import marshal # Exposed through the API.
21import optparse
22import os # Somewhat exposed through the API.
23import pickle # Exposed through the API.
24import re # Exposed through the API.
25import subprocess # Exposed through the API.
26import sys # Parts exposed through API.
27import tempfile # Exposed through the API.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000028import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000029import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000030import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000031import urllib2 # Exposed through the API.
maruel@chromium.org1e08c002009-05-28 19:09:33 +000032import warnings
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033
34# Local imports.
35# TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but
36# for now it would only be a couple of functions so hardly worth it.
37import gcl
maruel@chromium.org46a94102009-05-12 20:32:43 +000038import gclient
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000039import presubmit_canned_checks
40
41
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000042class NotImplementedException(Exception):
43 """We're leaving placeholders in a bunch of places to remind us of the
44 design of the API, but we have not implemented all of it yet. Implement as
45 the need arises.
46 """
47 pass
48
49
50def normpath(path):
51 '''Version of os.path.normpath that also changes backward slashes to
52 forward slashes when not running on Windows.
53 '''
54 # This is safe to always do because the Windows version of os.path.normpath
55 # will replace forward slashes with backward slashes.
56 path = path.replace(os.sep, '/')
57 return os.path.normpath(path)
58
59
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000060class OutputApi(object):
61 """This class (more like a module) gets passed to presubmit scripts so that
62 they can specify various types of results.
63 """
64
65 class PresubmitResult(object):
66 """Base class for result objects."""
67
68 def __init__(self, message, items=None, long_text=''):
69 """
70 message: A short one-line message to indicate errors.
71 items: A list of short strings to indicate where errors occurred.
72 long_text: multi-line text output, e.g. from another tool
73 """
74 self._message = message
75 self._items = []
76 if items:
77 self._items = items
78 self._long_text = long_text.rstrip()
79
80 def _Handle(self, output_stream, input_stream, may_prompt=True):
81 """Writes this result to the output stream.
82
83 Args:
84 output_stream: Where to write
85
86 Returns:
87 True if execution may continue, False otherwise.
88 """
89 output_stream.write(self._message)
90 output_stream.write('\n')
91 for item in self._items:
92 output_stream.write(' %s\n' % item)
93 if self._long_text:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +000094 output_stream.write('\n***************\n%s\n***************\n' %
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000095 self._long_text)
96
97 if self.ShouldPrompt() and may_prompt:
98 output_stream.write('Are you sure you want to continue? (y/N): ')
99 response = input_stream.readline()
100 if response.strip().lower() != 'y':
101 return False
102
103 return not self.IsFatal()
104
105 def IsFatal(self):
106 """An error that is fatal stops g4 mail/submit immediately, i.e. before
107 other presubmit scripts are run.
108 """
109 return False
110
111 def ShouldPrompt(self):
112 """Whether this presubmit result should result in a prompt warning."""
113 return False
114
115 class PresubmitError(PresubmitResult):
116 """A hard presubmit error."""
117 def IsFatal(self):
118 return True
119
120 class PresubmitPromptWarning(PresubmitResult):
121 """An warning that prompts the user if they want to continue."""
122 def ShouldPrompt(self):
123 return True
124
125 class PresubmitNotifyResult(PresubmitResult):
126 """Just print something to the screen -- but it's not even a warning."""
127 pass
128
129 class MailTextResult(PresubmitResult):
130 """A warning that should be included in the review request email."""
131 def __init__(self, *args, **kwargs):
132 raise NotImplementedException() # TODO(joi) Implement.
133
134
135class InputApi(object):
136 """An instance of this object is passed to presubmit scripts so they can
137 know stuff about the change they're looking at.
138 """
139
maruel@chromium.org3410d912009-06-09 20:56:16 +0000140 # File extensions that are considered source files from a style guide
141 # perspective. Don't modify this list from a presubmit script!
142 DEFAULT_WHITE_LIST = (
143 # C++ and friends
144 r".*\.c", r".*\.cc", r".*\.cpp", r".*\.h", r".*\.m", r".*\.mm",
145 r".*\.inl", r".*\.asm", r".*\.hxx", r".*\.hpp",
146 # Scripts
147 r".*\.js", r".*\.py", r".*\.json", r".*\.sh", r".*\.rb",
148 # No extension at all
149 r"(^|.*[\\\/])[^.]+$",
150 # Other
151 r".*\.java", r".*\.mk", r".*\.am",
152 )
153
154 # Path regexp that should be excluded from being considered containing source
155 # files. Don't modify this list from a presubmit script!
156 DEFAULT_BLACK_LIST = (
157 r".*\bexperimental[\\\/].*",
158 r".*\bthird_party[\\\/].*",
159 # Output directories (just in case)
160 r".*\bDebug[\\\/].*",
161 r".*\bRelease[\\\/].*",
162 r".*\bxcodebuild[\\\/].*",
163 r".*\bsconsbuild[\\\/].*",
164 # All caps files like README and LICENCE.
165 r".*\b[A-Z0-9_]+",
166 # SCM (can happen in dual SCM configuration)
167 r".*\b\.git[\\\/].*",
168 r".*\b\.svn[\\\/].*",
169 )
170
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000171 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000172 """Builds an InputApi object.
173
174 Args:
175 change: A presubmit.GclChange object.
176 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000177 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000178 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000179 # Version number of the presubmit_support script.
180 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000181 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000182 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000183
184 # We expose various modules and functions as attributes of the input_api
185 # so that presubmit scripts don't have to import them.
186 self.basename = os.path.basename
187 self.cPickle = cPickle
188 self.cStringIO = cStringIO
189 self.os_path = os.path
190 self.pickle = pickle
191 self.marshal = marshal
192 self.re = re
193 self.subprocess = subprocess
194 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000195 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000196 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000197 self.urllib2 = urllib2
198
199 # InputApi.platform is the platform you're currently running on.
200 self.platform = sys.platform
201
202 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000203 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000204
205 # We carry the canned checks so presubmit scripts can easily use them.
206 self.canned_checks = presubmit_canned_checks
207
208 def PresubmitLocalPath(self):
209 """Returns the local path of the presubmit script currently being run.
210
211 This is useful if you don't want to hard-code absolute paths in the
212 presubmit script. For example, It can be used to find another file
213 relative to the PRESUBMIT.py script, so the whole tree can be branched and
214 the presubmit script still works, without editing its content.
215 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000216 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000217
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000218 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000219 """Translate a depot path to a local path (relative to client root).
220
221 Args:
222 Depot path as a string.
223
224 Returns:
225 The local path of the depot path under the user's current client, or None
226 if the file is not mapped.
227
228 Remember to check for the None case and show an appropriate error!
229 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000230 local_path = gclient.CaptureSVNInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000231 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000232 return local_path
233
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000234 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000235 """Translate a local path to a depot path.
236
237 Args:
238 Local path (relative to current directory, or absolute) as a string.
239
240 Returns:
241 The depot path (SVN URL) of the file if mapped, otherwise None.
242 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000243 depot_path = gclient.CaptureSVNInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000244 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000245 return depot_path
246
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000247 def AffectedFiles(self, include_dirs=False, include_deletes=True):
248 """Same as input_api.change.AffectedFiles() except only lists files
249 (and optionally directories) in the same directory as the current presubmit
250 script, or subdirectories thereof.
251 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000252 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000253 if len(dir_with_slash) == 1:
254 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000255 return filter(
256 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
257 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000258
259 def LocalPaths(self, include_dirs=False):
260 """Returns local paths of input_api.AffectedFiles()."""
261 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
262
263 def AbsoluteLocalPaths(self, include_dirs=False):
264 """Returns absolute local paths of input_api.AffectedFiles()."""
265 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
266
267 def ServerPaths(self, include_dirs=False):
268 """Returns server paths of input_api.AffectedFiles()."""
269 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
270
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000271 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000272 """Same as input_api.change.AffectedTextFiles() except only lists files
273 in the same directory as the current presubmit script, or subdirectories
274 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000275 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000276 if include_deletes is not None:
277 warnings.warn("AffectedTextFiles(include_deletes=%s)"
278 " is deprecated and ignored" % str(include_deletes),
279 category=DeprecationWarning,
280 stacklevel=2)
281 return filter(lambda x: x.IsTextFile(),
282 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000283
maruel@chromium.org3410d912009-06-09 20:56:16 +0000284 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
285 """Filters out files that aren't considered "source file".
286
287 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
288 and InputApi.DEFAULT_BLACK_LIST is used respectively.
289
290 The lists will be compiled as regular expression and
291 AffectedFile.LocalPath() needs to pass both list.
292
293 Note: Copy-paste this function to suit your needs or use a lambda function.
294 """
295 def Find(affected_file, list):
296 for item in list:
297 if self.re.match(item, affected_file.LocalPath()):
298 return True
299 return False
300 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
301 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
302
303 def AffectedSourceFiles(self, source_file):
304 """Filter the list of AffectedTextFiles by the function source_file.
305
306 If source_file is None, InputApi.FilterSourceFile() is used.
307 """
308 if not source_file:
309 source_file = self.FilterSourceFile
310 return filter(source_file, self.AffectedTextFiles())
311
312 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000313 """An iterator over all text lines in "new" version of changed files.
314
315 Only lists lines from new or modified text files in the change that are
316 contained by the directory of the currently executing presubmit script.
317
318 This is useful for doing line-by-line regex checks, like checking for
319 trailing whitespace.
320
321 Yields:
322 a 3 tuple:
323 the AffectedFile instance of the current file;
324 integer line number (1-based); and
325 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000326
327 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000328 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000329 files = self.AffectedSourceFiles(source_file_filter)
330 return InputApi._RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000331
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000332 def ReadFile(self, file, mode='r'):
333 """Reads an arbitrary file.
334
335 Deny reading anything outside the repository.
336 """
337 if isinstance(file, AffectedFile):
338 file = file.AbsoluteLocalPath()
339 if not file.startswith(self.change.RepositoryRoot()):
340 raise IOError('Access outside the repository root is denied.')
341 return gcl.ReadFile(file, mode)
342
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000343 @staticmethod
344 def _RightHandSideLinesImpl(affected_files):
345 """Implements RightHandSideLines for InputApi and GclChange."""
346 for af in affected_files:
347 lines = af.NewContents()
348 line_number = 0
349 for line in lines:
350 line_number += 1
351 yield (af, line_number, line)
352
353
354class AffectedFile(object):
355 """Representation of a file in a change."""
356
357 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000358 self._path = path
359 self._action = action
360 self._repository_root = repository_root
361 self._is_directory = None
362 self._properties = {}
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000363 self.scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000364
365 def ServerPath(self):
366 """Returns a path string that identifies the file in the SCM system.
367
368 Returns the empty string if the file does not exist in SCM.
369 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000370 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000371
372 def LocalPath(self):
373 """Returns the path of this file on the local disk relative to client root.
374 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000375 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000376
377 def AbsoluteLocalPath(self):
378 """Returns the absolute path of this file on the local disk.
379 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000380 return normpath(os.path.join(self._repository_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000381
382 def IsDirectory(self):
383 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000384 if self._is_directory is None:
385 path = self.AbsoluteLocalPath()
386 self._is_directory = (os.path.exists(path) and
387 os.path.isdir(path))
388 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000389
390 def Action(self):
391 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000392 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
393 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000394 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000395
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000396 def Property(self, property_name):
397 """Returns the specified SCM property of this file, or None if no such
398 property.
399 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000400 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000401
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000402 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000403 """Returns True if the file is a text file and not a binary file.
404
405 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000406 raise NotImplementedError() # Implement when needed
407
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000408 def NewContents(self):
409 """Returns an iterator over the lines in the new version of file.
410
411 The new version is the file in the user's workspace, i.e. the "right hand
412 side".
413
414 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000415 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000416 """
417 if self.IsDirectory():
418 return []
419 else:
420 return gcl.ReadFile(self.AbsoluteLocalPath()).splitlines()
421
422 def OldContents(self):
423 """Returns an iterator over the lines in the old version of file.
424
425 The old version is the file in depot, i.e. the "left hand side".
426 """
427 raise NotImplementedError() # Implement when needed
428
429 def OldFileTempPath(self):
430 """Returns the path on local disk where the old contents resides.
431
432 The old version is the file in depot, i.e. the "left hand side".
433 This is a read-only cached copy of the old contents. *DO NOT* try to
434 modify this file.
435 """
436 raise NotImplementedError() # Implement if/when needed.
437
438
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000439class SvnAffectedFile(AffectedFile):
440 """Representation of a file in a change out of a Subversion checkout."""
441
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000442 def __init__(self, *args, **kwargs):
443 AffectedFile.__init__(self, *args, **kwargs)
444 self._server_path = None
445 self._is_text_file = None
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000446 self.scm = 'svn'
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000447
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000448 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000449 if self._server_path is None:
450 self._server_path = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000451 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000452 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000453
454 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000455 if self._is_directory is None:
456 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000457 if os.path.exists(path):
458 # Retrieve directly from the file system; it is much faster than
459 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000460 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000461 else:
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000462 self._is_directory = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000463 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000464 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000465
466 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000467 if not property_name in self._properties:
468 self._properties[property_name] = gcl.GetSVNFileProperty(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000469 self.AbsoluteLocalPath(), property_name)
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000470 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000471
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000472 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000473 if self._is_text_file is None:
474 if self.Action() == 'D':
475 # A deleted file is not a text file.
476 self._is_text_file = False
477 elif self.IsDirectory():
478 self._is_text_file = False
479 else:
480 mime_type = gcl.GetSVNFileProperty(self.AbsoluteLocalPath(),
481 'svn:mime-type')
482 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
483 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000484
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000485
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000486class GclChange(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000487 """Describe a change.
488
489 Used directly by the presubmit scripts to query the current change being
490 tested.
491
492 Instance members:
493 tags: Dictionnary of KEY=VALUE pairs found in the change description.
494 self.KEY: equivalent to tags['KEY']
495 """
496
497 # Matches key/value (or "tag") lines in changelist descriptions.
498 _tag_line_re = re.compile(
499 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000500
501 def __init__(self, change_info, repository_root=''):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000502 # Do not keep a reference to the original change_info.
503 self._name = change_info.name
504 self._full_description = change_info.description
505 self._repository_root = repository_root
maruel@chromium.org32ba2602009-06-06 18:44:48 +0000506 self.issue = change_info.issue
507 self.patchset = change_info.patchset
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000508
509 # From the description text, build up a dictionary of key/value pairs
510 # plus the description minus all key/value or "tag" lines.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000511 self._description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000512 self.tags = {}
513 for line in change_info.description.splitlines():
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000514 m = self._tag_line_re.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000515 if m:
516 self.tags[m.group('key')] = m.group('value')
517 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000518 self._description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000519
520 # Change back to text and remove whitespace at end.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000521 self._description_without_tags = '\n'.join(self._description_without_tags)
522 self._description_without_tags = self._description_without_tags.rstrip()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000523
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000524 self._affected_files = [
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000525 SvnAffectedFile(info[1], info[0].strip(), repository_root)
526 for info in change_info.files
527 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000528
529 def Change(self):
530 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000531 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000532
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000533 def DescriptionText(self):
534 """Returns the user-entered changelist description, minus tags.
535
536 Any line in the user-provided description starting with e.g. "FOO="
537 (whitespace permitted before and around) is considered a tag line. Such
538 lines are stripped out of the description this function returns.
539 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000540 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000541
542 def FullDescriptionText(self):
543 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000544 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000545
546 def RepositoryRoot(self):
547 """Returns the repository root for this change, as an absolute path."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000548 return self._repository_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000549
550 def __getattr__(self, attr):
551 """Return keys directly as attributes on the object.
552
553 You may use a friendly name (from SPECIAL_KEYS) or the actual name of
554 the key.
555 """
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000556 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000557
558 def AffectedFiles(self, include_dirs=False, include_deletes=True):
559 """Returns a list of AffectedFile instances for all files in the change.
560
561 Args:
562 include_deletes: If false, deleted files will be filtered out.
563 include_dirs: True to include directories in the list
564
565 Returns:
566 [AffectedFile(path, action), AffectedFile(path, action)]
567 """
568 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000569 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000570 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000571 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000572
573 if include_deletes:
574 return affected
575 else:
576 return filter(lambda x: x.Action() != 'D', affected)
577
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000578 def AffectedTextFiles(self, include_deletes=None):
579 """Return a list of the existing text files in a change."""
580 if include_deletes is not None:
581 warnings.warn("AffectedTextFiles(include_deletes=%s)"
582 " is deprecated and ignored" % str(include_deletes),
583 category=DeprecationWarning,
584 stacklevel=2)
585 return filter(lambda x: x.IsTextFile(),
586 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000587
588 def LocalPaths(self, include_dirs=False):
589 """Convenience function."""
590 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
591
592 def AbsoluteLocalPaths(self, include_dirs=False):
593 """Convenience function."""
594 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
595
596 def ServerPaths(self, include_dirs=False):
597 """Convenience function."""
598 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
599
600 def RightHandSideLines(self):
601 """An iterator over all text lines in "new" version of changed files.
602
603 Lists lines from new or modified text files in the change.
604
605 This is useful for doing line-by-line regex checks, like checking for
606 trailing whitespace.
607
608 Yields:
609 a 3 tuple:
610 the AffectedFile instance of the current file;
611 integer line number (1-based); and
612 the contents of the line as a string.
613 """
614 return InputApi._RightHandSideLinesImpl(
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000615 filter(lambda x: x.IsTextFile(),
616 self.AffectedFiles(include_deletes=False)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000617
618
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000619def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000620 """Finds all presubmit files that apply to a given set of source files.
621
622 Args:
623 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000624 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000625
626 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000627 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000628 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000629 entries = []
630 for f in files:
631 f = normpath(os.path.join(root, f))
632 while f:
633 f = os.path.dirname(f)
634 if f in entries:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000635 break
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000636 entries.append(f)
637 if f == root:
638 break
639 entries.sort()
640 entries = map(lambda x: os.path.join(x, 'PRESUBMIT.py'), entries)
641 return filter(lambda x: os.path.isfile(x), entries)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000642
643
644class PresubmitExecuter(object):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000645 def __init__(self, change_info, committing):
646 """
647 Args:
648 change_info: The ChangeInfo object for the change.
649 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
650 """
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000651 # TODO(maruel): Determine the SCM.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000652 self.change = GclChange(change_info, gcl.GetRepositoryRoot())
653 self.committing = committing
654
655 def ExecPresubmitScript(self, script_text, presubmit_path):
656 """Executes a single presubmit script.
657
658 Args:
659 script_text: The text of the presubmit script.
660 presubmit_path: The path to the presubmit file (this will be reported via
661 input_api.PresubmitLocalPath()).
662
663 Return:
664 A list of result objects, empty if no problems.
665 """
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000666 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000667 context = {}
668 exec script_text in context
669
670 # These function names must change if we make substantial changes to
671 # the presubmit API that are not backwards compatible.
672 if self.committing:
673 function_name = 'CheckChangeOnCommit'
674 else:
675 function_name = 'CheckChangeOnUpload'
676 if function_name in context:
677 context['__args'] = (input_api, OutputApi())
678 result = eval(function_name + '(*__args)', context)
679 if not (isinstance(result, types.TupleType) or
680 isinstance(result, types.ListType)):
681 raise exceptions.RuntimeError(
682 'Presubmit functions must return a tuple or list')
683 for item in result:
684 if not isinstance(item, OutputApi.PresubmitResult):
685 raise exceptions.RuntimeError(
686 'All presubmit results must be of types derived from '
687 'output_api.PresubmitResult')
688 else:
689 result = () # no error since the script doesn't care about current event.
690
691 return result
692
693
694def DoPresubmitChecks(change_info,
695 committing,
696 verbose,
697 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000698 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000699 default_presubmit,
700 may_prompt):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000701 """Runs all presubmit checks that apply to the files in the change.
702
703 This finds all PRESUBMIT.py files in directories enclosing the files in the
704 change (up to the repository root) and calls the relevant entrypoint function
705 depending on whether the change is being committed or uploaded.
706
707 Prints errors, warnings and notifications. Prompts the user for warnings
708 when needed.
709
710 Args:
711 change_info: The ChangeInfo object for the change.
712 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
713 verbose: Prints debug info.
714 output_stream: A stream to write output from presubmit tests to.
715 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000716 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000717 may_prompt: Enable (y/n) questions on warning or error.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000718
719 Return:
720 True if execution can continue, False if not.
721 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000722 checkout_root = gcl.GetRepositoryRoot()
723 presubmit_files = ListRelevantPresubmitFiles(change_info.FileList(),
724 checkout_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000725 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000726 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000727 results = []
728 executer = PresubmitExecuter(change_info, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000729 if default_presubmit:
730 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000731 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000732 fake_path = os.path.join(checkout_root, 'PRESUBMIT.py')
733 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000734 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000735 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000736 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000737 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000738 # Accept CRLF presubmit script.
maruel@chromium.org277003e2009-05-01 12:51:43 +0000739 presubmit_script = gcl.ReadFile(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000740 results += executer.ExecPresubmitScript(presubmit_script, filename)
741
742 errors = []
743 notifications = []
744 warnings = []
745 for result in results:
746 if not result.IsFatal() and not result.ShouldPrompt():
747 notifications.append(result)
748 elif result.ShouldPrompt():
749 warnings.append(result)
750 else:
751 errors.append(result)
752
753 error_count = 0
754 for name, items in (('Messages', notifications),
755 ('Warnings', warnings),
756 ('ERRORS', errors)):
757 if items:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000758 output_stream.write('** Presubmit %s **\n' % name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000759 for item in items:
760 if not item._Handle(output_stream, input_stream,
761 may_prompt=False):
762 error_count += 1
763 output_stream.write('\n')
764 if not errors and warnings:
765 output_stream.write(
766 'There were presubmit warnings. Sure you want to continue? (y/N): ')
767 response = input_stream.readline()
768 if response.strip().lower() != 'y':
769 error_count += 1
770 return (error_count == 0)
771
772
773def ScanSubDirs(mask, recursive):
774 if not recursive:
775 return [x for x in glob.glob(mask) if '.svn' not in x]
776 else:
777 results = []
778 for root, dirs, files in os.walk('.'):
779 if '.svn' in dirs:
780 dirs.remove('.svn')
781 for name in files:
782 if fnmatch.fnmatch(name, mask):
783 results.append(os.path.join(root, name))
784 return results
785
786
787def ParseFiles(args, recursive):
788 files = []
789 for arg in args:
790 files.extend([('M', file) for file in ScanSubDirs(arg, recursive)])
791 return files
792
793
794def Main(argv):
795 parser = optparse.OptionParser(usage="%prog [options]",
796 version="%prog " + str(__version__))
797 parser.add_option("-c", "--commit", action="store_true",
798 help="Use commit instead of upload checks")
799 parser.add_option("-r", "--recursive", action="store_true",
800 help="Act recursively")
801 parser.add_option("-v", "--verbose", action="store_true",
802 help="Verbose output")
803 options, args = parser.parse_args(argv[1:])
804 files = ParseFiles(args, options.recursive)
805 if options.verbose:
806 print "Found %d files." % len(files)
maruel@chromium.orgde0ba292009-06-06 19:43:27 +0000807 return not DoPresubmitChecks(gcl.ChangeInfo('No name', 0, 0, '', files),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000808 options.commit,
809 options.verbose,
810 sys.stdout,
811 sys.stdin,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000812 None,
813 False)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000814
815
816if __name__ == '__main__':
817 sys.exit(Main(sys.argv))