blob: 1368665ed6ba34c276a266bf2bffd71440e42681 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.orgb7d46902009-06-10 14:12:10 +00009__version__ = '1.3.2'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
20import marshal # Exposed through the API.
21import optparse
22import os # Somewhat exposed through the API.
23import pickle # Exposed through the API.
24import re # Exposed through the API.
25import subprocess # Exposed through the API.
26import sys # Parts exposed through API.
27import tempfile # Exposed through the API.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000028import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000029import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000030import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000031import urllib2 # Exposed through the API.
maruel@chromium.org1e08c002009-05-28 19:09:33 +000032import warnings
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033
34# Local imports.
35# TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but
36# for now it would only be a couple of functions so hardly worth it.
37import gcl
maruel@chromium.org46a94102009-05-12 20:32:43 +000038import gclient
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000039import presubmit_canned_checks
40
41
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000042class NotImplementedException(Exception):
43 """We're leaving placeholders in a bunch of places to remind us of the
44 design of the API, but we have not implemented all of it yet. Implement as
45 the need arises.
46 """
47 pass
48
49
50def normpath(path):
51 '''Version of os.path.normpath that also changes backward slashes to
52 forward slashes when not running on Windows.
53 '''
54 # This is safe to always do because the Windows version of os.path.normpath
55 # will replace forward slashes with backward slashes.
56 path = path.replace(os.sep, '/')
57 return os.path.normpath(path)
58
59
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000060class OutputApi(object):
61 """This class (more like a module) gets passed to presubmit scripts so that
62 they can specify various types of results.
63 """
64
65 class PresubmitResult(object):
66 """Base class for result objects."""
67
68 def __init__(self, message, items=None, long_text=''):
69 """
70 message: A short one-line message to indicate errors.
71 items: A list of short strings to indicate where errors occurred.
72 long_text: multi-line text output, e.g. from another tool
73 """
74 self._message = message
75 self._items = []
76 if items:
77 self._items = items
78 self._long_text = long_text.rstrip()
79
80 def _Handle(self, output_stream, input_stream, may_prompt=True):
81 """Writes this result to the output stream.
82
83 Args:
84 output_stream: Where to write
85
86 Returns:
87 True if execution may continue, False otherwise.
88 """
89 output_stream.write(self._message)
90 output_stream.write('\n')
91 for item in self._items:
maruel@chromium.org5de13972009-06-10 18:16:06 +000092 output_stream.write(' %s\n' % str(item))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000093 if self._long_text:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +000094 output_stream.write('\n***************\n%s\n***************\n' %
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000095 self._long_text)
96
97 if self.ShouldPrompt() and may_prompt:
98 output_stream.write('Are you sure you want to continue? (y/N): ')
99 response = input_stream.readline()
100 if response.strip().lower() != 'y':
101 return False
102
103 return not self.IsFatal()
104
105 def IsFatal(self):
106 """An error that is fatal stops g4 mail/submit immediately, i.e. before
107 other presubmit scripts are run.
108 """
109 return False
110
111 def ShouldPrompt(self):
112 """Whether this presubmit result should result in a prompt warning."""
113 return False
114
115 class PresubmitError(PresubmitResult):
116 """A hard presubmit error."""
117 def IsFatal(self):
118 return True
119
120 class PresubmitPromptWarning(PresubmitResult):
121 """An warning that prompts the user if they want to continue."""
122 def ShouldPrompt(self):
123 return True
124
125 class PresubmitNotifyResult(PresubmitResult):
126 """Just print something to the screen -- but it's not even a warning."""
127 pass
128
129 class MailTextResult(PresubmitResult):
130 """A warning that should be included in the review request email."""
131 def __init__(self, *args, **kwargs):
132 raise NotImplementedException() # TODO(joi) Implement.
133
134
135class InputApi(object):
136 """An instance of this object is passed to presubmit scripts so they can
137 know stuff about the change they're looking at.
138 """
139
maruel@chromium.org3410d912009-06-09 20:56:16 +0000140 # File extensions that are considered source files from a style guide
141 # perspective. Don't modify this list from a presubmit script!
142 DEFAULT_WHITE_LIST = (
143 # C++ and friends
144 r".*\.c", r".*\.cc", r".*\.cpp", r".*\.h", r".*\.m", r".*\.mm",
145 r".*\.inl", r".*\.asm", r".*\.hxx", r".*\.hpp",
146 # Scripts
147 r".*\.js", r".*\.py", r".*\.json", r".*\.sh", r".*\.rb",
148 # No extension at all
149 r"(^|.*[\\\/])[^.]+$",
150 # Other
151 r".*\.java", r".*\.mk", r".*\.am",
152 )
153
154 # Path regexp that should be excluded from being considered containing source
155 # files. Don't modify this list from a presubmit script!
156 DEFAULT_BLACK_LIST = (
157 r".*\bexperimental[\\\/].*",
158 r".*\bthird_party[\\\/].*",
159 # Output directories (just in case)
160 r".*\bDebug[\\\/].*",
161 r".*\bRelease[\\\/].*",
162 r".*\bxcodebuild[\\\/].*",
163 r".*\bsconsbuild[\\\/].*",
164 # All caps files like README and LICENCE.
165 r".*\b[A-Z0-9_]+",
166 # SCM (can happen in dual SCM configuration)
167 r".*\b\.git[\\\/].*",
168 r".*\b\.svn[\\\/].*",
169 )
170
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000171 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000172 """Builds an InputApi object.
173
174 Args:
175 change: A presubmit.GclChange object.
176 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000177 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000178 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000179 # Version number of the presubmit_support script.
180 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000181 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000182 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000183
184 # We expose various modules and functions as attributes of the input_api
185 # so that presubmit scripts don't have to import them.
186 self.basename = os.path.basename
187 self.cPickle = cPickle
188 self.cStringIO = cStringIO
189 self.os_path = os.path
190 self.pickle = pickle
191 self.marshal = marshal
192 self.re = re
193 self.subprocess = subprocess
194 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000195 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000196 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000197 self.urllib2 = urllib2
198
199 # InputApi.platform is the platform you're currently running on.
200 self.platform = sys.platform
201
202 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000203 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000204
205 # We carry the canned checks so presubmit scripts can easily use them.
206 self.canned_checks = presubmit_canned_checks
207
208 def PresubmitLocalPath(self):
209 """Returns the local path of the presubmit script currently being run.
210
211 This is useful if you don't want to hard-code absolute paths in the
212 presubmit script. For example, It can be used to find another file
213 relative to the PRESUBMIT.py script, so the whole tree can be branched and
214 the presubmit script still works, without editing its content.
215 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000216 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000217
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000218 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000219 """Translate a depot path to a local path (relative to client root).
220
221 Args:
222 Depot path as a string.
223
224 Returns:
225 The local path of the depot path under the user's current client, or None
226 if the file is not mapped.
227
228 Remember to check for the None case and show an appropriate error!
229 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000230 local_path = gclient.CaptureSVNInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000231 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000232 return local_path
233
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000234 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000235 """Translate a local path to a depot path.
236
237 Args:
238 Local path (relative to current directory, or absolute) as a string.
239
240 Returns:
241 The depot path (SVN URL) of the file if mapped, otherwise None.
242 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000243 depot_path = gclient.CaptureSVNInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000244 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000245 return depot_path
246
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000247 def AffectedFiles(self, include_dirs=False, include_deletes=True):
248 """Same as input_api.change.AffectedFiles() except only lists files
249 (and optionally directories) in the same directory as the current presubmit
250 script, or subdirectories thereof.
251 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000252 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000253 if len(dir_with_slash) == 1:
254 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000255 return filter(
256 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
257 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000258
259 def LocalPaths(self, include_dirs=False):
260 """Returns local paths of input_api.AffectedFiles()."""
261 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
262
263 def AbsoluteLocalPaths(self, include_dirs=False):
264 """Returns absolute local paths of input_api.AffectedFiles()."""
265 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
266
267 def ServerPaths(self, include_dirs=False):
268 """Returns server paths of input_api.AffectedFiles()."""
269 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
270
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000271 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000272 """Same as input_api.change.AffectedTextFiles() except only lists files
273 in the same directory as the current presubmit script, or subdirectories
274 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000275 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000276 if include_deletes is not None:
277 warnings.warn("AffectedTextFiles(include_deletes=%s)"
278 " is deprecated and ignored" % str(include_deletes),
279 category=DeprecationWarning,
280 stacklevel=2)
281 return filter(lambda x: x.IsTextFile(),
282 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000283
maruel@chromium.org3410d912009-06-09 20:56:16 +0000284 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
285 """Filters out files that aren't considered "source file".
286
287 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
288 and InputApi.DEFAULT_BLACK_LIST is used respectively.
289
290 The lists will be compiled as regular expression and
291 AffectedFile.LocalPath() needs to pass both list.
292
293 Note: Copy-paste this function to suit your needs or use a lambda function.
294 """
295 def Find(affected_file, list):
296 for item in list:
297 if self.re.match(item, affected_file.LocalPath()):
298 return True
299 return False
300 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
301 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
302
303 def AffectedSourceFiles(self, source_file):
304 """Filter the list of AffectedTextFiles by the function source_file.
305
306 If source_file is None, InputApi.FilterSourceFile() is used.
307 """
308 if not source_file:
309 source_file = self.FilterSourceFile
310 return filter(source_file, self.AffectedTextFiles())
311
312 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000313 """An iterator over all text lines in "new" version of changed files.
314
315 Only lists lines from new or modified text files in the change that are
316 contained by the directory of the currently executing presubmit script.
317
318 This is useful for doing line-by-line regex checks, like checking for
319 trailing whitespace.
320
321 Yields:
322 a 3 tuple:
323 the AffectedFile instance of the current file;
324 integer line number (1-based); and
325 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000326
327 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000328 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000329 files = self.AffectedSourceFiles(source_file_filter)
330 return InputApi._RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000331
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000332 def ReadFile(self, file, mode='r'):
333 """Reads an arbitrary file.
334
335 Deny reading anything outside the repository.
336 """
337 if isinstance(file, AffectedFile):
338 file = file.AbsoluteLocalPath()
339 if not file.startswith(self.change.RepositoryRoot()):
340 raise IOError('Access outside the repository root is denied.')
341 return gcl.ReadFile(file, mode)
342
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000343 @staticmethod
344 def _RightHandSideLinesImpl(affected_files):
345 """Implements RightHandSideLines for InputApi and GclChange."""
346 for af in affected_files:
347 lines = af.NewContents()
348 line_number = 0
349 for line in lines:
350 line_number += 1
351 yield (af, line_number, line)
352
353
354class AffectedFile(object):
355 """Representation of a file in a change."""
356
357 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000358 self._path = path
359 self._action = action
360 self._repository_root = repository_root
361 self._is_directory = None
362 self._properties = {}
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000363 self.scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000364
365 def ServerPath(self):
366 """Returns a path string that identifies the file in the SCM system.
367
368 Returns the empty string if the file does not exist in SCM.
369 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000370 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000371
372 def LocalPath(self):
373 """Returns the path of this file on the local disk relative to client root.
374 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000375 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000376
377 def AbsoluteLocalPath(self):
378 """Returns the absolute path of this file on the local disk.
379 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000380 return normpath(os.path.join(self._repository_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000381
382 def IsDirectory(self):
383 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000384 if self._is_directory is None:
385 path = self.AbsoluteLocalPath()
386 self._is_directory = (os.path.exists(path) and
387 os.path.isdir(path))
388 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000389
390 def Action(self):
391 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000392 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
393 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000394 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000395
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000396 def Property(self, property_name):
397 """Returns the specified SCM property of this file, or None if no such
398 property.
399 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000400 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000401
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000402 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000403 """Returns True if the file is a text file and not a binary file.
404
405 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000406 raise NotImplementedError() # Implement when needed
407
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000408 def NewContents(self):
409 """Returns an iterator over the lines in the new version of file.
410
411 The new version is the file in the user's workspace, i.e. the "right hand
412 side".
413
414 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000415 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000416 """
417 if self.IsDirectory():
418 return []
419 else:
420 return gcl.ReadFile(self.AbsoluteLocalPath()).splitlines()
421
422 def OldContents(self):
423 """Returns an iterator over the lines in the old version of file.
424
425 The old version is the file in depot, i.e. the "left hand side".
426 """
427 raise NotImplementedError() # Implement when needed
428
429 def OldFileTempPath(self):
430 """Returns the path on local disk where the old contents resides.
431
432 The old version is the file in depot, i.e. the "left hand side".
433 This is a read-only cached copy of the old contents. *DO NOT* try to
434 modify this file.
435 """
436 raise NotImplementedError() # Implement if/when needed.
437
maruel@chromium.org5de13972009-06-10 18:16:06 +0000438 def __str__(self):
439 return self.LocalPath()
440
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000441
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000442class SvnAffectedFile(AffectedFile):
443 """Representation of a file in a change out of a Subversion checkout."""
444
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000445 def __init__(self, *args, **kwargs):
446 AffectedFile.__init__(self, *args, **kwargs)
447 self._server_path = None
448 self._is_text_file = None
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000449 self.scm = 'svn'
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000450
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000451 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000452 if self._server_path is None:
453 self._server_path = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000454 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000455 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000456
457 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000458 if self._is_directory is None:
459 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000460 if os.path.exists(path):
461 # Retrieve directly from the file system; it is much faster than
462 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000463 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000464 else:
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000465 self._is_directory = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000466 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000467 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000468
469 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000470 if not property_name in self._properties:
471 self._properties[property_name] = gcl.GetSVNFileProperty(
maruel@chromium.org196f8cb2009-06-11 00:32:06 +0000472 self.AbsoluteLocalPath(), property_name).rstrip()
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000473 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000474
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000475 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000476 if self._is_text_file is None:
477 if self.Action() == 'D':
478 # A deleted file is not a text file.
479 self._is_text_file = False
480 elif self.IsDirectory():
481 self._is_text_file = False
482 else:
483 mime_type = gcl.GetSVNFileProperty(self.AbsoluteLocalPath(),
484 'svn:mime-type')
485 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
486 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000487
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000488
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000489class GclChange(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000490 """Describe a change.
491
492 Used directly by the presubmit scripts to query the current change being
493 tested.
494
495 Instance members:
496 tags: Dictionnary of KEY=VALUE pairs found in the change description.
497 self.KEY: equivalent to tags['KEY']
498 """
499
500 # Matches key/value (or "tag") lines in changelist descriptions.
501 _tag_line_re = re.compile(
502 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000503
504 def __init__(self, change_info, repository_root=''):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000505 # Do not keep a reference to the original change_info.
506 self._name = change_info.name
507 self._full_description = change_info.description
508 self._repository_root = repository_root
maruel@chromium.org32ba2602009-06-06 18:44:48 +0000509 self.issue = change_info.issue
510 self.patchset = change_info.patchset
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000511
512 # From the description text, build up a dictionary of key/value pairs
513 # plus the description minus all key/value or "tag" lines.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000514 self._description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000515 self.tags = {}
516 for line in change_info.description.splitlines():
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000517 m = self._tag_line_re.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000518 if m:
519 self.tags[m.group('key')] = m.group('value')
520 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000521 self._description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000522
523 # Change back to text and remove whitespace at end.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000524 self._description_without_tags = '\n'.join(self._description_without_tags)
525 self._description_without_tags = self._description_without_tags.rstrip()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000526
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000527 self._affected_files = [
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000528 SvnAffectedFile(info[1], info[0].strip(), repository_root)
529 for info in change_info.files
530 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000531
532 def Change(self):
533 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000534 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000535
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000536 def DescriptionText(self):
537 """Returns the user-entered changelist description, minus tags.
538
539 Any line in the user-provided description starting with e.g. "FOO="
540 (whitespace permitted before and around) is considered a tag line. Such
541 lines are stripped out of the description this function returns.
542 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000543 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000544
545 def FullDescriptionText(self):
546 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000547 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000548
549 def RepositoryRoot(self):
550 """Returns the repository root for this change, as an absolute path."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000551 return self._repository_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000552
553 def __getattr__(self, attr):
554 """Return keys directly as attributes on the object.
555
556 You may use a friendly name (from SPECIAL_KEYS) or the actual name of
557 the key.
558 """
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000559 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000560
561 def AffectedFiles(self, include_dirs=False, include_deletes=True):
562 """Returns a list of AffectedFile instances for all files in the change.
563
564 Args:
565 include_deletes: If false, deleted files will be filtered out.
566 include_dirs: True to include directories in the list
567
568 Returns:
569 [AffectedFile(path, action), AffectedFile(path, action)]
570 """
571 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000572 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000573 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000574 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000575
576 if include_deletes:
577 return affected
578 else:
579 return filter(lambda x: x.Action() != 'D', affected)
580
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000581 def AffectedTextFiles(self, include_deletes=None):
582 """Return a list of the existing text files in a change."""
583 if include_deletes is not None:
584 warnings.warn("AffectedTextFiles(include_deletes=%s)"
585 " is deprecated and ignored" % str(include_deletes),
586 category=DeprecationWarning,
587 stacklevel=2)
588 return filter(lambda x: x.IsTextFile(),
589 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000590
591 def LocalPaths(self, include_dirs=False):
592 """Convenience function."""
593 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
594
595 def AbsoluteLocalPaths(self, include_dirs=False):
596 """Convenience function."""
597 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
598
599 def ServerPaths(self, include_dirs=False):
600 """Convenience function."""
601 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
602
603 def RightHandSideLines(self):
604 """An iterator over all text lines in "new" version of changed files.
605
606 Lists lines from new or modified text files in the change.
607
608 This is useful for doing line-by-line regex checks, like checking for
609 trailing whitespace.
610
611 Yields:
612 a 3 tuple:
613 the AffectedFile instance of the current file;
614 integer line number (1-based); and
615 the contents of the line as a string.
616 """
617 return InputApi._RightHandSideLinesImpl(
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000618 filter(lambda x: x.IsTextFile(),
619 self.AffectedFiles(include_deletes=False)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000620
621
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000622def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000623 """Finds all presubmit files that apply to a given set of source files.
624
625 Args:
626 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000627 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000628
629 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000630 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000631 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000632 entries = []
633 for f in files:
634 f = normpath(os.path.join(root, f))
635 while f:
636 f = os.path.dirname(f)
637 if f in entries:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000638 break
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000639 entries.append(f)
640 if f == root:
641 break
642 entries.sort()
643 entries = map(lambda x: os.path.join(x, 'PRESUBMIT.py'), entries)
644 return filter(lambda x: os.path.isfile(x), entries)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000645
646
647class PresubmitExecuter(object):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000648 def __init__(self, change_info, committing):
649 """
650 Args:
651 change_info: The ChangeInfo object for the change.
652 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
653 """
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000654 # TODO(maruel): Determine the SCM.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000655 self.change = GclChange(change_info, gcl.GetRepositoryRoot())
656 self.committing = committing
657
658 def ExecPresubmitScript(self, script_text, presubmit_path):
659 """Executes a single presubmit script.
660
661 Args:
662 script_text: The text of the presubmit script.
663 presubmit_path: The path to the presubmit file (this will be reported via
664 input_api.PresubmitLocalPath()).
665
666 Return:
667 A list of result objects, empty if no problems.
668 """
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000669 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000670 context = {}
671 exec script_text in context
672
673 # These function names must change if we make substantial changes to
674 # the presubmit API that are not backwards compatible.
675 if self.committing:
676 function_name = 'CheckChangeOnCommit'
677 else:
678 function_name = 'CheckChangeOnUpload'
679 if function_name in context:
680 context['__args'] = (input_api, OutputApi())
681 result = eval(function_name + '(*__args)', context)
682 if not (isinstance(result, types.TupleType) or
683 isinstance(result, types.ListType)):
684 raise exceptions.RuntimeError(
685 'Presubmit functions must return a tuple or list')
686 for item in result:
687 if not isinstance(item, OutputApi.PresubmitResult):
688 raise exceptions.RuntimeError(
689 'All presubmit results must be of types derived from '
690 'output_api.PresubmitResult')
691 else:
692 result = () # no error since the script doesn't care about current event.
693
694 return result
695
696
697def DoPresubmitChecks(change_info,
698 committing,
699 verbose,
700 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000701 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000702 default_presubmit,
703 may_prompt):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000704 """Runs all presubmit checks that apply to the files in the change.
705
706 This finds all PRESUBMIT.py files in directories enclosing the files in the
707 change (up to the repository root) and calls the relevant entrypoint function
708 depending on whether the change is being committed or uploaded.
709
710 Prints errors, warnings and notifications. Prompts the user for warnings
711 when needed.
712
713 Args:
714 change_info: The ChangeInfo object for the change.
715 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
716 verbose: Prints debug info.
717 output_stream: A stream to write output from presubmit tests to.
718 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000719 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000720 may_prompt: Enable (y/n) questions on warning or error.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000721
722 Return:
723 True if execution can continue, False if not.
724 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000725 checkout_root = gcl.GetRepositoryRoot()
726 presubmit_files = ListRelevantPresubmitFiles(change_info.FileList(),
727 checkout_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000728 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000729 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000730 results = []
731 executer = PresubmitExecuter(change_info, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000732 if default_presubmit:
733 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000734 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000735 fake_path = os.path.join(checkout_root, 'PRESUBMIT.py')
736 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000737 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000738 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000739 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000740 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000741 # Accept CRLF presubmit script.
maruel@chromium.org277003e2009-05-01 12:51:43 +0000742 presubmit_script = gcl.ReadFile(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000743 results += executer.ExecPresubmitScript(presubmit_script, filename)
744
745 errors = []
746 notifications = []
747 warnings = []
748 for result in results:
749 if not result.IsFatal() and not result.ShouldPrompt():
750 notifications.append(result)
751 elif result.ShouldPrompt():
752 warnings.append(result)
753 else:
754 errors.append(result)
755
756 error_count = 0
757 for name, items in (('Messages', notifications),
758 ('Warnings', warnings),
759 ('ERRORS', errors)):
760 if items:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000761 output_stream.write('** Presubmit %s **\n' % name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000762 for item in items:
763 if not item._Handle(output_stream, input_stream,
764 may_prompt=False):
765 error_count += 1
766 output_stream.write('\n')
767 if not errors and warnings:
768 output_stream.write(
769 'There were presubmit warnings. Sure you want to continue? (y/N): ')
770 response = input_stream.readline()
771 if response.strip().lower() != 'y':
772 error_count += 1
773 return (error_count == 0)
774
775
776def ScanSubDirs(mask, recursive):
777 if not recursive:
778 return [x for x in glob.glob(mask) if '.svn' not in x]
779 else:
780 results = []
781 for root, dirs, files in os.walk('.'):
782 if '.svn' in dirs:
783 dirs.remove('.svn')
784 for name in files:
785 if fnmatch.fnmatch(name, mask):
786 results.append(os.path.join(root, name))
787 return results
788
789
790def ParseFiles(args, recursive):
791 files = []
792 for arg in args:
793 files.extend([('M', file) for file in ScanSubDirs(arg, recursive)])
794 return files
795
796
797def Main(argv):
798 parser = optparse.OptionParser(usage="%prog [options]",
799 version="%prog " + str(__version__))
800 parser.add_option("-c", "--commit", action="store_true",
801 help="Use commit instead of upload checks")
802 parser.add_option("-r", "--recursive", action="store_true",
803 help="Act recursively")
804 parser.add_option("-v", "--verbose", action="store_true",
805 help="Verbose output")
806 options, args = parser.parse_args(argv[1:])
807 files = ParseFiles(args, options.recursive)
808 if options.verbose:
809 print "Found %d files." % len(files)
maruel@chromium.orgde0ba292009-06-06 19:43:27 +0000810 return not DoPresubmitChecks(gcl.ChangeInfo('No name', 0, 0, '', files),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000811 options.commit,
812 options.verbose,
813 sys.stdout,
814 sys.stdin,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000815 None,
816 False)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000817
818
819if __name__ == '__main__':
820 sys.exit(Main(sys.argv))