blob: 8936ec5b1c3f8f7bcc59e7c2f13f41082e1749d6 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.orgb7d46902009-06-10 14:12:10 +00009__version__ = '1.3.2'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000020import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000021import marshal # Exposed through the API.
22import optparse
23import os # Somewhat exposed through the API.
24import pickle # Exposed through the API.
25import re # Exposed through the API.
26import subprocess # Exposed through the API.
27import sys # Parts exposed through API.
28import tempfile # Exposed through the API.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000029import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000031import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000032import urllib2 # Exposed through the API.
maruel@chromium.org1e08c002009-05-28 19:09:33 +000033import warnings
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000034
35# Local imports.
36# TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but
37# for now it would only be a couple of functions so hardly worth it.
38import gcl
maruel@chromium.org46a94102009-05-12 20:32:43 +000039import gclient
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040import presubmit_canned_checks
41
42
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000043class NotImplementedException(Exception):
44 """We're leaving placeholders in a bunch of places to remind us of the
45 design of the API, but we have not implemented all of it yet. Implement as
46 the need arises.
47 """
48 pass
49
50
51def normpath(path):
52 '''Version of os.path.normpath that also changes backward slashes to
53 forward slashes when not running on Windows.
54 '''
55 # This is safe to always do because the Windows version of os.path.normpath
56 # will replace forward slashes with backward slashes.
57 path = path.replace(os.sep, '/')
58 return os.path.normpath(path)
59
60
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000061class OutputApi(object):
62 """This class (more like a module) gets passed to presubmit scripts so that
63 they can specify various types of results.
64 """
65
66 class PresubmitResult(object):
67 """Base class for result objects."""
68
69 def __init__(self, message, items=None, long_text=''):
70 """
71 message: A short one-line message to indicate errors.
72 items: A list of short strings to indicate where errors occurred.
73 long_text: multi-line text output, e.g. from another tool
74 """
75 self._message = message
76 self._items = []
77 if items:
78 self._items = items
79 self._long_text = long_text.rstrip()
80
81 def _Handle(self, output_stream, input_stream, may_prompt=True):
82 """Writes this result to the output stream.
83
84 Args:
85 output_stream: Where to write
86
87 Returns:
88 True if execution may continue, False otherwise.
89 """
90 output_stream.write(self._message)
91 output_stream.write('\n')
92 for item in self._items:
maruel@chromium.org5de13972009-06-10 18:16:06 +000093 output_stream.write(' %s\n' % str(item))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000094 if self._long_text:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +000095 output_stream.write('\n***************\n%s\n***************\n' %
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000096 self._long_text)
97
98 if self.ShouldPrompt() and may_prompt:
99 output_stream.write('Are you sure you want to continue? (y/N): ')
100 response = input_stream.readline()
101 if response.strip().lower() != 'y':
102 return False
103
104 return not self.IsFatal()
105
106 def IsFatal(self):
107 """An error that is fatal stops g4 mail/submit immediately, i.e. before
108 other presubmit scripts are run.
109 """
110 return False
111
112 def ShouldPrompt(self):
113 """Whether this presubmit result should result in a prompt warning."""
114 return False
115
116 class PresubmitError(PresubmitResult):
117 """A hard presubmit error."""
118 def IsFatal(self):
119 return True
120
121 class PresubmitPromptWarning(PresubmitResult):
122 """An warning that prompts the user if they want to continue."""
123 def ShouldPrompt(self):
124 return True
125
126 class PresubmitNotifyResult(PresubmitResult):
127 """Just print something to the screen -- but it's not even a warning."""
128 pass
129
130 class MailTextResult(PresubmitResult):
131 """A warning that should be included in the review request email."""
132 def __init__(self, *args, **kwargs):
133 raise NotImplementedException() # TODO(joi) Implement.
134
135
136class InputApi(object):
137 """An instance of this object is passed to presubmit scripts so they can
138 know stuff about the change they're looking at.
139 """
140
maruel@chromium.org3410d912009-06-09 20:56:16 +0000141 # File extensions that are considered source files from a style guide
142 # perspective. Don't modify this list from a presubmit script!
143 DEFAULT_WHITE_LIST = (
144 # C++ and friends
145 r".*\.c", r".*\.cc", r".*\.cpp", r".*\.h", r".*\.m", r".*\.mm",
146 r".*\.inl", r".*\.asm", r".*\.hxx", r".*\.hpp",
147 # Scripts
148 r".*\.js", r".*\.py", r".*\.json", r".*\.sh", r".*\.rb",
149 # No extension at all
150 r"(^|.*[\\\/])[^.]+$",
151 # Other
152 r".*\.java", r".*\.mk", r".*\.am",
153 )
154
155 # Path regexp that should be excluded from being considered containing source
156 # files. Don't modify this list from a presubmit script!
157 DEFAULT_BLACK_LIST = (
158 r".*\bexperimental[\\\/].*",
159 r".*\bthird_party[\\\/].*",
160 # Output directories (just in case)
161 r".*\bDebug[\\\/].*",
162 r".*\bRelease[\\\/].*",
163 r".*\bxcodebuild[\\\/].*",
164 r".*\bsconsbuild[\\\/].*",
165 # All caps files like README and LICENCE.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000166 r".*\b[A-Z0-9_]+$",
167 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
168 r".*\.git[\\\/].*",
169 r".*\.svn[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000170 )
171
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000172 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000173 """Builds an InputApi object.
174
175 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000176 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000177 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000178 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000179 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000180 # Version number of the presubmit_support script.
181 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000182 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000183 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000184
185 # We expose various modules and functions as attributes of the input_api
186 # so that presubmit scripts don't have to import them.
187 self.basename = os.path.basename
188 self.cPickle = cPickle
189 self.cStringIO = cStringIO
190 self.os_path = os.path
191 self.pickle = pickle
192 self.marshal = marshal
193 self.re = re
194 self.subprocess = subprocess
195 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000196 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000197 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000198 self.urllib2 = urllib2
199
200 # InputApi.platform is the platform you're currently running on.
201 self.platform = sys.platform
202
203 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000204 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000205
206 # We carry the canned checks so presubmit scripts can easily use them.
207 self.canned_checks = presubmit_canned_checks
208
209 def PresubmitLocalPath(self):
210 """Returns the local path of the presubmit script currently being run.
211
212 This is useful if you don't want to hard-code absolute paths in the
213 presubmit script. For example, It can be used to find another file
214 relative to the PRESUBMIT.py script, so the whole tree can be branched and
215 the presubmit script still works, without editing its content.
216 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000217 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000218
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000219 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000220 """Translate a depot path to a local path (relative to client root).
221
222 Args:
223 Depot path as a string.
224
225 Returns:
226 The local path of the depot path under the user's current client, or None
227 if the file is not mapped.
228
229 Remember to check for the None case and show an appropriate error!
230 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000231 local_path = gclient.CaptureSVNInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000232 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000233 return local_path
234
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000235 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000236 """Translate a local path to a depot path.
237
238 Args:
239 Local path (relative to current directory, or absolute) as a string.
240
241 Returns:
242 The depot path (SVN URL) of the file if mapped, otherwise None.
243 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000244 depot_path = gclient.CaptureSVNInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000245 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000246 return depot_path
247
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000248 def AffectedFiles(self, include_dirs=False, include_deletes=True):
249 """Same as input_api.change.AffectedFiles() except only lists files
250 (and optionally directories) in the same directory as the current presubmit
251 script, or subdirectories thereof.
252 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000253 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000254 if len(dir_with_slash) == 1:
255 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000256 return filter(
257 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
258 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000259
260 def LocalPaths(self, include_dirs=False):
261 """Returns local paths of input_api.AffectedFiles()."""
262 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
263
264 def AbsoluteLocalPaths(self, include_dirs=False):
265 """Returns absolute local paths of input_api.AffectedFiles()."""
266 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
267
268 def ServerPaths(self, include_dirs=False):
269 """Returns server paths of input_api.AffectedFiles()."""
270 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
271
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000272 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000273 """Same as input_api.change.AffectedTextFiles() except only lists files
274 in the same directory as the current presubmit script, or subdirectories
275 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000276 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000277 if include_deletes is not None:
278 warnings.warn("AffectedTextFiles(include_deletes=%s)"
279 " is deprecated and ignored" % str(include_deletes),
280 category=DeprecationWarning,
281 stacklevel=2)
282 return filter(lambda x: x.IsTextFile(),
283 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284
maruel@chromium.org3410d912009-06-09 20:56:16 +0000285 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
286 """Filters out files that aren't considered "source file".
287
288 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
289 and InputApi.DEFAULT_BLACK_LIST is used respectively.
290
291 The lists will be compiled as regular expression and
292 AffectedFile.LocalPath() needs to pass both list.
293
294 Note: Copy-paste this function to suit your needs or use a lambda function.
295 """
296 def Find(affected_file, list):
297 for item in list:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000298 local_path = affected_file.LocalPath()
299 if self.re.match(item, local_path):
300 logging.debug("%s matched %s" % (item, local_path))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000301 return True
302 return False
303 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
304 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
305
306 def AffectedSourceFiles(self, source_file):
307 """Filter the list of AffectedTextFiles by the function source_file.
308
309 If source_file is None, InputApi.FilterSourceFile() is used.
310 """
311 if not source_file:
312 source_file = self.FilterSourceFile
313 return filter(source_file, self.AffectedTextFiles())
314
315 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000316 """An iterator over all text lines in "new" version of changed files.
317
318 Only lists lines from new or modified text files in the change that are
319 contained by the directory of the currently executing presubmit script.
320
321 This is useful for doing line-by-line regex checks, like checking for
322 trailing whitespace.
323
324 Yields:
325 a 3 tuple:
326 the AffectedFile instance of the current file;
327 integer line number (1-based); and
328 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000329
330 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000331 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000332 files = self.AffectedSourceFiles(source_file_filter)
333 return InputApi._RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000334
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000335 def ReadFile(self, file, mode='r'):
336 """Reads an arbitrary file.
337
338 Deny reading anything outside the repository.
339 """
340 if isinstance(file, AffectedFile):
341 file = file.AbsoluteLocalPath()
342 if not file.startswith(self.change.RepositoryRoot()):
343 raise IOError('Access outside the repository root is denied.')
344 return gcl.ReadFile(file, mode)
345
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000346 @staticmethod
347 def _RightHandSideLinesImpl(affected_files):
348 """Implements RightHandSideLines for InputApi and GclChange."""
349 for af in affected_files:
350 lines = af.NewContents()
351 line_number = 0
352 for line in lines:
353 line_number += 1
354 yield (af, line_number, line)
355
356
357class AffectedFile(object):
358 """Representation of a file in a change."""
359
360 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000361 self._path = path
362 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000363 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000364 self._is_directory = None
365 self._properties = {}
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000366 self.scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000367
368 def ServerPath(self):
369 """Returns a path string that identifies the file in the SCM system.
370
371 Returns the empty string if the file does not exist in SCM.
372 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000373 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000374
375 def LocalPath(self):
376 """Returns the path of this file on the local disk relative to client root.
377 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000378 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000379
380 def AbsoluteLocalPath(self):
381 """Returns the absolute path of this file on the local disk.
382 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000383 return normpath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000384
385 def IsDirectory(self):
386 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000387 if self._is_directory is None:
388 path = self.AbsoluteLocalPath()
389 self._is_directory = (os.path.exists(path) and
390 os.path.isdir(path))
391 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000392
393 def Action(self):
394 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000395 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
396 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000397 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000398
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000399 def Property(self, property_name):
400 """Returns the specified SCM property of this file, or None if no such
401 property.
402 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000403 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000404
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000405 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000406 """Returns True if the file is a text file and not a binary file.
407
408 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000409 raise NotImplementedError() # Implement when needed
410
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000411 def NewContents(self):
412 """Returns an iterator over the lines in the new version of file.
413
414 The new version is the file in the user's workspace, i.e. the "right hand
415 side".
416
417 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000418 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000419 """
420 if self.IsDirectory():
421 return []
422 else:
423 return gcl.ReadFile(self.AbsoluteLocalPath()).splitlines()
424
425 def OldContents(self):
426 """Returns an iterator over the lines in the old version of file.
427
428 The old version is the file in depot, i.e. the "left hand side".
429 """
430 raise NotImplementedError() # Implement when needed
431
432 def OldFileTempPath(self):
433 """Returns the path on local disk where the old contents resides.
434
435 The old version is the file in depot, i.e. the "left hand side".
436 This is a read-only cached copy of the old contents. *DO NOT* try to
437 modify this file.
438 """
439 raise NotImplementedError() # Implement if/when needed.
440
maruel@chromium.org5de13972009-06-10 18:16:06 +0000441 def __str__(self):
442 return self.LocalPath()
443
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000444
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000445class SvnAffectedFile(AffectedFile):
446 """Representation of a file in a change out of a Subversion checkout."""
447
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000448 def __init__(self, *args, **kwargs):
449 AffectedFile.__init__(self, *args, **kwargs)
450 self._server_path = None
451 self._is_text_file = None
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000452 self.scm = 'svn'
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000453
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000454 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000455 if self._server_path is None:
456 self._server_path = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000457 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000458 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000459
460 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000461 if self._is_directory is None:
462 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000463 if os.path.exists(path):
464 # Retrieve directly from the file system; it is much faster than
465 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000466 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000467 else:
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000468 self._is_directory = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000469 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000470 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000471
472 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000473 if not property_name in self._properties:
474 self._properties[property_name] = gcl.GetSVNFileProperty(
maruel@chromium.org196f8cb2009-06-11 00:32:06 +0000475 self.AbsoluteLocalPath(), property_name).rstrip()
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000476 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000477
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000478 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000479 if self._is_text_file is None:
480 if self.Action() == 'D':
481 # A deleted file is not a text file.
482 self._is_text_file = False
483 elif self.IsDirectory():
484 self._is_text_file = False
485 else:
486 mime_type = gcl.GetSVNFileProperty(self.AbsoluteLocalPath(),
487 'svn:mime-type')
488 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
489 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000490
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000491
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000492class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000493 """Describe a change.
494
495 Used directly by the presubmit scripts to query the current change being
496 tested.
497
498 Instance members:
499 tags: Dictionnary of KEY=VALUE pairs found in the change description.
500 self.KEY: equivalent to tags['KEY']
501 """
502
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000503 _AFFECTED_FILES = AffectedFile
504
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000505 # Matches key/value (or "tag") lines in changelist descriptions.
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000506 _TAG_LINE_RE = re.compile(
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000507 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000508
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000509 def __init__(self, name, description, local_root, files, issue, patchset):
510 if files is None:
511 files = []
512 self._name = name
513 self._full_description = description
514 self._local_root = local_root
515 self.issue = issue
516 self.patchset = patchset
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000517
518 # From the description text, build up a dictionary of key/value pairs
519 # plus the description minus all key/value or "tag" lines.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000520 self._description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000521 self.tags = {}
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000522 for line in self._full_description.splitlines():
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000523 m = self._TAG_LINE_RE.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000524 if m:
525 self.tags[m.group('key')] = m.group('value')
526 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000527 self._description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000528
529 # Change back to text and remove whitespace at end.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000530 self._description_without_tags = '\n'.join(self._description_without_tags)
531 self._description_without_tags = self._description_without_tags.rstrip()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000532
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000533 self._affected_files = [
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000534 self._AFFECTED_FILES(info[1], info[0].strip(), self._local_root)
535 for info in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000536 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000537
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000538 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000539 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000540 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000541
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000542 def DescriptionText(self):
543 """Returns the user-entered changelist description, minus tags.
544
545 Any line in the user-provided description starting with e.g. "FOO="
546 (whitespace permitted before and around) is considered a tag line. Such
547 lines are stripped out of the description this function returns.
548 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000549 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000550
551 def FullDescriptionText(self):
552 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000553 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000554
555 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000556 """Returns the repository (checkout) root directory for this change,
557 as an absolute path.
558 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000559 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000560
561 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000562 """Return tags directly as attributes on the object."""
563 if not re.match(r"^[A-Z_]*$", attr):
564 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000565 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000566
567 def AffectedFiles(self, include_dirs=False, include_deletes=True):
568 """Returns a list of AffectedFile instances for all files in the change.
569
570 Args:
571 include_deletes: If false, deleted files will be filtered out.
572 include_dirs: True to include directories in the list
573
574 Returns:
575 [AffectedFile(path, action), AffectedFile(path, action)]
576 """
577 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000578 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000579 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000580 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000581
582 if include_deletes:
583 return affected
584 else:
585 return filter(lambda x: x.Action() != 'D', affected)
586
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000587 def AffectedTextFiles(self, include_deletes=None):
588 """Return a list of the existing text files in a change."""
589 if include_deletes is not None:
590 warnings.warn("AffectedTextFiles(include_deletes=%s)"
591 " is deprecated and ignored" % str(include_deletes),
592 category=DeprecationWarning,
593 stacklevel=2)
594 return filter(lambda x: x.IsTextFile(),
595 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000596
597 def LocalPaths(self, include_dirs=False):
598 """Convenience function."""
599 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
600
601 def AbsoluteLocalPaths(self, include_dirs=False):
602 """Convenience function."""
603 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
604
605 def ServerPaths(self, include_dirs=False):
606 """Convenience function."""
607 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
608
609 def RightHandSideLines(self):
610 """An iterator over all text lines in "new" version of changed files.
611
612 Lists lines from new or modified text files in the change.
613
614 This is useful for doing line-by-line regex checks, like checking for
615 trailing whitespace.
616
617 Yields:
618 a 3 tuple:
619 the AffectedFile instance of the current file;
620 integer line number (1-based); and
621 the contents of the line as a string.
622 """
623 return InputApi._RightHandSideLinesImpl(
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000624 filter(lambda x: x.IsTextFile(),
625 self.AffectedFiles(include_deletes=False)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626
627
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000628class SvnChange(Change):
629 _AFFECTED_FILES = SvnAffectedFile
630
631
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000632def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000633 """Finds all presubmit files that apply to a given set of source files.
634
635 Args:
636 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000637 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000638
639 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000640 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000641 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000642 entries = []
643 for f in files:
644 f = normpath(os.path.join(root, f))
645 while f:
646 f = os.path.dirname(f)
647 if f in entries:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000648 break
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000649 entries.append(f)
650 if f == root:
651 break
652 entries.sort()
653 entries = map(lambda x: os.path.join(x, 'PRESUBMIT.py'), entries)
654 return filter(lambda x: os.path.isfile(x), entries)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000655
656
657class PresubmitExecuter(object):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000658 def __init__(self, change, committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000659 """
660 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000661 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000662 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
663 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000664 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000665 self.committing = committing
666
667 def ExecPresubmitScript(self, script_text, presubmit_path):
668 """Executes a single presubmit script.
669
670 Args:
671 script_text: The text of the presubmit script.
672 presubmit_path: The path to the presubmit file (this will be reported via
673 input_api.PresubmitLocalPath()).
674
675 Return:
676 A list of result objects, empty if no problems.
677 """
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000678 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000679 context = {}
680 exec script_text in context
681
682 # These function names must change if we make substantial changes to
683 # the presubmit API that are not backwards compatible.
684 if self.committing:
685 function_name = 'CheckChangeOnCommit'
686 else:
687 function_name = 'CheckChangeOnUpload'
688 if function_name in context:
689 context['__args'] = (input_api, OutputApi())
690 result = eval(function_name + '(*__args)', context)
691 if not (isinstance(result, types.TupleType) or
692 isinstance(result, types.ListType)):
693 raise exceptions.RuntimeError(
694 'Presubmit functions must return a tuple or list')
695 for item in result:
696 if not isinstance(item, OutputApi.PresubmitResult):
697 raise exceptions.RuntimeError(
698 'All presubmit results must be of types derived from '
699 'output_api.PresubmitResult')
700 else:
701 result = () # no error since the script doesn't care about current event.
702
703 return result
704
705
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000706def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000707 committing,
708 verbose,
709 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000710 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000711 default_presubmit,
712 may_prompt):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000713 """Runs all presubmit checks that apply to the files in the change.
714
715 This finds all PRESUBMIT.py files in directories enclosing the files in the
716 change (up to the repository root) and calls the relevant entrypoint function
717 depending on whether the change is being committed or uploaded.
718
719 Prints errors, warnings and notifications. Prompts the user for warnings
720 when needed.
721
722 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000723 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000724 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
725 verbose: Prints debug info.
726 output_stream: A stream to write output from presubmit tests to.
727 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000728 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000729 may_prompt: Enable (y/n) questions on warning or error.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000730
731 Return:
732 True if execution can continue, False if not.
733 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000734 presubmit_files = ListRelevantPresubmitFiles(change.AbsoluteLocalPaths(True),
735 change.RepositoryRoot())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000736 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000737 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000738 results = []
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000739 executer = PresubmitExecuter(change, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000740 if default_presubmit:
741 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000742 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000743 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000744 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000745 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000746 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000747 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000748 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000749 # Accept CRLF presubmit script.
maruel@chromium.org277003e2009-05-01 12:51:43 +0000750 presubmit_script = gcl.ReadFile(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000751 results += executer.ExecPresubmitScript(presubmit_script, filename)
752
753 errors = []
754 notifications = []
755 warnings = []
756 for result in results:
757 if not result.IsFatal() and not result.ShouldPrompt():
758 notifications.append(result)
759 elif result.ShouldPrompt():
760 warnings.append(result)
761 else:
762 errors.append(result)
763
764 error_count = 0
765 for name, items in (('Messages', notifications),
766 ('Warnings', warnings),
767 ('ERRORS', errors)):
768 if items:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000769 output_stream.write('** Presubmit %s **\n' % name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000770 for item in items:
771 if not item._Handle(output_stream, input_stream,
772 may_prompt=False):
773 error_count += 1
774 output_stream.write('\n')
maruel@chromium.org07bbc212009-06-11 02:08:41 +0000775 if not errors and warnings and may_prompt:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000776 output_stream.write(
777 'There were presubmit warnings. Sure you want to continue? (y/N): ')
778 response = input_stream.readline()
779 if response.strip().lower() != 'y':
780 error_count += 1
781 return (error_count == 0)
782
783
784def ScanSubDirs(mask, recursive):
785 if not recursive:
786 return [x for x in glob.glob(mask) if '.svn' not in x]
787 else:
788 results = []
789 for root, dirs, files in os.walk('.'):
790 if '.svn' in dirs:
791 dirs.remove('.svn')
792 for name in files:
793 if fnmatch.fnmatch(name, mask):
794 results.append(os.path.join(root, name))
795 return results
796
797
798def ParseFiles(args, recursive):
799 files = []
800 for arg in args:
801 files.extend([('M', file) for file in ScanSubDirs(arg, recursive)])
802 return files
803
804
805def Main(argv):
806 parser = optparse.OptionParser(usage="%prog [options]",
807 version="%prog " + str(__version__))
808 parser.add_option("-c", "--commit", action="store_true",
809 help="Use commit instead of upload checks")
810 parser.add_option("-r", "--recursive", action="store_true",
811 help="Act recursively")
812 parser.add_option("-v", "--verbose", action="store_true",
813 help="Verbose output")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000814 parser.add_option("--files", default='')
815 parser.add_option("--name", default='no name')
816 parser.add_option("--description", default='')
817 parser.add_option("--issue", type='int', default=0)
818 parser.add_option("--patchset", type='int', default=0)
819 parser.add_options("--root", default='')
820 parser.add_options("--default_presubmit", default='')
821 parser.add_options("--may_prompt", action='store_true')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000822 options, args = parser.parse_args(argv[1:])
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000823 if not options.files:
824 options.files = ParseFiles(args, options.recursive)
825 if not options.root:
826 options.root = gcl.GetRepositoryRoot()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000827 if options.verbose:
828 print "Found %d files." % len(files)
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000829 return not DoPresubmitChecks(SvnChange(options.name,
830 options.description,
831 options.root,
832 options.files,
833 options.issue,
834 options.patchset),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000835 options.commit,
836 options.verbose,
837 sys.stdout,
838 sys.stdin,
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000839 options.default_presubmit,
840 options.may_prompt)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000841
842
843if __name__ == '__main__':
844 sys.exit(Main(sys.argv))