blob: 91fcd047e7e142d36f9ef98d64808c7780f83510 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.orgb7d46902009-06-10 14:12:10 +00009__version__ = '1.3.2'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000020import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000021import marshal # Exposed through the API.
22import optparse
23import os # Somewhat exposed through the API.
24import pickle # Exposed through the API.
25import re # Exposed through the API.
26import subprocess # Exposed through the API.
27import sys # Parts exposed through API.
28import tempfile # Exposed through the API.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000029import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000031import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000032import urllib2 # Exposed through the API.
maruel@chromium.org1e08c002009-05-28 19:09:33 +000033import warnings
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000034
35# Local imports.
36# TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but
37# for now it would only be a couple of functions so hardly worth it.
38import gcl
maruel@chromium.org46a94102009-05-12 20:32:43 +000039import gclient
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040import presubmit_canned_checks
41
42
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000043class NotImplementedException(Exception):
44 """We're leaving placeholders in a bunch of places to remind us of the
45 design of the API, but we have not implemented all of it yet. Implement as
46 the need arises.
47 """
48 pass
49
50
51def normpath(path):
52 '''Version of os.path.normpath that also changes backward slashes to
53 forward slashes when not running on Windows.
54 '''
55 # This is safe to always do because the Windows version of os.path.normpath
56 # will replace forward slashes with backward slashes.
57 path = path.replace(os.sep, '/')
58 return os.path.normpath(path)
59
60
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000061class OutputApi(object):
62 """This class (more like a module) gets passed to presubmit scripts so that
63 they can specify various types of results.
64 """
65
66 class PresubmitResult(object):
67 """Base class for result objects."""
68
69 def __init__(self, message, items=None, long_text=''):
70 """
71 message: A short one-line message to indicate errors.
72 items: A list of short strings to indicate where errors occurred.
73 long_text: multi-line text output, e.g. from another tool
74 """
75 self._message = message
76 self._items = []
77 if items:
78 self._items = items
79 self._long_text = long_text.rstrip()
80
81 def _Handle(self, output_stream, input_stream, may_prompt=True):
82 """Writes this result to the output stream.
83
84 Args:
85 output_stream: Where to write
86
87 Returns:
88 True if execution may continue, False otherwise.
89 """
90 output_stream.write(self._message)
91 output_stream.write('\n')
92 for item in self._items:
maruel@chromium.org5de13972009-06-10 18:16:06 +000093 output_stream.write(' %s\n' % str(item))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000094 if self._long_text:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +000095 output_stream.write('\n***************\n%s\n***************\n' %
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000096 self._long_text)
97
98 if self.ShouldPrompt() and may_prompt:
99 output_stream.write('Are you sure you want to continue? (y/N): ')
100 response = input_stream.readline()
101 if response.strip().lower() != 'y':
102 return False
103
104 return not self.IsFatal()
105
106 def IsFatal(self):
107 """An error that is fatal stops g4 mail/submit immediately, i.e. before
108 other presubmit scripts are run.
109 """
110 return False
111
112 def ShouldPrompt(self):
113 """Whether this presubmit result should result in a prompt warning."""
114 return False
115
116 class PresubmitError(PresubmitResult):
117 """A hard presubmit error."""
118 def IsFatal(self):
119 return True
120
121 class PresubmitPromptWarning(PresubmitResult):
122 """An warning that prompts the user if they want to continue."""
123 def ShouldPrompt(self):
124 return True
125
126 class PresubmitNotifyResult(PresubmitResult):
127 """Just print something to the screen -- but it's not even a warning."""
128 pass
129
130 class MailTextResult(PresubmitResult):
131 """A warning that should be included in the review request email."""
132 def __init__(self, *args, **kwargs):
133 raise NotImplementedException() # TODO(joi) Implement.
134
135
136class InputApi(object):
137 """An instance of this object is passed to presubmit scripts so they can
138 know stuff about the change they're looking at.
139 """
140
maruel@chromium.org3410d912009-06-09 20:56:16 +0000141 # File extensions that are considered source files from a style guide
142 # perspective. Don't modify this list from a presubmit script!
143 DEFAULT_WHITE_LIST = (
144 # C++ and friends
145 r".*\.c", r".*\.cc", r".*\.cpp", r".*\.h", r".*\.m", r".*\.mm",
146 r".*\.inl", r".*\.asm", r".*\.hxx", r".*\.hpp",
147 # Scripts
148 r".*\.js", r".*\.py", r".*\.json", r".*\.sh", r".*\.rb",
149 # No extension at all
150 r"(^|.*[\\\/])[^.]+$",
151 # Other
152 r".*\.java", r".*\.mk", r".*\.am",
153 )
154
155 # Path regexp that should be excluded from being considered containing source
156 # files. Don't modify this list from a presubmit script!
157 DEFAULT_BLACK_LIST = (
158 r".*\bexperimental[\\\/].*",
159 r".*\bthird_party[\\\/].*",
160 # Output directories (just in case)
161 r".*\bDebug[\\\/].*",
162 r".*\bRelease[\\\/].*",
163 r".*\bxcodebuild[\\\/].*",
164 r".*\bsconsbuild[\\\/].*",
165 # All caps files like README and LICENCE.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000166 r".*\b[A-Z0-9_]+$",
167 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
168 r".*\.git[\\\/].*",
169 r".*\.svn[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000170 )
171
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000172 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000173 """Builds an InputApi object.
174
175 Args:
176 change: A presubmit.GclChange object.
177 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000178 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000179 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000180 # Version number of the presubmit_support script.
181 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000182 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000183 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000184
185 # We expose various modules and functions as attributes of the input_api
186 # so that presubmit scripts don't have to import them.
187 self.basename = os.path.basename
188 self.cPickle = cPickle
189 self.cStringIO = cStringIO
190 self.os_path = os.path
191 self.pickle = pickle
192 self.marshal = marshal
193 self.re = re
194 self.subprocess = subprocess
195 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000196 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000197 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000198 self.urllib2 = urllib2
199
200 # InputApi.platform is the platform you're currently running on.
201 self.platform = sys.platform
202
203 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000204 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000205
206 # We carry the canned checks so presubmit scripts can easily use them.
207 self.canned_checks = presubmit_canned_checks
208
209 def PresubmitLocalPath(self):
210 """Returns the local path of the presubmit script currently being run.
211
212 This is useful if you don't want to hard-code absolute paths in the
213 presubmit script. For example, It can be used to find another file
214 relative to the PRESUBMIT.py script, so the whole tree can be branched and
215 the presubmit script still works, without editing its content.
216 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000217 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000218
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000219 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000220 """Translate a depot path to a local path (relative to client root).
221
222 Args:
223 Depot path as a string.
224
225 Returns:
226 The local path of the depot path under the user's current client, or None
227 if the file is not mapped.
228
229 Remember to check for the None case and show an appropriate error!
230 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000231 local_path = gclient.CaptureSVNInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000232 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000233 return local_path
234
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000235 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000236 """Translate a local path to a depot path.
237
238 Args:
239 Local path (relative to current directory, or absolute) as a string.
240
241 Returns:
242 The depot path (SVN URL) of the file if mapped, otherwise None.
243 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000244 depot_path = gclient.CaptureSVNInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000245 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000246 return depot_path
247
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000248 def AffectedFiles(self, include_dirs=False, include_deletes=True):
249 """Same as input_api.change.AffectedFiles() except only lists files
250 (and optionally directories) in the same directory as the current presubmit
251 script, or subdirectories thereof.
252 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000253 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000254 if len(dir_with_slash) == 1:
255 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000256 return filter(
257 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
258 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000259
260 def LocalPaths(self, include_dirs=False):
261 """Returns local paths of input_api.AffectedFiles()."""
262 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
263
264 def AbsoluteLocalPaths(self, include_dirs=False):
265 """Returns absolute local paths of input_api.AffectedFiles()."""
266 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
267
268 def ServerPaths(self, include_dirs=False):
269 """Returns server paths of input_api.AffectedFiles()."""
270 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
271
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000272 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000273 """Same as input_api.change.AffectedTextFiles() except only lists files
274 in the same directory as the current presubmit script, or subdirectories
275 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000276 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000277 if include_deletes is not None:
278 warnings.warn("AffectedTextFiles(include_deletes=%s)"
279 " is deprecated and ignored" % str(include_deletes),
280 category=DeprecationWarning,
281 stacklevel=2)
282 return filter(lambda x: x.IsTextFile(),
283 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284
maruel@chromium.org3410d912009-06-09 20:56:16 +0000285 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
286 """Filters out files that aren't considered "source file".
287
288 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
289 and InputApi.DEFAULT_BLACK_LIST is used respectively.
290
291 The lists will be compiled as regular expression and
292 AffectedFile.LocalPath() needs to pass both list.
293
294 Note: Copy-paste this function to suit your needs or use a lambda function.
295 """
296 def Find(affected_file, list):
297 for item in list:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000298 local_path = affected_file.LocalPath()
299 if self.re.match(item, local_path):
300 logging.debug("%s matched %s" % (item, local_path))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000301 return True
302 return False
303 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
304 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
305
306 def AffectedSourceFiles(self, source_file):
307 """Filter the list of AffectedTextFiles by the function source_file.
308
309 If source_file is None, InputApi.FilterSourceFile() is used.
310 """
311 if not source_file:
312 source_file = self.FilterSourceFile
313 return filter(source_file, self.AffectedTextFiles())
314
315 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000316 """An iterator over all text lines in "new" version of changed files.
317
318 Only lists lines from new or modified text files in the change that are
319 contained by the directory of the currently executing presubmit script.
320
321 This is useful for doing line-by-line regex checks, like checking for
322 trailing whitespace.
323
324 Yields:
325 a 3 tuple:
326 the AffectedFile instance of the current file;
327 integer line number (1-based); and
328 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000329
330 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000331 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000332 files = self.AffectedSourceFiles(source_file_filter)
333 return InputApi._RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000334
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000335 def ReadFile(self, file, mode='r'):
336 """Reads an arbitrary file.
337
338 Deny reading anything outside the repository.
339 """
340 if isinstance(file, AffectedFile):
341 file = file.AbsoluteLocalPath()
342 if not file.startswith(self.change.RepositoryRoot()):
343 raise IOError('Access outside the repository root is denied.')
344 return gcl.ReadFile(file, mode)
345
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000346 @staticmethod
347 def _RightHandSideLinesImpl(affected_files):
348 """Implements RightHandSideLines for InputApi and GclChange."""
349 for af in affected_files:
350 lines = af.NewContents()
351 line_number = 0
352 for line in lines:
353 line_number += 1
354 yield (af, line_number, line)
355
356
357class AffectedFile(object):
358 """Representation of a file in a change."""
359
360 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000361 self._path = path
362 self._action = action
363 self._repository_root = repository_root
364 self._is_directory = None
365 self._properties = {}
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000366 self.scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000367
368 def ServerPath(self):
369 """Returns a path string that identifies the file in the SCM system.
370
371 Returns the empty string if the file does not exist in SCM.
372 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000373 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000374
375 def LocalPath(self):
376 """Returns the path of this file on the local disk relative to client root.
377 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000378 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000379
380 def AbsoluteLocalPath(self):
381 """Returns the absolute path of this file on the local disk.
382 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000383 return normpath(os.path.join(self._repository_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000384
385 def IsDirectory(self):
386 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000387 if self._is_directory is None:
388 path = self.AbsoluteLocalPath()
389 self._is_directory = (os.path.exists(path) and
390 os.path.isdir(path))
391 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000392
393 def Action(self):
394 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000395 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
396 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000397 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000398
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000399 def Property(self, property_name):
400 """Returns the specified SCM property of this file, or None if no such
401 property.
402 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000403 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000404
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000405 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000406 """Returns True if the file is a text file and not a binary file.
407
408 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000409 raise NotImplementedError() # Implement when needed
410
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000411 def NewContents(self):
412 """Returns an iterator over the lines in the new version of file.
413
414 The new version is the file in the user's workspace, i.e. the "right hand
415 side".
416
417 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000418 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000419 """
420 if self.IsDirectory():
421 return []
422 else:
423 return gcl.ReadFile(self.AbsoluteLocalPath()).splitlines()
424
425 def OldContents(self):
426 """Returns an iterator over the lines in the old version of file.
427
428 The old version is the file in depot, i.e. the "left hand side".
429 """
430 raise NotImplementedError() # Implement when needed
431
432 def OldFileTempPath(self):
433 """Returns the path on local disk where the old contents resides.
434
435 The old version is the file in depot, i.e. the "left hand side".
436 This is a read-only cached copy of the old contents. *DO NOT* try to
437 modify this file.
438 """
439 raise NotImplementedError() # Implement if/when needed.
440
maruel@chromium.org5de13972009-06-10 18:16:06 +0000441 def __str__(self):
442 return self.LocalPath()
443
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000444
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000445class SvnAffectedFile(AffectedFile):
446 """Representation of a file in a change out of a Subversion checkout."""
447
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000448 def __init__(self, *args, **kwargs):
449 AffectedFile.__init__(self, *args, **kwargs)
450 self._server_path = None
451 self._is_text_file = None
maruel@chromium.orgb7d46902009-06-10 14:12:10 +0000452 self.scm = 'svn'
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000453
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000454 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000455 if self._server_path is None:
456 self._server_path = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000457 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000458 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000459
460 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000461 if self._is_directory is None:
462 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000463 if os.path.exists(path):
464 # Retrieve directly from the file system; it is much faster than
465 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000466 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000467 else:
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000468 self._is_directory = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000469 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000470 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000471
472 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000473 if not property_name in self._properties:
474 self._properties[property_name] = gcl.GetSVNFileProperty(
maruel@chromium.org196f8cb2009-06-11 00:32:06 +0000475 self.AbsoluteLocalPath(), property_name).rstrip()
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000476 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000477
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000478 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000479 if self._is_text_file is None:
480 if self.Action() == 'D':
481 # A deleted file is not a text file.
482 self._is_text_file = False
483 elif self.IsDirectory():
484 self._is_text_file = False
485 else:
486 mime_type = gcl.GetSVNFileProperty(self.AbsoluteLocalPath(),
487 'svn:mime-type')
488 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
489 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000490
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000491
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000492class GclChange(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000493 """Describe a change.
494
495 Used directly by the presubmit scripts to query the current change being
496 tested.
497
498 Instance members:
499 tags: Dictionnary of KEY=VALUE pairs found in the change description.
500 self.KEY: equivalent to tags['KEY']
501 """
502
503 # Matches key/value (or "tag") lines in changelist descriptions.
504 _tag_line_re = re.compile(
505 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000506
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000507 def __init__(self, change_info):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000508 # Do not keep a reference to the original change_info.
509 self._name = change_info.name
510 self._full_description = change_info.description
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000511 self._repository_root = change_info.GetLocalRoot()
maruel@chromium.org32ba2602009-06-06 18:44:48 +0000512 self.issue = change_info.issue
513 self.patchset = change_info.patchset
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000514
515 # From the description text, build up a dictionary of key/value pairs
516 # plus the description minus all key/value or "tag" lines.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000517 self._description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000518 self.tags = {}
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000519 for line in self._full_description.splitlines():
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000520 m = self._tag_line_re.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000521 if m:
522 self.tags[m.group('key')] = m.group('value')
523 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000524 self._description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000525
526 # Change back to text and remove whitespace at end.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000527 self._description_without_tags = '\n'.join(self._description_without_tags)
528 self._description_without_tags = self._description_without_tags.rstrip()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000529
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000530 self._affected_files = [
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000531 SvnAffectedFile(info[1], info[0].strip(), self._repository_root)
532 for info in change_info.GetFiles()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000533 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000534
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000535 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000536 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000537 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000538
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000539 def DescriptionText(self):
540 """Returns the user-entered changelist description, minus tags.
541
542 Any line in the user-provided description starting with e.g. "FOO="
543 (whitespace permitted before and around) is considered a tag line. Such
544 lines are stripped out of the description this function returns.
545 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000546 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000547
548 def FullDescriptionText(self):
549 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000550 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000551
552 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000553 """Returns the repository (checkout) root directory for this change,
554 as an absolute path.
555 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000556 return self._repository_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000557
558 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000559 """Return tags directly as attributes on the object."""
560 if not re.match(r"^[A-Z_]*$", attr):
561 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000562 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000563
564 def AffectedFiles(self, include_dirs=False, include_deletes=True):
565 """Returns a list of AffectedFile instances for all files in the change.
566
567 Args:
568 include_deletes: If false, deleted files will be filtered out.
569 include_dirs: True to include directories in the list
570
571 Returns:
572 [AffectedFile(path, action), AffectedFile(path, action)]
573 """
574 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000575 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000576 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000577 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000578
579 if include_deletes:
580 return affected
581 else:
582 return filter(lambda x: x.Action() != 'D', affected)
583
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000584 def AffectedTextFiles(self, include_deletes=None):
585 """Return a list of the existing text files in a change."""
586 if include_deletes is not None:
587 warnings.warn("AffectedTextFiles(include_deletes=%s)"
588 " is deprecated and ignored" % str(include_deletes),
589 category=DeprecationWarning,
590 stacklevel=2)
591 return filter(lambda x: x.IsTextFile(),
592 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000593
594 def LocalPaths(self, include_dirs=False):
595 """Convenience function."""
596 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
597
598 def AbsoluteLocalPaths(self, include_dirs=False):
599 """Convenience function."""
600 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
601
602 def ServerPaths(self, include_dirs=False):
603 """Convenience function."""
604 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
605
606 def RightHandSideLines(self):
607 """An iterator over all text lines in "new" version of changed files.
608
609 Lists lines from new or modified text files in the change.
610
611 This is useful for doing line-by-line regex checks, like checking for
612 trailing whitespace.
613
614 Yields:
615 a 3 tuple:
616 the AffectedFile instance of the current file;
617 integer line number (1-based); and
618 the contents of the line as a string.
619 """
620 return InputApi._RightHandSideLinesImpl(
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000621 filter(lambda x: x.IsTextFile(),
622 self.AffectedFiles(include_deletes=False)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000623
624
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000625def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626 """Finds all presubmit files that apply to a given set of source files.
627
628 Args:
629 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000630 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000631
632 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000633 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000634 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000635 entries = []
636 for f in files:
637 f = normpath(os.path.join(root, f))
638 while f:
639 f = os.path.dirname(f)
640 if f in entries:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000641 break
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000642 entries.append(f)
643 if f == root:
644 break
645 entries.sort()
646 entries = map(lambda x: os.path.join(x, 'PRESUBMIT.py'), entries)
647 return filter(lambda x: os.path.isfile(x), entries)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000648
649
650class PresubmitExecuter(object):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000651 def __init__(self, change_info, committing):
652 """
653 Args:
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000654 change_info: The gcl.ChangeInfo object for the change.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000655 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
656 """
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000657 # TODO(maruel): Determine the SCM.
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000658 self.change = GclChange(change_info)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000659 self.committing = committing
660
661 def ExecPresubmitScript(self, script_text, presubmit_path):
662 """Executes a single presubmit script.
663
664 Args:
665 script_text: The text of the presubmit script.
666 presubmit_path: The path to the presubmit file (this will be reported via
667 input_api.PresubmitLocalPath()).
668
669 Return:
670 A list of result objects, empty if no problems.
671 """
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000672 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000673 context = {}
674 exec script_text in context
675
676 # These function names must change if we make substantial changes to
677 # the presubmit API that are not backwards compatible.
678 if self.committing:
679 function_name = 'CheckChangeOnCommit'
680 else:
681 function_name = 'CheckChangeOnUpload'
682 if function_name in context:
683 context['__args'] = (input_api, OutputApi())
684 result = eval(function_name + '(*__args)', context)
685 if not (isinstance(result, types.TupleType) or
686 isinstance(result, types.ListType)):
687 raise exceptions.RuntimeError(
688 'Presubmit functions must return a tuple or list')
689 for item in result:
690 if not isinstance(item, OutputApi.PresubmitResult):
691 raise exceptions.RuntimeError(
692 'All presubmit results must be of types derived from '
693 'output_api.PresubmitResult')
694 else:
695 result = () # no error since the script doesn't care about current event.
696
697 return result
698
699
700def DoPresubmitChecks(change_info,
701 committing,
702 verbose,
703 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000704 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000705 default_presubmit,
706 may_prompt):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000707 """Runs all presubmit checks that apply to the files in the change.
708
709 This finds all PRESUBMIT.py files in directories enclosing the files in the
710 change (up to the repository root) and calls the relevant entrypoint function
711 depending on whether the change is being committed or uploaded.
712
713 Prints errors, warnings and notifications. Prompts the user for warnings
714 when needed.
715
716 Args:
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000717 change_info: The gcl.ChangeInfo object for the change.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000718 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
719 verbose: Prints debug info.
720 output_stream: A stream to write output from presubmit tests to.
721 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000722 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000723 may_prompt: Enable (y/n) questions on warning or error.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000724
725 Return:
726 True if execution can continue, False if not.
727 """
maruel@chromium.org17f59f22009-06-12 13:27:24 +0000728 presubmit_files = ListRelevantPresubmitFiles(change_info.GetFileNames(),
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000729 change_info.GetLocalRoot())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000730 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000731 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000732 results = []
733 executer = PresubmitExecuter(change_info, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000734 if default_presubmit:
735 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000736 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000737 fake_path = os.path.join(change_info.GetLocalRoot(), 'PRESUBMIT.py')
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000738 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000739 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000740 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000741 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000742 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000743 # Accept CRLF presubmit script.
maruel@chromium.org277003e2009-05-01 12:51:43 +0000744 presubmit_script = gcl.ReadFile(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000745 results += executer.ExecPresubmitScript(presubmit_script, filename)
746
747 errors = []
748 notifications = []
749 warnings = []
750 for result in results:
751 if not result.IsFatal() and not result.ShouldPrompt():
752 notifications.append(result)
753 elif result.ShouldPrompt():
754 warnings.append(result)
755 else:
756 errors.append(result)
757
758 error_count = 0
759 for name, items in (('Messages', notifications),
760 ('Warnings', warnings),
761 ('ERRORS', errors)):
762 if items:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000763 output_stream.write('** Presubmit %s **\n' % name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000764 for item in items:
765 if not item._Handle(output_stream, input_stream,
766 may_prompt=False):
767 error_count += 1
768 output_stream.write('\n')
maruel@chromium.org07bbc212009-06-11 02:08:41 +0000769 if not errors and warnings and may_prompt:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000770 output_stream.write(
771 'There were presubmit warnings. Sure you want to continue? (y/N): ')
772 response = input_stream.readline()
773 if response.strip().lower() != 'y':
774 error_count += 1
775 return (error_count == 0)
776
777
778def ScanSubDirs(mask, recursive):
779 if not recursive:
780 return [x for x in glob.glob(mask) if '.svn' not in x]
781 else:
782 results = []
783 for root, dirs, files in os.walk('.'):
784 if '.svn' in dirs:
785 dirs.remove('.svn')
786 for name in files:
787 if fnmatch.fnmatch(name, mask):
788 results.append(os.path.join(root, name))
789 return results
790
791
792def ParseFiles(args, recursive):
793 files = []
794 for arg in args:
795 files.extend([('M', file) for file in ScanSubDirs(arg, recursive)])
796 return files
797
798
799def Main(argv):
800 parser = optparse.OptionParser(usage="%prog [options]",
801 version="%prog " + str(__version__))
802 parser.add_option("-c", "--commit", action="store_true",
803 help="Use commit instead of upload checks")
804 parser.add_option("-r", "--recursive", action="store_true",
805 help="Act recursively")
806 parser.add_option("-v", "--verbose", action="store_true",
807 help="Verbose output")
808 options, args = parser.parse_args(argv[1:])
809 files = ParseFiles(args, options.recursive)
810 if options.verbose:
811 print "Found %d files." % len(files)
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000812 return not DoPresubmitChecks(gcl.ChangeInfo('No name', 0, 0, '', files,
813 gcl.GetRepositoryRoot()),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000814 options.commit,
815 options.verbose,
816 sys.stdout,
817 sys.stdin,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000818 None,
819 False)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000820
821
822if __name__ == '__main__':
823 sys.exit(Main(sys.argv))