blob: ccbca74052398e92d18aa41795cafdcf84ced66b [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
2# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.org44a17ad2009-06-08 14:14:35 +00009__version__ = '1.3.1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
20import marshal # Exposed through the API.
21import optparse
22import os # Somewhat exposed through the API.
23import pickle # Exposed through the API.
24import re # Exposed through the API.
25import subprocess # Exposed through the API.
26import sys # Parts exposed through API.
27import tempfile # Exposed through the API.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000028import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000029import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000030import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000031import urllib2 # Exposed through the API.
maruel@chromium.org1e08c002009-05-28 19:09:33 +000032import warnings
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033
34# Local imports.
35# TODO(joi) Would be cleaner to factor out utils in gcl to separate module, but
36# for now it would only be a couple of functions so hardly worth it.
37import gcl
maruel@chromium.org46a94102009-05-12 20:32:43 +000038import gclient
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000039import presubmit_canned_checks
40
41
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000042class NotImplementedException(Exception):
43 """We're leaving placeholders in a bunch of places to remind us of the
44 design of the API, but we have not implemented all of it yet. Implement as
45 the need arises.
46 """
47 pass
48
49
50def normpath(path):
51 '''Version of os.path.normpath that also changes backward slashes to
52 forward slashes when not running on Windows.
53 '''
54 # This is safe to always do because the Windows version of os.path.normpath
55 # will replace forward slashes with backward slashes.
56 path = path.replace(os.sep, '/')
57 return os.path.normpath(path)
58
59
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000060class OutputApi(object):
61 """This class (more like a module) gets passed to presubmit scripts so that
62 they can specify various types of results.
63 """
64
65 class PresubmitResult(object):
66 """Base class for result objects."""
67
68 def __init__(self, message, items=None, long_text=''):
69 """
70 message: A short one-line message to indicate errors.
71 items: A list of short strings to indicate where errors occurred.
72 long_text: multi-line text output, e.g. from another tool
73 """
74 self._message = message
75 self._items = []
76 if items:
77 self._items = items
78 self._long_text = long_text.rstrip()
79
80 def _Handle(self, output_stream, input_stream, may_prompt=True):
81 """Writes this result to the output stream.
82
83 Args:
84 output_stream: Where to write
85
86 Returns:
87 True if execution may continue, False otherwise.
88 """
89 output_stream.write(self._message)
90 output_stream.write('\n')
91 for item in self._items:
92 output_stream.write(' %s\n' % item)
93 if self._long_text:
94 output_stream.write('\n***************\n%s\n***************\n\n' %
95 self._long_text)
96
97 if self.ShouldPrompt() and may_prompt:
98 output_stream.write('Are you sure you want to continue? (y/N): ')
99 response = input_stream.readline()
100 if response.strip().lower() != 'y':
101 return False
102
103 return not self.IsFatal()
104
105 def IsFatal(self):
106 """An error that is fatal stops g4 mail/submit immediately, i.e. before
107 other presubmit scripts are run.
108 """
109 return False
110
111 def ShouldPrompt(self):
112 """Whether this presubmit result should result in a prompt warning."""
113 return False
114
115 class PresubmitError(PresubmitResult):
116 """A hard presubmit error."""
117 def IsFatal(self):
118 return True
119
120 class PresubmitPromptWarning(PresubmitResult):
121 """An warning that prompts the user if they want to continue."""
122 def ShouldPrompt(self):
123 return True
124
125 class PresubmitNotifyResult(PresubmitResult):
126 """Just print something to the screen -- but it's not even a warning."""
127 pass
128
129 class MailTextResult(PresubmitResult):
130 """A warning that should be included in the review request email."""
131 def __init__(self, *args, **kwargs):
132 raise NotImplementedException() # TODO(joi) Implement.
133
134
135class InputApi(object):
136 """An instance of this object is passed to presubmit scripts so they can
137 know stuff about the change they're looking at.
138 """
139
maruel@chromium.org3410d912009-06-09 20:56:16 +0000140 # File extensions that are considered source files from a style guide
141 # perspective. Don't modify this list from a presubmit script!
142 DEFAULT_WHITE_LIST = (
143 # C++ and friends
144 r".*\.c", r".*\.cc", r".*\.cpp", r".*\.h", r".*\.m", r".*\.mm",
145 r".*\.inl", r".*\.asm", r".*\.hxx", r".*\.hpp",
146 # Scripts
147 r".*\.js", r".*\.py", r".*\.json", r".*\.sh", r".*\.rb",
148 # No extension at all
149 r"(^|.*[\\\/])[^.]+$",
150 # Other
151 r".*\.java", r".*\.mk", r".*\.am",
152 )
153
154 # Path regexp that should be excluded from being considered containing source
155 # files. Don't modify this list from a presubmit script!
156 DEFAULT_BLACK_LIST = (
157 r".*\bexperimental[\\\/].*",
158 r".*\bthird_party[\\\/].*",
159 # Output directories (just in case)
160 r".*\bDebug[\\\/].*",
161 r".*\bRelease[\\\/].*",
162 r".*\bxcodebuild[\\\/].*",
163 r".*\bsconsbuild[\\\/].*",
164 # All caps files like README and LICENCE.
165 r".*\b[A-Z0-9_]+",
166 # SCM (can happen in dual SCM configuration)
167 r".*\b\.git[\\\/].*",
168 r".*\b\.svn[\\\/].*",
169 )
170
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000171 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000172 """Builds an InputApi object.
173
174 Args:
175 change: A presubmit.GclChange object.
176 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000177 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000178 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000179 # Version number of the presubmit_support script.
180 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000181 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000182 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000183
184 # We expose various modules and functions as attributes of the input_api
185 # so that presubmit scripts don't have to import them.
186 self.basename = os.path.basename
187 self.cPickle = cPickle
188 self.cStringIO = cStringIO
189 self.os_path = os.path
190 self.pickle = pickle
191 self.marshal = marshal
192 self.re = re
193 self.subprocess = subprocess
194 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000195 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000196 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000197 self.urllib2 = urllib2
198
199 # InputApi.platform is the platform you're currently running on.
200 self.platform = sys.platform
201
202 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000203 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000204
205 # We carry the canned checks so presubmit scripts can easily use them.
206 self.canned_checks = presubmit_canned_checks
207
208 def PresubmitLocalPath(self):
209 """Returns the local path of the presubmit script currently being run.
210
211 This is useful if you don't want to hard-code absolute paths in the
212 presubmit script. For example, It can be used to find another file
213 relative to the PRESUBMIT.py script, so the whole tree can be branched and
214 the presubmit script still works, without editing its content.
215 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000216 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000217
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000218 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000219 """Translate a depot path to a local path (relative to client root).
220
221 Args:
222 Depot path as a string.
223
224 Returns:
225 The local path of the depot path under the user's current client, or None
226 if the file is not mapped.
227
228 Remember to check for the None case and show an appropriate error!
229 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000230 local_path = gclient.CaptureSVNInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000231 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000232 return local_path
233
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000234 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000235 """Translate a local path to a depot path.
236
237 Args:
238 Local path (relative to current directory, or absolute) as a string.
239
240 Returns:
241 The depot path (SVN URL) of the file if mapped, otherwise None.
242 """
maruel@chromium.org46a94102009-05-12 20:32:43 +0000243 depot_path = gclient.CaptureSVNInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000244 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000245 return depot_path
246
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000247 def AffectedFiles(self, include_dirs=False, include_deletes=True):
248 """Same as input_api.change.AffectedFiles() except only lists files
249 (and optionally directories) in the same directory as the current presubmit
250 script, or subdirectories thereof.
251 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000252 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000253 if len(dir_with_slash) == 1:
254 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000255 return filter(
256 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
257 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000258
259 def LocalPaths(self, include_dirs=False):
260 """Returns local paths of input_api.AffectedFiles()."""
261 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
262
263 def AbsoluteLocalPaths(self, include_dirs=False):
264 """Returns absolute local paths of input_api.AffectedFiles()."""
265 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
266
267 def ServerPaths(self, include_dirs=False):
268 """Returns server paths of input_api.AffectedFiles()."""
269 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
270
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000271 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000272 """Same as input_api.change.AffectedTextFiles() except only lists files
273 in the same directory as the current presubmit script, or subdirectories
274 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000275 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000276 if include_deletes is not None:
277 warnings.warn("AffectedTextFiles(include_deletes=%s)"
278 " is deprecated and ignored" % str(include_deletes),
279 category=DeprecationWarning,
280 stacklevel=2)
281 return filter(lambda x: x.IsTextFile(),
282 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000283
maruel@chromium.org3410d912009-06-09 20:56:16 +0000284 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
285 """Filters out files that aren't considered "source file".
286
287 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
288 and InputApi.DEFAULT_BLACK_LIST is used respectively.
289
290 The lists will be compiled as regular expression and
291 AffectedFile.LocalPath() needs to pass both list.
292
293 Note: Copy-paste this function to suit your needs or use a lambda function.
294 """
295 def Find(affected_file, list):
296 for item in list:
297 if self.re.match(item, affected_file.LocalPath()):
298 return True
299 return False
300 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
301 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
302
303 def AffectedSourceFiles(self, source_file):
304 """Filter the list of AffectedTextFiles by the function source_file.
305
306 If source_file is None, InputApi.FilterSourceFile() is used.
307 """
308 if not source_file:
309 source_file = self.FilterSourceFile
310 return filter(source_file, self.AffectedTextFiles())
311
312 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000313 """An iterator over all text lines in "new" version of changed files.
314
315 Only lists lines from new or modified text files in the change that are
316 contained by the directory of the currently executing presubmit script.
317
318 This is useful for doing line-by-line regex checks, like checking for
319 trailing whitespace.
320
321 Yields:
322 a 3 tuple:
323 the AffectedFile instance of the current file;
324 integer line number (1-based); and
325 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000326
327 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000328 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000329 files = self.AffectedSourceFiles(source_file_filter)
330 return InputApi._RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000331
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000332 def ReadFile(self, file, mode='r'):
333 """Reads an arbitrary file.
334
335 Deny reading anything outside the repository.
336 """
337 if isinstance(file, AffectedFile):
338 file = file.AbsoluteLocalPath()
339 if not file.startswith(self.change.RepositoryRoot()):
340 raise IOError('Access outside the repository root is denied.')
341 return gcl.ReadFile(file, mode)
342
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000343 @staticmethod
344 def _RightHandSideLinesImpl(affected_files):
345 """Implements RightHandSideLines for InputApi and GclChange."""
346 for af in affected_files:
347 lines = af.NewContents()
348 line_number = 0
349 for line in lines:
350 line_number += 1
351 yield (af, line_number, line)
352
353
354class AffectedFile(object):
355 """Representation of a file in a change."""
356
357 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000358 self._path = path
359 self._action = action
360 self._repository_root = repository_root
361 self._is_directory = None
362 self._properties = {}
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000363
364 def ServerPath(self):
365 """Returns a path string that identifies the file in the SCM system.
366
367 Returns the empty string if the file does not exist in SCM.
368 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000369 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000370
371 def LocalPath(self):
372 """Returns the path of this file on the local disk relative to client root.
373 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000374 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000375
376 def AbsoluteLocalPath(self):
377 """Returns the absolute path of this file on the local disk.
378 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000379 return normpath(os.path.join(self._repository_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000380
381 def IsDirectory(self):
382 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000383 if self._is_directory is None:
384 path = self.AbsoluteLocalPath()
385 self._is_directory = (os.path.exists(path) and
386 os.path.isdir(path))
387 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000388
389 def Action(self):
390 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000391 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
392 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000393 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000394
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000395 def Property(self, property_name):
396 """Returns the specified SCM property of this file, or None if no such
397 property.
398 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000399 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000400
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000401 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000402 """Returns True if the file is a text file and not a binary file.
403
404 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000405 raise NotImplementedError() # Implement when needed
406
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000407 def NewContents(self):
408 """Returns an iterator over the lines in the new version of file.
409
410 The new version is the file in the user's workspace, i.e. the "right hand
411 side".
412
413 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000414 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000415 """
416 if self.IsDirectory():
417 return []
418 else:
419 return gcl.ReadFile(self.AbsoluteLocalPath()).splitlines()
420
421 def OldContents(self):
422 """Returns an iterator over the lines in the old version of file.
423
424 The old version is the file in depot, i.e. the "left hand side".
425 """
426 raise NotImplementedError() # Implement when needed
427
428 def OldFileTempPath(self):
429 """Returns the path on local disk where the old contents resides.
430
431 The old version is the file in depot, i.e. the "left hand side".
432 This is a read-only cached copy of the old contents. *DO NOT* try to
433 modify this file.
434 """
435 raise NotImplementedError() # Implement if/when needed.
436
437
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000438class SvnAffectedFile(AffectedFile):
439 """Representation of a file in a change out of a Subversion checkout."""
440
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000441 def __init__(self, *args, **kwargs):
442 AffectedFile.__init__(self, *args, **kwargs)
443 self._server_path = None
444 self._is_text_file = None
445
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000446 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000447 if self._server_path is None:
448 self._server_path = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000449 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000450 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000451
452 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000453 if self._is_directory is None:
454 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000455 if os.path.exists(path):
456 # Retrieve directly from the file system; it is much faster than
457 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000458 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000459 else:
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000460 self._is_directory = gclient.CaptureSVNInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000461 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000462 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000463
464 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000465 if not property_name in self._properties:
466 self._properties[property_name] = gcl.GetSVNFileProperty(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000467 self.AbsoluteLocalPath(), property_name)
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000468 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000469
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000470 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000471 if self._is_text_file is None:
472 if self.Action() == 'D':
473 # A deleted file is not a text file.
474 self._is_text_file = False
475 elif self.IsDirectory():
476 self._is_text_file = False
477 else:
478 mime_type = gcl.GetSVNFileProperty(self.AbsoluteLocalPath(),
479 'svn:mime-type')
480 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
481 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000482
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000483
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000484class GclChange(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000485 """Describe a change.
486
487 Used directly by the presubmit scripts to query the current change being
488 tested.
489
490 Instance members:
491 tags: Dictionnary of KEY=VALUE pairs found in the change description.
492 self.KEY: equivalent to tags['KEY']
493 """
494
495 # Matches key/value (or "tag") lines in changelist descriptions.
496 _tag_line_re = re.compile(
497 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000498
499 def __init__(self, change_info, repository_root=''):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000500 # Do not keep a reference to the original change_info.
501 self._name = change_info.name
502 self._full_description = change_info.description
503 self._repository_root = repository_root
maruel@chromium.org32ba2602009-06-06 18:44:48 +0000504 self.issue = change_info.issue
505 self.patchset = change_info.patchset
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000506
507 # From the description text, build up a dictionary of key/value pairs
508 # plus the description minus all key/value or "tag" lines.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000509 self._description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000510 self.tags = {}
511 for line in change_info.description.splitlines():
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000512 m = self._tag_line_re.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000513 if m:
514 self.tags[m.group('key')] = m.group('value')
515 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000516 self._description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000517
518 # Change back to text and remove whitespace at end.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000519 self._description_without_tags = '\n'.join(self._description_without_tags)
520 self._description_without_tags = self._description_without_tags.rstrip()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000521
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000522 self._affected_files = [
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000523 SvnAffectedFile(info[1], info[0].strip(), repository_root)
524 for info in change_info.files
525 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000526
527 def Change(self):
528 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000529 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000530
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000531 def DescriptionText(self):
532 """Returns the user-entered changelist description, minus tags.
533
534 Any line in the user-provided description starting with e.g. "FOO="
535 (whitespace permitted before and around) is considered a tag line. Such
536 lines are stripped out of the description this function returns.
537 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000538 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000539
540 def FullDescriptionText(self):
541 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000542 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000543
544 def RepositoryRoot(self):
545 """Returns the repository root for this change, as an absolute path."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000546 return self._repository_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000547
548 def __getattr__(self, attr):
549 """Return keys directly as attributes on the object.
550
551 You may use a friendly name (from SPECIAL_KEYS) or the actual name of
552 the key.
553 """
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000554 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000555
556 def AffectedFiles(self, include_dirs=False, include_deletes=True):
557 """Returns a list of AffectedFile instances for all files in the change.
558
559 Args:
560 include_deletes: If false, deleted files will be filtered out.
561 include_dirs: True to include directories in the list
562
563 Returns:
564 [AffectedFile(path, action), AffectedFile(path, action)]
565 """
566 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000567 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000568 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000569 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000570
571 if include_deletes:
572 return affected
573 else:
574 return filter(lambda x: x.Action() != 'D', affected)
575
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000576 def AffectedTextFiles(self, include_deletes=None):
577 """Return a list of the existing text files in a change."""
578 if include_deletes is not None:
579 warnings.warn("AffectedTextFiles(include_deletes=%s)"
580 " is deprecated and ignored" % str(include_deletes),
581 category=DeprecationWarning,
582 stacklevel=2)
583 return filter(lambda x: x.IsTextFile(),
584 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000585
586 def LocalPaths(self, include_dirs=False):
587 """Convenience function."""
588 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
589
590 def AbsoluteLocalPaths(self, include_dirs=False):
591 """Convenience function."""
592 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
593
594 def ServerPaths(self, include_dirs=False):
595 """Convenience function."""
596 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
597
598 def RightHandSideLines(self):
599 """An iterator over all text lines in "new" version of changed files.
600
601 Lists lines from new or modified text files in the change.
602
603 This is useful for doing line-by-line regex checks, like checking for
604 trailing whitespace.
605
606 Yields:
607 a 3 tuple:
608 the AffectedFile instance of the current file;
609 integer line number (1-based); and
610 the contents of the line as a string.
611 """
612 return InputApi._RightHandSideLinesImpl(
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000613 filter(lambda x: x.IsTextFile(),
614 self.AffectedFiles(include_deletes=False)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000615
616
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000617def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000618 """Finds all presubmit files that apply to a given set of source files.
619
620 Args:
621 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000622 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000623
624 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000625 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000627 entries = []
628 for f in files:
629 f = normpath(os.path.join(root, f))
630 while f:
631 f = os.path.dirname(f)
632 if f in entries:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000633 break
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000634 entries.append(f)
635 if f == root:
636 break
637 entries.sort()
638 entries = map(lambda x: os.path.join(x, 'PRESUBMIT.py'), entries)
639 return filter(lambda x: os.path.isfile(x), entries)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000640
641
642class PresubmitExecuter(object):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000643 def __init__(self, change_info, committing):
644 """
645 Args:
646 change_info: The ChangeInfo object for the change.
647 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
648 """
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000649 # TODO(maruel): Determine the SCM.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000650 self.change = GclChange(change_info, gcl.GetRepositoryRoot())
651 self.committing = committing
652
653 def ExecPresubmitScript(self, script_text, presubmit_path):
654 """Executes a single presubmit script.
655
656 Args:
657 script_text: The text of the presubmit script.
658 presubmit_path: The path to the presubmit file (this will be reported via
659 input_api.PresubmitLocalPath()).
660
661 Return:
662 A list of result objects, empty if no problems.
663 """
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000664 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000665 context = {}
666 exec script_text in context
667
668 # These function names must change if we make substantial changes to
669 # the presubmit API that are not backwards compatible.
670 if self.committing:
671 function_name = 'CheckChangeOnCommit'
672 else:
673 function_name = 'CheckChangeOnUpload'
674 if function_name in context:
675 context['__args'] = (input_api, OutputApi())
676 result = eval(function_name + '(*__args)', context)
677 if not (isinstance(result, types.TupleType) or
678 isinstance(result, types.ListType)):
679 raise exceptions.RuntimeError(
680 'Presubmit functions must return a tuple or list')
681 for item in result:
682 if not isinstance(item, OutputApi.PresubmitResult):
683 raise exceptions.RuntimeError(
684 'All presubmit results must be of types derived from '
685 'output_api.PresubmitResult')
686 else:
687 result = () # no error since the script doesn't care about current event.
688
689 return result
690
691
692def DoPresubmitChecks(change_info,
693 committing,
694 verbose,
695 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000696 input_stream,
697 default_presubmit):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000698 """Runs all presubmit checks that apply to the files in the change.
699
700 This finds all PRESUBMIT.py files in directories enclosing the files in the
701 change (up to the repository root) and calls the relevant entrypoint function
702 depending on whether the change is being committed or uploaded.
703
704 Prints errors, warnings and notifications. Prompts the user for warnings
705 when needed.
706
707 Args:
708 change_info: The ChangeInfo object for the change.
709 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
710 verbose: Prints debug info.
711 output_stream: A stream to write output from presubmit tests to.
712 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000713 default_presubmit: A default presubmit script to execute in any case.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000714
715 Return:
716 True if execution can continue, False if not.
717 """
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000718 checkout_root = gcl.GetRepositoryRoot()
719 presubmit_files = ListRelevantPresubmitFiles(change_info.FileList(),
720 checkout_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000721 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000722 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000723 results = []
724 executer = PresubmitExecuter(change_info, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000725 if default_presubmit:
726 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000727 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000728 fake_path = os.path.join(checkout_root, 'PRESUBMIT.py')
729 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000730 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000731 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000732 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000733 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000734 # Accept CRLF presubmit script.
maruel@chromium.org277003e2009-05-01 12:51:43 +0000735 presubmit_script = gcl.ReadFile(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000736 results += executer.ExecPresubmitScript(presubmit_script, filename)
737
738 errors = []
739 notifications = []
740 warnings = []
741 for result in results:
742 if not result.IsFatal() and not result.ShouldPrompt():
743 notifications.append(result)
744 elif result.ShouldPrompt():
745 warnings.append(result)
746 else:
747 errors.append(result)
748
749 error_count = 0
750 for name, items in (('Messages', notifications),
751 ('Warnings', warnings),
752 ('ERRORS', errors)):
753 if items:
754 output_stream.write('\n** Presubmit %s **\n\n' % name)
755 for item in items:
756 if not item._Handle(output_stream, input_stream,
757 may_prompt=False):
758 error_count += 1
759 output_stream.write('\n')
760 if not errors and warnings:
761 output_stream.write(
762 'There were presubmit warnings. Sure you want to continue? (y/N): ')
763 response = input_stream.readline()
764 if response.strip().lower() != 'y':
765 error_count += 1
766 return (error_count == 0)
767
768
769def ScanSubDirs(mask, recursive):
770 if not recursive:
771 return [x for x in glob.glob(mask) if '.svn' not in x]
772 else:
773 results = []
774 for root, dirs, files in os.walk('.'):
775 if '.svn' in dirs:
776 dirs.remove('.svn')
777 for name in files:
778 if fnmatch.fnmatch(name, mask):
779 results.append(os.path.join(root, name))
780 return results
781
782
783def ParseFiles(args, recursive):
784 files = []
785 for arg in args:
786 files.extend([('M', file) for file in ScanSubDirs(arg, recursive)])
787 return files
788
789
790def Main(argv):
791 parser = optparse.OptionParser(usage="%prog [options]",
792 version="%prog " + str(__version__))
793 parser.add_option("-c", "--commit", action="store_true",
794 help="Use commit instead of upload checks")
795 parser.add_option("-r", "--recursive", action="store_true",
796 help="Act recursively")
797 parser.add_option("-v", "--verbose", action="store_true",
798 help="Verbose output")
799 options, args = parser.parse_args(argv[1:])
800 files = ParseFiles(args, options.recursive)
801 if options.verbose:
802 print "Found %d files." % len(files)
maruel@chromium.orgde0ba292009-06-06 19:43:27 +0000803 return not DoPresubmitChecks(gcl.ChangeInfo('No name', 0, 0, '', files),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000804 options.commit,
805 options.verbose,
806 sys.stdout,
807 sys.stdin,
808 default_presubmit=None)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000809
810
811if __name__ == '__main__':
812 sys.exit(Main(sys.argv))