blob: 79bc306bdcb306be12afca6929d25e0b548e4177 [file] [log] [blame]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001#!/usr/bin/python
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00002# Copyright (c) 2010 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00009__version__ = '1.3.5'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
15import cPickle # Exposed through the API.
16import cStringIO # Exposed through the API.
17import exceptions
18import fnmatch
19import glob
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000020import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000021import marshal # Exposed through the API.
22import optparse
23import os # Somewhat exposed through the API.
24import pickle # Exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000025import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000026import re # Exposed through the API.
27import subprocess # Exposed through the API.
28import sys # Parts exposed through API.
29import tempfile # Exposed through the API.
jam@chromium.org2a891dc2009-08-20 20:33:37 +000030import time
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000031import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000032import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000033import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000034import urllib2 # Exposed through the API.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000035from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000036
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +000037try:
38 import simplejson as json
39except ImportError:
40 try:
41 import json
tony@chromium.org5e9f2ed2010-04-08 01:38:19 +000042 # Some versions of python2.5 have an incomplete json module. Check to make
43 # sure loads exists.
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +000044 # Statement seems to have no effect
45 # pylint: disable=W0104
tony@chromium.org5e9f2ed2010-04-08 01:38:19 +000046 json.loads
47 except (ImportError, AttributeError):
maruel@chromium.org59c7ba62010-03-20 00:13:07 +000048 # Import the one included in depot_tools.
maruel@chromium.orgd08cb1e2010-03-20 00:25:19 +000049 sys.path.append(os.path.join(os.path.dirname(__file__), 'third_party'))
50 import simplejson as json
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +000051
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000052# Local imports.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000053import gclient_utils
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000054import presubmit_canned_checks
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000055import scm
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000056
57
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000058# Ask for feedback only once in program lifetime.
59_ASKED_FOR_FEEDBACK = False
60
61
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000062class NotImplementedException(Exception):
63 """We're leaving placeholders in a bunch of places to remind us of the
64 design of the API, but we have not implemented all of it yet. Implement as
65 the need arises.
66 """
67 pass
68
69
70def normpath(path):
71 '''Version of os.path.normpath that also changes backward slashes to
72 forward slashes when not running on Windows.
73 '''
74 # This is safe to always do because the Windows version of os.path.normpath
75 # will replace forward slashes with backward slashes.
76 path = path.replace(os.sep, '/')
77 return os.path.normpath(path)
78
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000079
gspencer@google.comefb94502009-10-09 17:57:08 +000080def PromptYesNo(input_stream, output_stream, prompt):
81 output_stream.write(prompt)
82 response = input_stream.readline().strip().lower()
83 return response == 'y' or response == 'yes'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000084
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000085
86def _RightHandSideLinesImpl(affected_files):
87 """Implements RightHandSideLines for InputApi and GclChange."""
88 for af in affected_files:
89 lines = af.NewContents()
90 line_number = 0
91 for line in lines:
92 line_number += 1
93 yield (af, line_number, line)
94
95
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000096class OutputApi(object):
97 """This class (more like a module) gets passed to presubmit scripts so that
98 they can specify various types of results.
99 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000100 # Method could be a function
101 # pylint: disable=R0201
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000102 class PresubmitResult(object):
103 """Base class for result objects."""
104
105 def __init__(self, message, items=None, long_text=''):
106 """
107 message: A short one-line message to indicate errors.
108 items: A list of short strings to indicate where errors occurred.
109 long_text: multi-line text output, e.g. from another tool
110 """
111 self._message = message
112 self._items = []
113 if items:
114 self._items = items
115 self._long_text = long_text.rstrip()
116
117 def _Handle(self, output_stream, input_stream, may_prompt=True):
118 """Writes this result to the output stream.
119
120 Args:
121 output_stream: Where to write
122
123 Returns:
124 True if execution may continue, False otherwise.
125 """
126 output_stream.write(self._message)
127 output_stream.write('\n')
estade@chromium.org593258b2010-01-28 00:04:36 +0000128 if len(self._items) > 0:
129 output_stream.write(' ' + ' \\\n '.join(map(str, self._items)) + '\n')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000130 if self._long_text:
maruel@chromium.org310b3552010-11-01 13:23:35 +0000131 # Sometimes self._long_text is a ascii string, a codepage string
132 # (on windows), or a unicode object.
133 try:
134 long_text = self._long_text.decode()
135 except UnicodeDecodeError:
136 long_text = self._long_text.decode('ascii', 'replace')
137
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000138 output_stream.write('\n***************\n%s\n***************\n' %
maruel@chromium.org310b3552010-11-01 13:23:35 +0000139 long_text)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000140
141 if self.ShouldPrompt() and may_prompt:
gspencer@google.comefb94502009-10-09 17:57:08 +0000142 if not PromptYesNo(input_stream, output_stream,
143 'Are you sure you want to continue? (y/N): '):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000144 return False
145
146 return not self.IsFatal()
147
148 def IsFatal(self):
149 """An error that is fatal stops g4 mail/submit immediately, i.e. before
150 other presubmit scripts are run.
151 """
152 return False
153
154 def ShouldPrompt(self):
155 """Whether this presubmit result should result in a prompt warning."""
156 return False
157
158 class PresubmitError(PresubmitResult):
159 """A hard presubmit error."""
160 def IsFatal(self):
161 return True
162
163 class PresubmitPromptWarning(PresubmitResult):
164 """An warning that prompts the user if they want to continue."""
165 def ShouldPrompt(self):
166 return True
167
168 class PresubmitNotifyResult(PresubmitResult):
169 """Just print something to the screen -- but it's not even a warning."""
170 pass
171
172 class MailTextResult(PresubmitResult):
173 """A warning that should be included in the review request email."""
174 def __init__(self, *args, **kwargs):
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000175 super(OutputApi.MailTextResult, self).__init__()
176 raise NotImplementedException()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000177
178
179class InputApi(object):
180 """An instance of this object is passed to presubmit scripts so they can
181 know stuff about the change they're looking at.
182 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000183 # Method could be a function
184 # pylint: disable=R0201
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000185
maruel@chromium.org3410d912009-06-09 20:56:16 +0000186 # File extensions that are considered source files from a style guide
187 # perspective. Don't modify this list from a presubmit script!
188 DEFAULT_WHITE_LIST = (
189 # C++ and friends
maruel@chromium.orga73d7932010-01-28 23:50:06 +0000190 r".*\.c$", r".*\.cc$", r".*\.cpp$", r".*\.h$", r".*\.m$", r".*\.mm$",
191 r".*\.inl$", r".*\.asm$", r".*\.hxx$", r".*\.hpp$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000192 # Scripts
maruel@chromium.orga73d7932010-01-28 23:50:06 +0000193 r".*\.js$", r".*\.py$", r".*\.sh$", r".*\.rb$", r".*\.pl$", r".*\.pm$",
maruel@chromium.orgef776482010-01-28 16:04:32 +0000194 # No extension at all, note that ALL CAPS files are black listed in
195 # DEFAULT_BLACK_LIST below.
maruel@chromium.orga73d7932010-01-28 23:50:06 +0000196 r"(^|.*?[\\\/])[^.]+$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000197 # Other
maruel@chromium.orga73d7932010-01-28 23:50:06 +0000198 r".*\.java$", r".*\.mk$", r".*\.am$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000199 )
200
201 # Path regexp that should be excluded from being considered containing source
202 # files. Don't modify this list from a presubmit script!
203 DEFAULT_BLACK_LIST = (
204 r".*\bexperimental[\\\/].*",
205 r".*\bthird_party[\\\/].*",
206 # Output directories (just in case)
207 r".*\bDebug[\\\/].*",
208 r".*\bRelease[\\\/].*",
209 r".*\bxcodebuild[\\\/].*",
210 r".*\bsconsbuild[\\\/].*",
211 # All caps files like README and LICENCE.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000212 r".*\b[A-Z0-9_]+$",
213 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000214 r"(|.*[\\\/])\.git[\\\/].*",
215 r"(|.*[\\\/])\.svn[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000216 )
217
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000218 def __init__(self, change, presubmit_path, is_committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000219 """Builds an InputApi object.
220
221 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000222 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000223 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000224 is_committing: True if the change is about to be committed.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000225 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000226 # Version number of the presubmit_support script.
227 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000228 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000229 self.is_committing = is_committing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000230
231 # We expose various modules and functions as attributes of the input_api
232 # so that presubmit scripts don't have to import them.
233 self.basename = os.path.basename
234 self.cPickle = cPickle
235 self.cStringIO = cStringIO
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000236 self.json = json
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000237 self.os_path = os.path
238 self.pickle = pickle
239 self.marshal = marshal
240 self.re = re
241 self.subprocess = subprocess
242 self.tempfile = tempfile
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000243 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000244 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000245 self.urllib2 = urllib2
246
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000247 # To easily fork python.
248 self.python_executable = sys.executable
249 self.environ = os.environ
250
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000251 # InputApi.platform is the platform you're currently running on.
252 self.platform = sys.platform
253
254 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000255 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000256
257 # We carry the canned checks so presubmit scripts can easily use them.
258 self.canned_checks = presubmit_canned_checks
259
260 def PresubmitLocalPath(self):
261 """Returns the local path of the presubmit script currently being run.
262
263 This is useful if you don't want to hard-code absolute paths in the
264 presubmit script. For example, It can be used to find another file
265 relative to the PRESUBMIT.py script, so the whole tree can be branched and
266 the presubmit script still works, without editing its content.
267 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000268 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000269
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000270 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000271 """Translate a depot path to a local path (relative to client root).
272
273 Args:
274 Depot path as a string.
275
276 Returns:
277 The local path of the depot path under the user's current client, or None
278 if the file is not mapped.
279
280 Remember to check for the None case and show an appropriate error!
281 """
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000282 local_path = scm.SVN.CaptureInfo(depot_path).get('Path')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000283 if local_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284 return local_path
285
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000286 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000287 """Translate a local path to a depot path.
288
289 Args:
290 Local path (relative to current directory, or absolute) as a string.
291
292 Returns:
293 The depot path (SVN URL) of the file if mapped, otherwise None.
294 """
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000295 depot_path = scm.SVN.CaptureInfo(local_path).get('URL')
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000296 if depot_path:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000297 return depot_path
298
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000299 def AffectedFiles(self, include_dirs=False, include_deletes=True):
300 """Same as input_api.change.AffectedFiles() except only lists files
301 (and optionally directories) in the same directory as the current presubmit
302 script, or subdirectories thereof.
303 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000304 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000305 if len(dir_with_slash) == 1:
306 dir_with_slash = ''
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000307 return filter(
308 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
309 self.change.AffectedFiles(include_dirs, include_deletes))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000310
311 def LocalPaths(self, include_dirs=False):
312 """Returns local paths of input_api.AffectedFiles()."""
313 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
314
315 def AbsoluteLocalPaths(self, include_dirs=False):
316 """Returns absolute local paths of input_api.AffectedFiles()."""
317 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
318
319 def ServerPaths(self, include_dirs=False):
320 """Returns server paths of input_api.AffectedFiles()."""
321 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
322
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000323 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000324 """Same as input_api.change.AffectedTextFiles() except only lists files
325 in the same directory as the current presubmit script, or subdirectories
326 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000327 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000328 if include_deletes is not None:
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000329 warn("AffectedTextFiles(include_deletes=%s)"
330 " is deprecated and ignored" % str(include_deletes),
331 category=DeprecationWarning,
332 stacklevel=2)
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000333 return filter(lambda x: x.IsTextFile(),
334 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000335
maruel@chromium.org3410d912009-06-09 20:56:16 +0000336 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
337 """Filters out files that aren't considered "source file".
338
339 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
340 and InputApi.DEFAULT_BLACK_LIST is used respectively.
341
342 The lists will be compiled as regular expression and
343 AffectedFile.LocalPath() needs to pass both list.
344
345 Note: Copy-paste this function to suit your needs or use a lambda function.
346 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000347 def Find(affected_file, items):
348 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000349 local_path = affected_file.LocalPath()
350 if self.re.match(item, local_path):
351 logging.debug("%s matched %s" % (item, local_path))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000352 return True
353 return False
354 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
355 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
356
357 def AffectedSourceFiles(self, source_file):
358 """Filter the list of AffectedTextFiles by the function source_file.
359
360 If source_file is None, InputApi.FilterSourceFile() is used.
361 """
362 if not source_file:
363 source_file = self.FilterSourceFile
364 return filter(source_file, self.AffectedTextFiles())
365
366 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000367 """An iterator over all text lines in "new" version of changed files.
368
369 Only lists lines from new or modified text files in the change that are
370 contained by the directory of the currently executing presubmit script.
371
372 This is useful for doing line-by-line regex checks, like checking for
373 trailing whitespace.
374
375 Yields:
376 a 3 tuple:
377 the AffectedFile instance of the current file;
378 integer line number (1-based); and
379 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000380
381 Note: The cariage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000382 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000383 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000384 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000385
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000386 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000387 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000388
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000389 Deny reading anything outside the repository.
390 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000391 if isinstance(file_item, AffectedFile):
392 file_item = file_item.AbsoluteLocalPath()
393 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000394 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000395 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000396
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000397
398class AffectedFile(object):
399 """Representation of a file in a change."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000400 # Method could be a function
401 # pylint: disable=R0201
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000402 def __init__(self, path, action, repository_root=''):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000403 self._path = path
404 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000405 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000406 self._is_directory = None
407 self._properties = {}
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000408 logging.debug('%s(%s)' % (self.__class__.__name__, self._path))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000409
410 def ServerPath(self):
411 """Returns a path string that identifies the file in the SCM system.
412
413 Returns the empty string if the file does not exist in SCM.
414 """
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000415 return ""
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000416
417 def LocalPath(self):
418 """Returns the path of this file on the local disk relative to client root.
419 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000420 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000421
422 def AbsoluteLocalPath(self):
423 """Returns the absolute path of this file on the local disk.
424 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000425 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000426
427 def IsDirectory(self):
428 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000429 if self._is_directory is None:
430 path = self.AbsoluteLocalPath()
431 self._is_directory = (os.path.exists(path) and
432 os.path.isdir(path))
433 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000434
435 def Action(self):
436 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000437 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
438 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000439 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000440
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000441 def Property(self, property_name):
442 """Returns the specified SCM property of this file, or None if no such
443 property.
444 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000445 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000446
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000447 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000448 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000449
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000450 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000451 raise NotImplementedError() # Implement when needed
452
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000453 def NewContents(self):
454 """Returns an iterator over the lines in the new version of file.
455
456 The new version is the file in the user's workspace, i.e. the "right hand
457 side".
458
459 Contents will be empty if the file is a directory or does not exist.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000460 Note: The cariage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000461 """
462 if self.IsDirectory():
463 return []
464 else:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000465 return gclient_utils.FileRead(self.AbsoluteLocalPath(),
466 'rU').splitlines()
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000467
468 def OldContents(self):
469 """Returns an iterator over the lines in the old version of file.
470
471 The old version is the file in depot, i.e. the "left hand side".
472 """
473 raise NotImplementedError() # Implement when needed
474
475 def OldFileTempPath(self):
476 """Returns the path on local disk where the old contents resides.
477
478 The old version is the file in depot, i.e. the "left hand side".
479 This is a read-only cached copy of the old contents. *DO NOT* try to
480 modify this file.
481 """
482 raise NotImplementedError() # Implement if/when needed.
483
maruel@chromium.org5de13972009-06-10 18:16:06 +0000484 def __str__(self):
485 return self.LocalPath()
486
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000487
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000488class SvnAffectedFile(AffectedFile):
489 """Representation of a file in a change out of a Subversion checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000490 # Method 'NNN' is abstract in class 'NNN' but is not overridden
491 # pylint: disable=W0223
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000492
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000493 def __init__(self, *args, **kwargs):
494 AffectedFile.__init__(self, *args, **kwargs)
495 self._server_path = None
496 self._is_text_file = None
497
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000498 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000499 if self._server_path is None:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000500 self._server_path = scm.SVN.CaptureInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000501 self.AbsoluteLocalPath()).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000502 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000503
504 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000505 if self._is_directory is None:
506 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000507 if os.path.exists(path):
508 # Retrieve directly from the file system; it is much faster than
509 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000510 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000511 else:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000512 self._is_directory = scm.SVN.CaptureInfo(
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000513 path).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000514 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000515
516 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000517 if not property_name in self._properties:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000518 self._properties[property_name] = scm.SVN.GetFileProperty(
maruel@chromium.org196f8cb2009-06-11 00:32:06 +0000519 self.AbsoluteLocalPath(), property_name).rstrip()
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000520 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000521
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000522 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000523 if self._is_text_file is None:
524 if self.Action() == 'D':
525 # A deleted file is not a text file.
526 self._is_text_file = False
527 elif self.IsDirectory():
528 self._is_text_file = False
529 else:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000530 mime_type = scm.SVN.GetFileProperty(self.AbsoluteLocalPath(),
531 'svn:mime-type')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000532 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
533 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000534
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000535
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000536class GitAffectedFile(AffectedFile):
537 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000538 # Method 'NNN' is abstract in class 'NNN' but is not overridden
539 # pylint: disable=W0223
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000540
541 def __init__(self, *args, **kwargs):
542 AffectedFile.__init__(self, *args, **kwargs)
543 self._server_path = None
544 self._is_text_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000545
546 def ServerPath(self):
547 if self._server_path is None:
548 raise NotImplementedException() # TODO(maruel) Implement.
549 return self._server_path
550
551 def IsDirectory(self):
552 if self._is_directory is None:
553 path = self.AbsoluteLocalPath()
554 if os.path.exists(path):
555 # Retrieve directly from the file system; it is much faster than
556 # querying subversion, especially on Windows.
557 self._is_directory = os.path.isdir(path)
558 else:
559 # raise NotImplementedException() # TODO(maruel) Implement.
560 self._is_directory = False
561 return self._is_directory
562
563 def Property(self, property_name):
564 if not property_name in self._properties:
565 raise NotImplementedException() # TODO(maruel) Implement.
566 return self._properties[property_name]
567
568 def IsTextFile(self):
569 if self._is_text_file is None:
570 if self.Action() == 'D':
571 # A deleted file is not a text file.
572 self._is_text_file = False
573 elif self.IsDirectory():
574 self._is_text_file = False
575 else:
576 # raise NotImplementedException() # TODO(maruel) Implement.
577 self._is_text_file = os.path.isfile(self.AbsoluteLocalPath())
578 return self._is_text_file
579
580
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000581class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000582 """Describe a change.
583
584 Used directly by the presubmit scripts to query the current change being
585 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000586
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000587 Instance members:
588 tags: Dictionnary of KEY=VALUE pairs found in the change description.
589 self.KEY: equivalent to tags['KEY']
590 """
591
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000592 _AFFECTED_FILES = AffectedFile
593
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000594 # Matches key/value (or "tag") lines in changelist descriptions.
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000595 _TAG_LINE_RE = re.compile(
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000596 '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000597
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000598 def __init__(self, name, description, local_root, files, issue, patchset):
599 if files is None:
600 files = []
601 self._name = name
602 self._full_description = description
chase@chromium.org8e416c82009-10-06 04:30:44 +0000603 # Convert root into an absolute path.
604 self._local_root = os.path.abspath(local_root)
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000605 self.issue = issue
606 self.patchset = patchset
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000607 self.scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000608
609 # From the description text, build up a dictionary of key/value pairs
610 # plus the description minus all key/value or "tag" lines.
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000611 description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000612 self.tags = {}
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000613 for line in self._full_description.splitlines():
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000614 m = self._TAG_LINE_RE.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000615 if m:
616 self.tags[m.group('key')] = m.group('value')
617 else:
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000618 description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000619
620 # Change back to text and remove whitespace at end.
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000621 self._description_without_tags = (
622 '\n'.join(description_without_tags).rstrip())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000623
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000624 self._affected_files = [
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000625 self._AFFECTED_FILES(info[1], info[0].strip(), self._local_root)
626 for info in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000627 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000628
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000629 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000630 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000631 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000632
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000633 def DescriptionText(self):
634 """Returns the user-entered changelist description, minus tags.
635
636 Any line in the user-provided description starting with e.g. "FOO="
637 (whitespace permitted before and around) is considered a tag line. Such
638 lines are stripped out of the description this function returns.
639 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000640 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000641
642 def FullDescriptionText(self):
643 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000644 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000645
646 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000647 """Returns the repository (checkout) root directory for this change,
648 as an absolute path.
649 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000650 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000651
652 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000653 """Return tags directly as attributes on the object."""
654 if not re.match(r"^[A-Z_]*$", attr):
655 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000656 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000657
658 def AffectedFiles(self, include_dirs=False, include_deletes=True):
659 """Returns a list of AffectedFile instances for all files in the change.
660
661 Args:
662 include_deletes: If false, deleted files will be filtered out.
663 include_dirs: True to include directories in the list
664
665 Returns:
666 [AffectedFile(path, action), AffectedFile(path, action)]
667 """
668 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000669 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000670 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000671 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000672
673 if include_deletes:
674 return affected
675 else:
676 return filter(lambda x: x.Action() != 'D', affected)
677
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000678 def AffectedTextFiles(self, include_deletes=None):
679 """Return a list of the existing text files in a change."""
680 if include_deletes is not None:
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000681 warn("AffectedTextFiles(include_deletes=%s)"
682 " is deprecated and ignored" % str(include_deletes),
683 category=DeprecationWarning,
684 stacklevel=2)
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000685 return filter(lambda x: x.IsTextFile(),
686 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000687
688 def LocalPaths(self, include_dirs=False):
689 """Convenience function."""
690 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
691
692 def AbsoluteLocalPaths(self, include_dirs=False):
693 """Convenience function."""
694 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
695
696 def ServerPaths(self, include_dirs=False):
697 """Convenience function."""
698 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
699
700 def RightHandSideLines(self):
701 """An iterator over all text lines in "new" version of changed files.
702
703 Lists lines from new or modified text files in the change.
704
705 This is useful for doing line-by-line regex checks, like checking for
706 trailing whitespace.
707
708 Yields:
709 a 3 tuple:
710 the AffectedFile instance of the current file;
711 integer line number (1-based); and
712 the contents of the line as a string.
713 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000714 return _RightHandSideLinesImpl(
715 x for x in self.AffectedFiles(include_deletes=False)
716 if x.IsTextFile())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000717
718
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000719class SvnChange(Change):
720 _AFFECTED_FILES = SvnAffectedFile
721
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000722 def __init__(self, *args, **kwargs):
723 Change.__init__(self, *args, **kwargs)
724 self.scm = 'svn'
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000725 self._changelists = None
726
727 def _GetChangeLists(self):
728 """Get all change lists."""
729 if self._changelists == None:
730 previous_cwd = os.getcwd()
731 os.chdir(self.RepositoryRoot())
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000732 # Need to import here to avoid circular dependency.
733 import gcl
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000734 self._changelists = gcl.GetModifiedFiles()
735 os.chdir(previous_cwd)
736 return self._changelists
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000737
738 def GetAllModifiedFiles(self):
739 """Get all modified files."""
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000740 changelists = self._GetChangeLists()
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000741 all_modified_files = []
742 for cl in changelists.values():
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000743 all_modified_files.extend(
744 [os.path.join(self.RepositoryRoot(), f[1]) for f in cl])
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000745 return all_modified_files
746
747 def GetModifiedFiles(self):
748 """Get modified files in the current CL."""
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000749 changelists = self._GetChangeLists()
750 return [os.path.join(self.RepositoryRoot(), f[1])
751 for f in changelists[self.Name()]]
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000752
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000753
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000754class GitChange(Change):
755 _AFFECTED_FILES = GitAffectedFile
756
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000757 def __init__(self, *args, **kwargs):
758 Change.__init__(self, *args, **kwargs)
759 self.scm = 'git'
760
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000761
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000762def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000763 """Finds all presubmit files that apply to a given set of source files.
764
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000765 If inherit-review-settings-ok is present right under root, looks for
766 PRESUBMIT.py in directories enclosing root.
767
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000768 Args:
769 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000770 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000771
772 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000773 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000774 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000775 files = [normpath(os.path.join(root, f)) for f in files]
776
777 # List all the individual directories containing files.
778 directories = set([os.path.dirname(f) for f in files])
779
780 # Ignore root if inherit-review-settings-ok is present.
781 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
782 root = None
783
784 # Collect all unique directories that may contain PRESUBMIT.py.
785 candidates = set()
786 for directory in directories:
787 while True:
788 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000789 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000790 candidates.add(directory)
791 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000792 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000793 parent_dir = os.path.dirname(directory)
794 if parent_dir == directory:
795 # We hit the system root directory.
796 break
797 directory = parent_dir
798
799 # Look for PRESUBMIT.py in all candidate directories.
800 results = []
801 for directory in sorted(list(candidates)):
802 p = os.path.join(directory, 'PRESUBMIT.py')
803 if os.path.isfile(p):
804 results.append(p)
805
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000806 logging.debug('Presubmit files: %s' % ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000807 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000808
809
thestig@chromium.orgde243452009-10-06 21:02:56 +0000810class GetTrySlavesExecuter(object):
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000811 @staticmethod
812 def ExecPresubmitScript(script_text):
thestig@chromium.orgde243452009-10-06 21:02:56 +0000813 """Executes GetPreferredTrySlaves() from a single presubmit script.
814
815 Args:
816 script_text: The text of the presubmit script.
817
818 Return:
819 A list of try slaves.
820 """
821 context = {}
822 exec script_text in context
823
824 function_name = 'GetPreferredTrySlaves'
825 if function_name in context:
826 result = eval(function_name + '()', context)
827 if not isinstance(result, types.ListType):
828 raise exceptions.RuntimeError(
829 'Presubmit functions must return a list, got a %s instead: %s' %
830 (type(result), str(result)))
831 for item in result:
832 if not isinstance(item, basestring):
833 raise exceptions.RuntimeError('All try slaves names must be strings.')
834 if item != item.strip():
835 raise exceptions.RuntimeError('Try slave names cannot start/end'
836 'with whitespace')
837 else:
838 result = []
839 return result
840
841
842def DoGetTrySlaves(changed_files,
843 repository_root,
844 default_presubmit,
845 verbose,
846 output_stream):
847 """Get the list of try servers from the presubmit scripts.
848
849 Args:
850 changed_files: List of modified files.
851 repository_root: The repository root.
852 default_presubmit: A default presubmit script to execute in any case.
853 verbose: Prints debug info.
854 output_stream: A stream to write debug output to.
855
856 Return:
857 List of try slaves
858 """
859 presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
860 if not presubmit_files and verbose:
861 output_stream.write("Warning, no presubmit.py found.\n")
862 results = []
863 executer = GetTrySlavesExecuter()
864 if default_presubmit:
865 if verbose:
866 output_stream.write("Running default presubmit script.\n")
867 results += executer.ExecPresubmitScript(default_presubmit)
868 for filename in presubmit_files:
869 filename = os.path.abspath(filename)
870 if verbose:
871 output_stream.write("Running %s\n" % filename)
872 # Accept CRLF presubmit script.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000873 presubmit_script = gclient_utils.FileRead(filename, 'rU')
thestig@chromium.orgde243452009-10-06 21:02:56 +0000874 results += executer.ExecPresubmitScript(presubmit_script)
875
876 slaves = list(set(results))
877 if slaves and verbose:
878 output_stream.write(', '.join(slaves))
879 output_stream.write('\n')
880 return slaves
881
882
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000883class PresubmitExecuter(object):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000884 def __init__(self, change, committing):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000885 """
886 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000887 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000888 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
889 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000890 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000891 self.committing = committing
892
893 def ExecPresubmitScript(self, script_text, presubmit_path):
894 """Executes a single presubmit script.
895
896 Args:
897 script_text: The text of the presubmit script.
898 presubmit_path: The path to the presubmit file (this will be reported via
899 input_api.PresubmitLocalPath()).
900
901 Return:
902 A list of result objects, empty if no problems.
903 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000904
905 # Change to the presubmit file's directory to support local imports.
906 main_path = os.getcwd()
907 os.chdir(os.path.dirname(presubmit_path))
908
909 # Load the presubmit script into context.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000910 input_api = InputApi(self.change, presubmit_path, self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000911 context = {}
912 exec script_text in context
913
914 # These function names must change if we make substantial changes to
915 # the presubmit API that are not backwards compatible.
916 if self.committing:
917 function_name = 'CheckChangeOnCommit'
918 else:
919 function_name = 'CheckChangeOnUpload'
920 if function_name in context:
921 context['__args'] = (input_api, OutputApi())
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000922 logging.debug('Running %s in %s' % (function_name, presubmit_path))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000923 result = eval(function_name + '(*__args)', context)
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000924 logging.debug('Running %s done.' % function_name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000925 if not (isinstance(result, types.TupleType) or
926 isinstance(result, types.ListType)):
927 raise exceptions.RuntimeError(
928 'Presubmit functions must return a tuple or list')
929 for item in result:
930 if not isinstance(item, OutputApi.PresubmitResult):
931 raise exceptions.RuntimeError(
932 'All presubmit results must be of types derived from '
933 'output_api.PresubmitResult')
934 else:
935 result = () # no error since the script doesn't care about current event.
936
chase@chromium.org8e416c82009-10-06 04:30:44 +0000937 # Return the process to the original working directory.
938 os.chdir(main_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000939 return result
940
941
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000942def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000943 committing,
944 verbose,
945 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000946 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000947 default_presubmit,
948 may_prompt):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000949 """Runs all presubmit checks that apply to the files in the change.
950
951 This finds all PRESUBMIT.py files in directories enclosing the files in the
952 change (up to the repository root) and calls the relevant entrypoint function
953 depending on whether the change is being committed or uploaded.
954
955 Prints errors, warnings and notifications. Prompts the user for warnings
956 when needed.
957
958 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000959 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000960 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
961 verbose: Prints debug info.
962 output_stream: A stream to write output from presubmit tests to.
963 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000964 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +0000965 may_prompt: Enable (y/n) questions on warning or error.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000966
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +0000967 Warning:
968 If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream
969 SHOULD be sys.stdin.
970
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000971 Return:
972 True if execution can continue, False if not.
973 """
maruel@chromium.org8d195232010-10-05 12:58:49 +0000974 print "Running presubmit hooks..."
jam@chromium.org2a891dc2009-08-20 20:33:37 +0000975 start_time = time.time()
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000976 presubmit_files = ListRelevantPresubmitFiles(change.AbsoluteLocalPaths(True),
977 change.RepositoryRoot())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000978 if not presubmit_files and verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000979 output_stream.write("Warning, no presubmit.py found.\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000980 results = []
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000981 executer = PresubmitExecuter(change, committing)
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +0000982 if default_presubmit:
983 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000984 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000985 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000986 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000987 for filename in presubmit_files:
maruel@chromium.org3d235242009-05-15 12:40:48 +0000988 filename = os.path.abspath(filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000989 if verbose:
maruel@chromium.orgf3eee562009-05-27 00:51:10 +0000990 output_stream.write("Running %s\n" % filename)
maruel@chromium.orgc1675e22009-04-27 20:30:48 +0000991 # Accept CRLF presubmit script.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000992 presubmit_script = gclient_utils.FileRead(filename, 'rU')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000993 results += executer.ExecPresubmitScript(presubmit_script, filename)
994
995 errors = []
996 notifications = []
997 warnings = []
998 for result in results:
999 if not result.IsFatal() and not result.ShouldPrompt():
1000 notifications.append(result)
1001 elif result.ShouldPrompt():
1002 warnings.append(result)
1003 else:
1004 errors.append(result)
1005
1006 error_count = 0
1007 for name, items in (('Messages', notifications),
1008 ('Warnings', warnings),
1009 ('ERRORS', errors)):
1010 if items:
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001011 output_stream.write('** Presubmit %s **\n' % name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001012 for item in items:
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +00001013 # Access to a protected member XXX of a client class
1014 # pylint: disable=W0212
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001015 if not item._Handle(output_stream, input_stream,
1016 may_prompt=False):
1017 error_count += 1
1018 output_stream.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001019
1020 total_time = time.time() - start_time
1021 if total_time > 1.0:
1022 print "Presubmit checks took %.1fs to calculate." % total_time
1023
maruel@chromium.org07bbc212009-06-11 02:08:41 +00001024 if not errors and warnings and may_prompt:
gspencer@google.comefb94502009-10-09 17:57:08 +00001025 if not PromptYesNo(input_stream, output_stream,
1026 'There were presubmit warnings. '
1027 'Are you sure you wish to continue? (y/N): '):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001028 error_count += 1
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001029
1030 global _ASKED_FOR_FEEDBACK
1031 # Ask for feedback one time out of 5.
1032 if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
1033 output_stream.write("Was the presubmit check useful? Please send feedback "
1034 "& hate mail to maruel@chromium.org!\n")
1035 _ASKED_FOR_FEEDBACK = True
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001036 return (error_count == 0)
1037
1038
1039def ScanSubDirs(mask, recursive):
1040 if not recursive:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001041 return [x for x in glob.glob(mask) if '.svn' not in x and '.git' not in x]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001042 else:
1043 results = []
1044 for root, dirs, files in os.walk('.'):
1045 if '.svn' in dirs:
1046 dirs.remove('.svn')
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001047 if '.git' in dirs:
1048 dirs.remove('.git')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001049 for name in files:
1050 if fnmatch.fnmatch(name, mask):
1051 results.append(os.path.join(root, name))
1052 return results
1053
1054
1055def ParseFiles(args, recursive):
1056 files = []
1057 for arg in args:
maruel@chromium.orge3608df2009-11-10 20:22:57 +00001058 files.extend([('M', f) for f in ScanSubDirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001059 return files
1060
1061
1062def Main(argv):
1063 parser = optparse.OptionParser(usage="%prog [options]",
1064 version="%prog " + str(__version__))
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001065 parser.add_option("-c", "--commit", action="store_true", default=False,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001066 help="Use commit instead of upload checks")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001067 parser.add_option("-u", "--upload", action="store_false", dest='commit',
1068 help="Use upload instead of commit checks")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001069 parser.add_option("-r", "--recursive", action="store_true",
1070 help="Act recursively")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001071 parser.add_option("-v", "--verbose", action="store_true", default=False,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001072 help="Verbose output")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001073 parser.add_option("--files")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001074 parser.add_option("--name", default='no name')
1075 parser.add_option("--description", default='')
1076 parser.add_option("--issue", type='int', default=0)
1077 parser.add_option("--patchset", type='int', default=0)
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001078 parser.add_option("--root", default=os.getcwd(),
1079 help="Search for PRESUBMIT.py up to this directory. "
1080 "If inherit-review-settings-ok is present in this "
1081 "directory, parent directories up to the root file "
1082 "system directories will also be searched.")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001083 parser.add_option("--default_presubmit")
1084 parser.add_option("--may_prompt", action='store_true', default=False)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001085 options, args = parser.parse_args(argv[1:])
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001086 if os.path.isdir(os.path.join(options.root, '.git')):
1087 change_class = GitChange
1088 if not options.files:
1089 if args:
1090 options.files = ParseFiles(args, options.recursive)
1091 else:
1092 # Grab modified files.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00001093 options.files = scm.GIT.CaptureStatus([options.root])
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001094 elif os.path.isdir(os.path.join(options.root, '.svn')):
1095 change_class = SvnChange
1096 if not options.files:
1097 if args:
1098 options.files = ParseFiles(args, options.recursive)
1099 else:
1100 # Grab modified files.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00001101 options.files = scm.SVN.CaptureStatus([options.root])
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001102 else:
1103 # Doesn't seem under source control.
1104 change_class = Change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001105 if options.verbose:
chase@chromium.org8e416c82009-10-06 04:30:44 +00001106 if len(options.files) != 1:
1107 print "Found %d files." % len(options.files)
1108 else:
1109 print "Found 1 file."
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001110 return not DoPresubmitChecks(change_class(options.name,
1111 options.description,
1112 options.root,
1113 options.files,
1114 options.issue,
1115 options.patchset),
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001116 options.commit,
1117 options.verbose,
1118 sys.stdout,
1119 sys.stdin,
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001120 options.default_presubmit,
1121 options.may_prompt)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001122
1123
1124if __name__ == '__main__':
1125 sys.exit(Main(sys.argv))