blob: 6f796b6bf11369c2797cf4b20111bf22034e282d [file] [log] [blame]
maruel@chromium.org725f1c32011-04-01 20:24:54 +00001#!/usr/bin/env python
maruel@chromium.org3bbf2942012-01-10 16:52:06 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
enne@chromium.orge72c5f52013-04-16 00:36:40 +00009__version__ = '1.6.2'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000010
11# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
12# caching (between all different invocations of presubmit scripts for a given
13# change). We should add it as our presubmit scripts start feeling slow.
14
enne@chromium.orge72c5f52013-04-16 00:36:40 +000015import cpplint
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000016import cPickle # Exposed through the API.
17import cStringIO # Exposed through the API.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000018import collections
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +000019import contextlib
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000020import fnmatch
21import glob
asvitkine@chromium.org15169952011-09-27 14:30:53 +000022import inspect
maruel@chromium.org4f6852c2012-04-20 20:39:20 +000023import json # Exposed through the API.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000024import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000025import marshal # Exposed through the API.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000026import multiprocessing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000027import optparse
28import os # Somewhat exposed through the API.
29import pickle # Exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000030import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000031import re # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000032import sys # Parts exposed through API.
33import tempfile # Exposed through the API.
jam@chromium.org2a891dc2009-08-20 20:33:37 +000034import time
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000035import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000036import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000037import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000038import urllib2 # Exposed through the API.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000039from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040
41# Local imports.
maruel@chromium.org35625c72011-03-23 17:34:02 +000042import fix_encoding
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000043import gclient_utils
dpranke@chromium.org2a009622011-03-01 02:43:31 +000044import owners
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000045import presubmit_canned_checks
maruel@chromium.org239f4112011-06-03 20:08:23 +000046import rietveld
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000047import scm
maruel@chromium.org84f4fe32011-04-06 13:26:45 +000048import subprocess2 as subprocess # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000049
50
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000051# Ask for feedback only once in program lifetime.
52_ASKED_FOR_FEEDBACK = False
53
54
maruel@chromium.org899e1c12011-04-07 17:03:18 +000055class PresubmitFailure(Exception):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000056 pass
57
58
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000059CommandData = collections.namedtuple('CommandData',
60 ['name', 'cmd', 'kwargs', 'message'])
61
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000062def normpath(path):
63 '''Version of os.path.normpath that also changes backward slashes to
64 forward slashes when not running on Windows.
65 '''
66 # This is safe to always do because the Windows version of os.path.normpath
67 # will replace forward slashes with backward slashes.
68 path = path.replace(os.sep, '/')
69 return os.path.normpath(path)
70
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000071
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000072def _RightHandSideLinesImpl(affected_files):
73 """Implements RightHandSideLines for InputApi and GclChange."""
74 for af in affected_files:
maruel@chromium.orgab05d582011-02-09 23:41:22 +000075 lines = af.ChangedContents()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000076 for line in lines:
maruel@chromium.orgab05d582011-02-09 23:41:22 +000077 yield (af, line[0], line[1])
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000078
79
dpranke@chromium.org5ac21012011-03-16 02:58:25 +000080class PresubmitOutput(object):
81 def __init__(self, input_stream=None, output_stream=None):
82 self.input_stream = input_stream
83 self.output_stream = output_stream
84 self.reviewers = []
85 self.written_output = []
86 self.error_count = 0
87
88 def prompt_yes_no(self, prompt_string):
89 self.write(prompt_string)
90 if self.input_stream:
91 response = self.input_stream.readline().strip().lower()
92 if response not in ('y', 'yes'):
93 self.fail()
94 else:
95 self.fail()
96
97 def fail(self):
98 self.error_count += 1
99
100 def should_continue(self):
101 return not self.error_count
102
103 def write(self, s):
104 self.written_output.append(s)
105 if self.output_stream:
106 self.output_stream.write(s)
107
108 def getvalue(self):
109 return ''.join(self.written_output)
110
111
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000112# Top level object so multiprocessing can pickle
113# Public access through OutputApi object.
114class _PresubmitResult(object):
115 """Base class for result objects."""
116 fatal = False
117 should_prompt = False
118
119 def __init__(self, message, items=None, long_text=''):
120 """
121 message: A short one-line message to indicate errors.
122 items: A list of short strings to indicate where errors occurred.
123 long_text: multi-line text output, e.g. from another tool
124 """
125 self._message = message
126 self._items = items or []
127 if items:
128 self._items = items
129 self._long_text = long_text.rstrip()
130
131 def handle(self, output):
132 output.write(self._message)
133 output.write('\n')
134 for index, item in enumerate(self._items):
135 output.write(' ')
136 # Write separately in case it's unicode.
137 output.write(str(item))
138 if index < len(self._items) - 1:
139 output.write(' \\')
140 output.write('\n')
141 if self._long_text:
142 output.write('\n***************\n')
143 # Write separately in case it's unicode.
144 output.write(self._long_text)
145 output.write('\n***************\n')
146 if self.fatal:
147 output.fail()
148
149
150# Top level object so multiprocessing can pickle
151# Public access through OutputApi object.
152class _PresubmitAddReviewers(_PresubmitResult):
153 """Add some suggested reviewers to the change."""
154 def __init__(self, reviewers):
155 super(_PresubmitAddReviewers, self).__init__('')
156 self.reviewers = reviewers
157
158 def handle(self, output):
159 output.reviewers.extend(self.reviewers)
160
161
162# Top level object so multiprocessing can pickle
163# Public access through OutputApi object.
164class _PresubmitError(_PresubmitResult):
165 """A hard presubmit error."""
166 fatal = True
167
168
169# Top level object so multiprocessing can pickle
170# Public access through OutputApi object.
171class _PresubmitPromptWarning(_PresubmitResult):
172 """An warning that prompts the user if they want to continue."""
173 should_prompt = True
174
175
176# Top level object so multiprocessing can pickle
177# Public access through OutputApi object.
178class _PresubmitNotifyResult(_PresubmitResult):
179 """Just print something to the screen -- but it's not even a warning."""
180 pass
181
182
183# Top level object so multiprocessing can pickle
184# Public access through OutputApi object.
185class _MailTextResult(_PresubmitResult):
186 """A warning that should be included in the review request email."""
187 def __init__(self, *args, **kwargs):
188 super(_MailTextResult, self).__init__()
189 raise NotImplementedError()
190
191
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000192class OutputApi(object):
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000193 """An instance of OutputApi gets passed to presubmit scripts so that they
194 can output various types of results.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000195 """
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000196 PresubmitResult = _PresubmitResult
197 PresubmitAddReviewers = _PresubmitAddReviewers
198 PresubmitError = _PresubmitError
199 PresubmitPromptWarning = _PresubmitPromptWarning
200 PresubmitNotifyResult = _PresubmitNotifyResult
201 MailTextResult = _MailTextResult
202
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000203 def __init__(self, is_committing):
204 self.is_committing = is_committing
205
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000206 def PresubmitPromptOrNotify(self, *args, **kwargs):
207 """Warn the user when uploading, but only notify if committing."""
208 if self.is_committing:
209 return self.PresubmitNotifyResult(*args, **kwargs)
210 return self.PresubmitPromptWarning(*args, **kwargs)
211
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000212
213class InputApi(object):
214 """An instance of this object is passed to presubmit scripts so they can
215 know stuff about the change they're looking at.
216 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000217 # Method could be a function
218 # pylint: disable=R0201
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000219
maruel@chromium.org3410d912009-06-09 20:56:16 +0000220 # File extensions that are considered source files from a style guide
221 # perspective. Don't modify this list from a presubmit script!
maruel@chromium.orgc33455a2011-06-24 19:14:18 +0000222 #
223 # Files without an extension aren't included in the list. If you want to
224 # filter them as source files, add r"(^|.*?[\\\/])[^.]+$" to the white list.
225 # Note that ALL CAPS files are black listed in DEFAULT_BLACK_LIST below.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000226 DEFAULT_WHITE_LIST = (
227 # C++ and friends
maruel@chromium.orgfe1211a2011-05-28 18:54:17 +0000228 r".+\.c$", r".+\.cc$", r".+\.cpp$", r".+\.h$", r".+\.m$", r".+\.mm$",
229 r".+\.inl$", r".+\.asm$", r".+\.hxx$", r".+\.hpp$", r".+\.s$", r".+\.S$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000230 # Scripts
maruel@chromium.orgfe1211a2011-05-28 18:54:17 +0000231 r".+\.js$", r".+\.py$", r".+\.sh$", r".+\.rb$", r".+\.pl$", r".+\.pm$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000232 # Other
estade@chromium.orgae7af922012-01-27 14:51:13 +0000233 r".+\.java$", r".+\.mk$", r".+\.am$", r".+\.css$"
maruel@chromium.org3410d912009-06-09 20:56:16 +0000234 )
235
236 # Path regexp that should be excluded from being considered containing source
237 # files. Don't modify this list from a presubmit script!
238 DEFAULT_BLACK_LIST = (
gavinp@chromium.org656326d2012-08-13 00:43:57 +0000239 r"testing_support[\\\/]google_appengine[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000240 r".*\bexperimental[\\\/].*",
241 r".*\bthird_party[\\\/].*",
242 # Output directories (just in case)
243 r".*\bDebug[\\\/].*",
244 r".*\bRelease[\\\/].*",
245 r".*\bxcodebuild[\\\/].*",
246 r".*\bsconsbuild[\\\/].*",
247 # All caps files like README and LICENCE.
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000248 r".*\b[A-Z0-9_]{2,}$",
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000249 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000250 r"(|.*[\\\/])\.git[\\\/].*",
251 r"(|.*[\\\/])\.svn[\\\/].*",
maruel@chromium.org7ccb4bb2011-11-07 20:26:20 +0000252 # There is no point in processing a patch file.
253 r".+\.diff$",
254 r".+\.patch$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000255 )
256
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000257 def __init__(self, change, presubmit_path, is_committing,
maruel@chromium.org239f4112011-06-03 20:08:23 +0000258 rietveld_obj, verbose):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000259 """Builds an InputApi object.
260
261 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000262 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000263 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000264 is_committing: True if the change is about to be committed.
maruel@chromium.org239f4112011-06-03 20:08:23 +0000265 rietveld_obj: rietveld.Rietveld client object
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000266 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000267 # Version number of the presubmit_support script.
268 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000269 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000270 self.is_committing = is_committing
maruel@chromium.org239f4112011-06-03 20:08:23 +0000271 self.rietveld = rietveld_obj
maruel@chromium.orgcab38e92011-04-09 00:30:51 +0000272 # TBD
273 self.host_url = 'http://codereview.chromium.org'
274 if self.rietveld:
maruel@chromium.org239f4112011-06-03 20:08:23 +0000275 self.host_url = self.rietveld.url
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000276
277 # We expose various modules and functions as attributes of the input_api
278 # so that presubmit scripts don't have to import them.
279 self.basename = os.path.basename
280 self.cPickle = cPickle
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000281 self.cpplint = cpplint
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000282 self.cStringIO = cStringIO
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000283 self.glob = glob.glob
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000284 self.json = json
maruel@chromium.org6fba34d2011-06-02 13:45:12 +0000285 self.logging = logging.getLogger('PRESUBMIT')
maruel@chromium.org2b5ce562011-03-31 16:15:44 +0000286 self.os_listdir = os.listdir
287 self.os_walk = os.walk
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000288 self.os_path = os.path
289 self.pickle = pickle
290 self.marshal = marshal
291 self.re = re
292 self.subprocess = subprocess
293 self.tempfile = tempfile
dpranke@chromium.org0d1bdea2011-03-24 22:54:38 +0000294 self.time = time
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000295 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000296 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000297 self.urllib2 = urllib2
298
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000299 # To easily fork python.
300 self.python_executable = sys.executable
301 self.environ = os.environ
302
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000303 # InputApi.platform is the platform you're currently running on.
304 self.platform = sys.platform
305
306 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000307 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000308
309 # We carry the canned checks so presubmit scripts can easily use them.
310 self.canned_checks = presubmit_canned_checks
311
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000312 # TODO(dpranke): figure out a list of all approved owners for a repo
313 # in order to be able to handle wildcard OWNERS files?
314 self.owners_db = owners.Database(change.RepositoryRoot(),
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000315 fopen=file, os_path=self.os_path, glob=self.glob)
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000316 self.verbose = verbose
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000317 self.Command = CommandData
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000318
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000319 # Replace <hash_map> and <hash_set> as headers that need to be included
320 # with "base/hash_tables.h" instead.
321 # Access to a protected member _XX of a client class
322 # pylint: disable=W0212
323 self.cpplint._re_pattern_templates = [
324 (a, b, 'base/hash_tables.h')
325 if header in ('<hash_map>', '<hash_set>') else (a, b, header)
326 for (a, b, header) in cpplint._re_pattern_templates
327 ]
328
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000329 def PresubmitLocalPath(self):
330 """Returns the local path of the presubmit script currently being run.
331
332 This is useful if you don't want to hard-code absolute paths in the
333 presubmit script. For example, It can be used to find another file
334 relative to the PRESUBMIT.py script, so the whole tree can be branched and
335 the presubmit script still works, without editing its content.
336 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000337 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000338
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000339 def DepotToLocalPath(self, depot_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000340 """Translate a depot path to a local path (relative to client root).
341
342 Args:
343 Depot path as a string.
344
345 Returns:
346 The local path of the depot path under the user's current client, or None
347 if the file is not mapped.
348
349 Remember to check for the None case and show an appropriate error!
350 """
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000351 return scm.SVN.CaptureLocalInfo([depot_path], self.change.RepositoryRoot()
352 ).get('Path')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000353
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000354 def LocalToDepotPath(self, local_path):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000355 """Translate a local path to a depot path.
356
357 Args:
358 Local path (relative to current directory, or absolute) as a string.
359
360 Returns:
361 The depot path (SVN URL) of the file if mapped, otherwise None.
362 """
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000363 return scm.SVN.CaptureLocalInfo([local_path], self.change.RepositoryRoot()
364 ).get('URL')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000365
sail@chromium.org5538e022011-05-12 17:53:16 +0000366 def AffectedFiles(self, include_dirs=False, include_deletes=True,
367 file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000368 """Same as input_api.change.AffectedFiles() except only lists files
369 (and optionally directories) in the same directory as the current presubmit
370 script, or subdirectories thereof.
371 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000372 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000373 if len(dir_with_slash) == 1:
374 dir_with_slash = ''
sail@chromium.org5538e022011-05-12 17:53:16 +0000375
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000376 return filter(
377 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
sail@chromium.org5538e022011-05-12 17:53:16 +0000378 self.change.AffectedFiles(include_dirs, include_deletes, file_filter))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000379
380 def LocalPaths(self, include_dirs=False):
381 """Returns local paths of input_api.AffectedFiles()."""
382 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
383
384 def AbsoluteLocalPaths(self, include_dirs=False):
385 """Returns absolute local paths of input_api.AffectedFiles()."""
386 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
387
388 def ServerPaths(self, include_dirs=False):
389 """Returns server paths of input_api.AffectedFiles()."""
390 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
391
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000392 def AffectedTextFiles(self, include_deletes=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000393 """Same as input_api.change.AffectedTextFiles() except only lists files
394 in the same directory as the current presubmit script, or subdirectories
395 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000396 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000397 if include_deletes is not None:
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000398 warn("AffectedTextFiles(include_deletes=%s)"
399 " is deprecated and ignored" % str(include_deletes),
400 category=DeprecationWarning,
401 stacklevel=2)
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000402 return filter(lambda x: x.IsTextFile(),
403 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000404
maruel@chromium.org3410d912009-06-09 20:56:16 +0000405 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
406 """Filters out files that aren't considered "source file".
407
408 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
409 and InputApi.DEFAULT_BLACK_LIST is used respectively.
410
411 The lists will be compiled as regular expression and
412 AffectedFile.LocalPath() needs to pass both list.
413
414 Note: Copy-paste this function to suit your needs or use a lambda function.
415 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000416 def Find(affected_file, items):
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000417 local_path = affected_file.LocalPath()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000418 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000419 if self.re.match(item, local_path):
420 logging.debug("%s matched %s" % (item, local_path))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000421 return True
422 return False
423 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
424 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
425
426 def AffectedSourceFiles(self, source_file):
427 """Filter the list of AffectedTextFiles by the function source_file.
428
429 If source_file is None, InputApi.FilterSourceFile() is used.
430 """
431 if not source_file:
432 source_file = self.FilterSourceFile
433 return filter(source_file, self.AffectedTextFiles())
434
435 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000436 """An iterator over all text lines in "new" version of changed files.
437
438 Only lists lines from new or modified text files in the change that are
439 contained by the directory of the currently executing presubmit script.
440
441 This is useful for doing line-by-line regex checks, like checking for
442 trailing whitespace.
443
444 Yields:
445 a 3 tuple:
446 the AffectedFile instance of the current file;
447 integer line number (1-based); and
448 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000449
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000450 Note: The carriage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000451 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000452 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000453 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000454
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000455 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000456 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000457
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000458 Deny reading anything outside the repository.
459 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000460 if isinstance(file_item, AffectedFile):
461 file_item = file_item.AbsoluteLocalPath()
462 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000463 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000464 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000465
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000466 @property
467 def tbr(self):
468 """Returns if a change is TBR'ed."""
469 return 'TBR' in self.change.tags
470
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000471 @staticmethod
472 def RunTests(tests_mix, parallel=True):
473 tests = []
474 msgs = []
475 for t in tests_mix:
476 if isinstance(t, OutputApi.PresubmitResult):
477 msgs.append(t)
478 else:
479 assert issubclass(t.message, _PresubmitResult)
480 tests.append(t)
ilevy@chromium.org5678d332013-05-18 01:34:14 +0000481 if len(tests) > 1 and parallel:
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000482 pool = multiprocessing.Pool()
483 # async recipe works around multiprocessing bug handling Ctrl-C
484 msgs.extend(pool.map_async(CallCommand, tests).get(99999))
485 pool.close()
486 pool.join()
487 else:
488 msgs.extend(map(CallCommand, tests))
489 return [m for m in msgs if m]
490
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000491
nick@chromium.orgff526192013-06-10 19:30:26 +0000492class _DiffCache(object):
493 """Caches diffs retrieved from a particular SCM."""
494
495 def GetDiff(self, path, local_root):
496 """Get the diff for a particular path."""
497 raise NotImplementedError()
498
499
500class _SvnDiffCache(_DiffCache):
501 """DiffCache implementation for subversion."""
502 def __init__(self):
503 super(_SvnDiffCache, self).__init__()
504 self._diffs_by_file = {}
505
506 def GetDiff(self, path, local_root):
507 if path not in self._diffs_by_file:
508 self._diffs_by_file[path] = scm.SVN.GenerateDiff([path], local_root,
509 False, None)
510 return self._diffs_by_file[path]
511
512
513class _GitDiffCache(_DiffCache):
514 """DiffCache implementation for git; gets all file diffs at once."""
515 def __init__(self):
516 super(_GitDiffCache, self).__init__()
517 self._diffs_by_file = None
518
519 def GetDiff(self, path, local_root):
520 if not self._diffs_by_file:
521 # Compute a single diff for all files and parse the output; should
522 # with git this is much faster than computing one diff for each file.
523 diffs = {}
524
525 # Don't specify any filenames below, because there are command line length
526 # limits on some platforms and GenerateDiff would fail.
527 unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True)
528
529 # This regex matches the path twice, separated by a space. Note that
530 # filename itself may contain spaces.
531 file_marker = re.compile('^diff --git (?P<filename>.*) (?P=filename)$')
532 current_diff = []
533 keep_line_endings = True
534 for x in unified_diff.splitlines(keep_line_endings):
535 match = file_marker.match(x)
536 if match:
537 # Marks the start of a new per-file section.
538 diffs[match.group('filename')] = current_diff = [x]
539 elif x.startswith('diff --git'):
540 raise PresubmitFailure('Unexpected diff line: %s' % x)
541 else:
542 current_diff.append(x)
543
544 self._diffs_by_file = dict(
545 (normpath(path), ''.join(diff)) for path, diff in diffs.items())
546
547 if path not in self._diffs_by_file:
548 raise PresubmitFailure(
549 'Unified diff did not contain entry for file %s' % path)
550
551 return self._diffs_by_file[path]
552
553
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000554class AffectedFile(object):
555 """Representation of a file in a change."""
nick@chromium.orgff526192013-06-10 19:30:26 +0000556
557 DIFF_CACHE = _DiffCache
558
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000559 # Method could be a function
560 # pylint: disable=R0201
nick@chromium.orgff526192013-06-10 19:30:26 +0000561 def __init__(self, path, action, repository_root, diff_cache=None):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000562 self._path = path
563 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000564 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000565 self._is_directory = None
566 self._properties = {}
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000567 self._cached_changed_contents = None
568 self._cached_new_contents = None
nick@chromium.orgff526192013-06-10 19:30:26 +0000569 if diff_cache:
570 self._diff_cache = diff_cache
571 else:
572 self._diff_cache = self.DIFF_CACHE()
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000573 logging.debug('%s(%s)' % (self.__class__.__name__, self._path))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000574
575 def ServerPath(self):
576 """Returns a path string that identifies the file in the SCM system.
577
578 Returns the empty string if the file does not exist in SCM.
579 """
nick@chromium.orgff526192013-06-10 19:30:26 +0000580 return ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000581
582 def LocalPath(self):
583 """Returns the path of this file on the local disk relative to client root.
584 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000585 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000586
587 def AbsoluteLocalPath(self):
588 """Returns the absolute path of this file on the local disk.
589 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000590 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000591
592 def IsDirectory(self):
593 """Returns true if this object is a directory."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000594 if self._is_directory is None:
595 path = self.AbsoluteLocalPath()
596 self._is_directory = (os.path.exists(path) and
597 os.path.isdir(path))
598 return self._is_directory
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000599
600 def Action(self):
601 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000602 # TODO(maruel): Somewhat crappy, Could be "A" or "A +" for svn but
603 # different for other SCM.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000604 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000605
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000606 def Property(self, property_name):
607 """Returns the specified SCM property of this file, or None if no such
608 property.
609 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000610 return self._properties.get(property_name, None)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000611
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000612 def IsTextFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000613 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000614
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000615 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000616 raise NotImplementedError() # Implement when needed
617
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000618 def NewContents(self):
619 """Returns an iterator over the lines in the new version of file.
620
621 The new version is the file in the user's workspace, i.e. the "right hand
622 side".
623
624 Contents will be empty if the file is a directory or does not exist.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000625 Note: The carriage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000626 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000627 if self._cached_new_contents is None:
628 self._cached_new_contents = []
629 if not self.IsDirectory():
630 try:
631 self._cached_new_contents = gclient_utils.FileRead(
632 self.AbsoluteLocalPath(), 'rU').splitlines()
633 except IOError:
634 pass # File not found? That's fine; maybe it was deleted.
635 return self._cached_new_contents[:]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000636
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000637 def ChangedContents(self):
638 """Returns a list of tuples (line number, line text) of all new lines.
639
640 This relies on the scm diff output describing each changed code section
641 with a line of the form
642
643 ^@@ <old line num>,<old size> <new line num>,<new size> @@$
644 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000645 if self._cached_changed_contents is not None:
646 return self._cached_changed_contents[:]
647 self._cached_changed_contents = []
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000648 line_num = 0
649
650 if self.IsDirectory():
651 return []
652
653 for line in self.GenerateScmDiff().splitlines():
654 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
655 if m:
656 line_num = int(m.groups(1)[0])
657 continue
658 if line.startswith('+') and not line.startswith('++'):
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000659 self._cached_changed_contents.append((line_num, line[1:]))
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000660 if not line.startswith('-'):
661 line_num += 1
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000662 return self._cached_changed_contents[:]
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000663
maruel@chromium.org5de13972009-06-10 18:16:06 +0000664 def __str__(self):
665 return self.LocalPath()
666
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000667 def GenerateScmDiff(self):
nick@chromium.orgff526192013-06-10 19:30:26 +0000668 return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000669
maruel@chromium.org58407af2011-04-12 23:15:57 +0000670
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000671class SvnAffectedFile(AffectedFile):
672 """Representation of a file in a change out of a Subversion checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000673 # Method 'NNN' is abstract in class 'NNN' but is not overridden
674 # pylint: disable=W0223
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000675
nick@chromium.orgff526192013-06-10 19:30:26 +0000676 DIFF_CACHE = _SvnDiffCache
677
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000678 def __init__(self, *args, **kwargs):
679 AffectedFile.__init__(self, *args, **kwargs)
680 self._server_path = None
681 self._is_text_file = None
682
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000683 def ServerPath(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000684 if self._server_path is None:
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000685 self._server_path = scm.SVN.CaptureLocalInfo(
686 [self.LocalPath()], self._local_root).get('URL', '')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000687 return self._server_path
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000688
689 def IsDirectory(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000690 if self._is_directory is None:
691 path = self.AbsoluteLocalPath()
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000692 if os.path.exists(path):
693 # Retrieve directly from the file system; it is much faster than
694 # querying subversion, especially on Windows.
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000695 self._is_directory = os.path.isdir(path)
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000696 else:
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000697 self._is_directory = scm.SVN.CaptureLocalInfo(
698 [self.LocalPath()], self._local_root
699 ).get('Node Kind') in ('dir', 'directory')
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000700 return self._is_directory
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000701
702 def Property(self, property_name):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000703 if not property_name in self._properties:
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000704 self._properties[property_name] = scm.SVN.GetFileProperty(
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000705 self.LocalPath(), property_name, self._local_root).rstrip()
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000706 return self._properties[property_name]
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000707
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000708 def IsTextFile(self):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000709 if self._is_text_file is None:
710 if self.Action() == 'D':
711 # A deleted file is not a text file.
712 self._is_text_file = False
713 elif self.IsDirectory():
714 self._is_text_file = False
715 else:
maruel@chromium.orgd579fcf2011-12-13 20:36:03 +0000716 mime_type = scm.SVN.GetFileProperty(
717 self.LocalPath(), 'svn:mime-type', self._local_root)
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000718 self._is_text_file = (not mime_type or mime_type.startswith('text/'))
719 return self._is_text_file
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000720
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000721
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000722class GitAffectedFile(AffectedFile):
723 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000724 # Method 'NNN' is abstract in class 'NNN' but is not overridden
725 # pylint: disable=W0223
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000726
nick@chromium.orgff526192013-06-10 19:30:26 +0000727 DIFF_CACHE = _GitDiffCache
728
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000729 def __init__(self, *args, **kwargs):
730 AffectedFile.__init__(self, *args, **kwargs)
731 self._server_path = None
732 self._is_text_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000733
734 def ServerPath(self):
735 if self._server_path is None:
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000736 raise NotImplementedError('TODO(maruel) Implement.')
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000737 return self._server_path
738
739 def IsDirectory(self):
740 if self._is_directory is None:
741 path = self.AbsoluteLocalPath()
742 if os.path.exists(path):
743 # Retrieve directly from the file system; it is much faster than
744 # querying subversion, especially on Windows.
745 self._is_directory = os.path.isdir(path)
746 else:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000747 self._is_directory = False
748 return self._is_directory
749
750 def Property(self, property_name):
751 if not property_name in self._properties:
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000752 raise NotImplementedError('TODO(maruel) Implement.')
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000753 return self._properties[property_name]
754
755 def IsTextFile(self):
756 if self._is_text_file is None:
757 if self.Action() == 'D':
758 # A deleted file is not a text file.
759 self._is_text_file = False
760 elif self.IsDirectory():
761 self._is_text_file = False
762 else:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000763 self._is_text_file = os.path.isfile(self.AbsoluteLocalPath())
764 return self._is_text_file
765
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000766
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000767class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000768 """Describe a change.
769
770 Used directly by the presubmit scripts to query the current change being
771 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000772
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000773 Instance members:
nick@chromium.orgff526192013-06-10 19:30:26 +0000774 tags: Dictionary of KEY=VALUE pairs found in the change description.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000775 self.KEY: equivalent to tags['KEY']
776 """
777
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000778 _AFFECTED_FILES = AffectedFile
779
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000780 # Matches key/value (or "tag") lines in changelist descriptions.
maruel@chromium.org428342a2011-11-10 15:46:33 +0000781 TAG_LINE_RE = re.compile(
maruel@chromium.orgc6f60e82013-04-19 17:01:57 +0000782 '^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000783 scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000784
maruel@chromium.org58407af2011-04-12 23:15:57 +0000785 def __init__(
786 self, name, description, local_root, files, issue, patchset, author):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000787 if files is None:
788 files = []
789 self._name = name
790 self._full_description = description
chase@chromium.org8e416c82009-10-06 04:30:44 +0000791 # Convert root into an absolute path.
792 self._local_root = os.path.abspath(local_root)
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000793 self.issue = issue
794 self.patchset = patchset
maruel@chromium.org58407af2011-04-12 23:15:57 +0000795 self.author_email = author
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000796
797 # From the description text, build up a dictionary of key/value pairs
798 # plus the description minus all key/value or "tag" lines.
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000799 description_without_tags = []
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000800 self.tags = {}
maruel@chromium.org8d5c9a52009-06-12 15:59:08 +0000801 for line in self._full_description.splitlines():
maruel@chromium.org428342a2011-11-10 15:46:33 +0000802 m = self.TAG_LINE_RE.match(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000803 if m:
804 self.tags[m.group('key')] = m.group('value')
805 else:
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000806 description_without_tags.append(line)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000807
808 # Change back to text and remove whitespace at end.
maruel@chromium.orgfa410372010-09-10 17:01:01 +0000809 self._description_without_tags = (
810 '\n'.join(description_without_tags).rstrip())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000811
maruel@chromium.orge085d812011-10-10 19:49:15 +0000812 assert all(
813 (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files
814
nick@chromium.orgff526192013-06-10 19:30:26 +0000815 diff_cache = self._AFFECTED_FILES.DIFF_CACHE()
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000816 self._affected_files = [
nick@chromium.orgff526192013-06-10 19:30:26 +0000817 self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache)
818 for action, path in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000819 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000820
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000821 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000822 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000823 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000824
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000825 def DescriptionText(self):
826 """Returns the user-entered changelist description, minus tags.
827
828 Any line in the user-provided description starting with e.g. "FOO="
829 (whitespace permitted before and around) is considered a tag line. Such
830 lines are stripped out of the description this function returns.
831 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000832 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000833
834 def FullDescriptionText(self):
835 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000836 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000837
838 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000839 """Returns the repository (checkout) root directory for this change,
840 as an absolute path.
841 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000842 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000843
844 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000845 """Return tags directly as attributes on the object."""
846 if not re.match(r"^[A-Z_]*$", attr):
847 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +0000848 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000849
sail@chromium.org5538e022011-05-12 17:53:16 +0000850 def AffectedFiles(self, include_dirs=False, include_deletes=True,
851 file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000852 """Returns a list of AffectedFile instances for all files in the change.
853
854 Args:
855 include_deletes: If false, deleted files will be filtered out.
856 include_dirs: True to include directories in the list
sail@chromium.org5538e022011-05-12 17:53:16 +0000857 file_filter: An additional filter to apply.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000858
859 Returns:
860 [AffectedFile(path, action), AffectedFile(path, action)]
861 """
862 if include_dirs:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000863 affected = self._affected_files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000864 else:
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000865 affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000866
sail@chromium.org5538e022011-05-12 17:53:16 +0000867 affected = filter(file_filter, affected)
868
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000869 if include_deletes:
870 return affected
871 else:
872 return filter(lambda x: x.Action() != 'D', affected)
873
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000874 def AffectedTextFiles(self, include_deletes=None):
875 """Return a list of the existing text files in a change."""
876 if include_deletes is not None:
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000877 warn("AffectedTextFiles(include_deletes=%s)"
878 " is deprecated and ignored" % str(include_deletes),
879 category=DeprecationWarning,
880 stacklevel=2)
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000881 return filter(lambda x: x.IsTextFile(),
882 self.AffectedFiles(include_dirs=False, include_deletes=False))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000883
884 def LocalPaths(self, include_dirs=False):
885 """Convenience function."""
886 return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]
887
888 def AbsoluteLocalPaths(self, include_dirs=False):
889 """Convenience function."""
890 return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]
891
892 def ServerPaths(self, include_dirs=False):
893 """Convenience function."""
894 return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]
895
896 def RightHandSideLines(self):
897 """An iterator over all text lines in "new" version of changed files.
898
899 Lists lines from new or modified text files in the change.
900
901 This is useful for doing line-by-line regex checks, like checking for
902 trailing whitespace.
903
904 Yields:
905 a 3 tuple:
906 the AffectedFile instance of the current file;
907 integer line number (1-based); and
908 the contents of the line as a string.
909 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000910 return _RightHandSideLinesImpl(
911 x for x in self.AffectedFiles(include_deletes=False)
912 if x.IsTextFile())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000913
914
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000915class SvnChange(Change):
916 _AFFECTED_FILES = SvnAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000917 scm = 'svn'
918 _changelists = None
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000919
920 def _GetChangeLists(self):
921 """Get all change lists."""
922 if self._changelists == None:
923 previous_cwd = os.getcwd()
924 os.chdir(self.RepositoryRoot())
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000925 # Need to import here to avoid circular dependency.
926 import gcl
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000927 self._changelists = gcl.GetModifiedFiles()
928 os.chdir(previous_cwd)
929 return self._changelists
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000930
931 def GetAllModifiedFiles(self):
932 """Get all modified files."""
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000933 changelists = self._GetChangeLists()
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000934 all_modified_files = []
935 for cl in changelists.values():
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000936 all_modified_files.extend(
937 [os.path.join(self.RepositoryRoot(), f[1]) for f in cl])
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000938 return all_modified_files
939
940 def GetModifiedFiles(self):
941 """Get modified files in the current CL."""
thestig@chromium.org6bd31702009-09-02 23:29:07 +0000942 changelists = self._GetChangeLists()
943 return [os.path.join(self.RepositoryRoot(), f[1])
944 for f in changelists[self.Name()]]
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000945
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000946
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000947class GitChange(Change):
948 _AFFECTED_FILES = GitAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000949 scm = 'git'
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000950
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000951
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000952def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000953 """Finds all presubmit files that apply to a given set of source files.
954
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000955 If inherit-review-settings-ok is present right under root, looks for
956 PRESUBMIT.py in directories enclosing root.
957
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000958 Args:
959 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000960 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000961
962 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000963 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000964 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000965 files = [normpath(os.path.join(root, f)) for f in files]
966
967 # List all the individual directories containing files.
968 directories = set([os.path.dirname(f) for f in files])
969
970 # Ignore root if inherit-review-settings-ok is present.
971 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
972 root = None
973
974 # Collect all unique directories that may contain PRESUBMIT.py.
975 candidates = set()
976 for directory in directories:
977 while True:
978 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000979 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000980 candidates.add(directory)
981 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000982 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000983 parent_dir = os.path.dirname(directory)
984 if parent_dir == directory:
985 # We hit the system root directory.
986 break
987 directory = parent_dir
988
989 # Look for PRESUBMIT.py in all candidate directories.
990 results = []
991 for directory in sorted(list(candidates)):
992 p = os.path.join(directory, 'PRESUBMIT.py')
993 if os.path.isfile(p):
994 results.append(p)
995
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000996 logging.debug('Presubmit files: %s' % ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +0000997 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000998
999
thestig@chromium.orgde243452009-10-06 21:02:56 +00001000class GetTrySlavesExecuter(object):
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001001 @staticmethod
asvitkine@chromium.org15169952011-09-27 14:30:53 +00001002 def ExecPresubmitScript(script_text, presubmit_path, project, change):
thestig@chromium.orgde243452009-10-06 21:02:56 +00001003 """Executes GetPreferredTrySlaves() from a single presubmit script.
1004
1005 Args:
1006 script_text: The text of the presubmit script.
bradnelson@google.com78230022011-05-24 18:55:19 +00001007 presubmit_path: Project script to run.
1008 project: Project name to pass to presubmit script for bot selection.
thestig@chromium.orgde243452009-10-06 21:02:56 +00001009
1010 Return:
1011 A list of try slaves.
1012 """
1013 context = {}
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001014 try:
1015 exec script_text in context
1016 except Exception, e:
1017 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
thestig@chromium.orgde243452009-10-06 21:02:56 +00001018
1019 function_name = 'GetPreferredTrySlaves'
1020 if function_name in context:
asvitkine@chromium.org15169952011-09-27 14:30:53 +00001021 get_preferred_try_slaves = context[function_name]
1022 function_info = inspect.getargspec(get_preferred_try_slaves)
1023 if len(function_info[0]) == 1:
1024 result = get_preferred_try_slaves(project)
1025 elif len(function_info[0]) == 2:
1026 result = get_preferred_try_slaves(project, change)
1027 else:
1028 result = get_preferred_try_slaves()
thestig@chromium.orgde243452009-10-06 21:02:56 +00001029 if not isinstance(result, types.ListType):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001030 raise PresubmitFailure(
thestig@chromium.orgde243452009-10-06 21:02:56 +00001031 'Presubmit functions must return a list, got a %s instead: %s' %
1032 (type(result), str(result)))
1033 for item in result:
1034 if not isinstance(item, basestring):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001035 raise PresubmitFailure('All try slaves names must be strings.')
thestig@chromium.orgde243452009-10-06 21:02:56 +00001036 if item != item.strip():
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001037 raise PresubmitFailure(
1038 'Try slave names cannot start/end with whitespace')
maruel@chromium.org3ecc8ea2012-03-10 01:47:46 +00001039 if ',' in item:
1040 raise PresubmitFailure(
1041 'Do not use \',\' separated builder or test names: %s' % item)
thestig@chromium.orgde243452009-10-06 21:02:56 +00001042 else:
1043 result = []
1044 return result
1045
1046
asvitkine@chromium.org15169952011-09-27 14:30:53 +00001047def DoGetTrySlaves(change,
1048 changed_files,
thestig@chromium.orgde243452009-10-06 21:02:56 +00001049 repository_root,
1050 default_presubmit,
bradnelson@google.com78230022011-05-24 18:55:19 +00001051 project,
thestig@chromium.orgde243452009-10-06 21:02:56 +00001052 verbose,
1053 output_stream):
1054 """Get the list of try servers from the presubmit scripts.
1055
1056 Args:
1057 changed_files: List of modified files.
1058 repository_root: The repository root.
1059 default_presubmit: A default presubmit script to execute in any case.
bradnelson@google.com78230022011-05-24 18:55:19 +00001060 project: Optional name of a project used in selecting trybots.
thestig@chromium.orgde243452009-10-06 21:02:56 +00001061 verbose: Prints debug info.
1062 output_stream: A stream to write debug output to.
1063
1064 Return:
1065 List of try slaves
1066 """
1067 presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
1068 if not presubmit_files and verbose:
1069 output_stream.write("Warning, no presubmit.py found.\n")
1070 results = []
1071 executer = GetTrySlavesExecuter()
1072 if default_presubmit:
1073 if verbose:
1074 output_stream.write("Running default presubmit script.\n")
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001075 fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
bradnelson@google.com78230022011-05-24 18:55:19 +00001076 results += executer.ExecPresubmitScript(
asvitkine@chromium.org15169952011-09-27 14:30:53 +00001077 default_presubmit, fake_path, project, change)
thestig@chromium.orgde243452009-10-06 21:02:56 +00001078 for filename in presubmit_files:
1079 filename = os.path.abspath(filename)
1080 if verbose:
1081 output_stream.write("Running %s\n" % filename)
1082 # Accept CRLF presubmit script.
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +00001083 presubmit_script = gclient_utils.FileRead(filename, 'rU')
bradnelson@google.com78230022011-05-24 18:55:19 +00001084 results += executer.ExecPresubmitScript(
asvitkine@chromium.org15169952011-09-27 14:30:53 +00001085 presubmit_script, filename, project, change)
thestig@chromium.orgde243452009-10-06 21:02:56 +00001086
1087 slaves = list(set(results))
1088 if slaves and verbose:
1089 output_stream.write(', '.join(slaves))
1090 output_stream.write('\n')
1091 return slaves
1092
1093
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001094class PresubmitExecuter(object):
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +00001095 def __init__(self, change, committing, rietveld_obj, verbose):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001096 """
1097 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001098 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001099 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
maruel@chromium.org239f4112011-06-03 20:08:23 +00001100 rietveld_obj: rietveld.Rietveld client object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001101 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001102 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001103 self.committing = committing
maruel@chromium.org239f4112011-06-03 20:08:23 +00001104 self.rietveld = rietveld_obj
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001105 self.verbose = verbose
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001106
1107 def ExecPresubmitScript(self, script_text, presubmit_path):
1108 """Executes a single presubmit script.
1109
1110 Args:
1111 script_text: The text of the presubmit script.
1112 presubmit_path: The path to the presubmit file (this will be reported via
1113 input_api.PresubmitLocalPath()).
1114
1115 Return:
1116 A list of result objects, empty if no problems.
1117 """
chase@chromium.org8e416c82009-10-06 04:30:44 +00001118
1119 # Change to the presubmit file's directory to support local imports.
1120 main_path = os.getcwd()
1121 os.chdir(os.path.dirname(presubmit_path))
1122
1123 # Load the presubmit script into context.
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001124 input_api = InputApi(self.change, presubmit_path, self.committing,
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +00001125 self.rietveld, self.verbose)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001126 context = {}
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001127 try:
1128 exec script_text in context
1129 except Exception, e:
1130 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001131
1132 # These function names must change if we make substantial changes to
1133 # the presubmit API that are not backwards compatible.
1134 if self.committing:
1135 function_name = 'CheckChangeOnCommit'
1136 else:
1137 function_name = 'CheckChangeOnUpload'
1138 if function_name in context:
wez@chromium.orga6d011e2013-03-26 17:31:49 +00001139 context['__args'] = (input_api, OutputApi(self.committing))
maruel@chromium.org5d0dc432011-01-03 02:40:37 +00001140 logging.debug('Running %s in %s' % (function_name, presubmit_path))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001141 result = eval(function_name + '(*__args)', context)
maruel@chromium.org5d0dc432011-01-03 02:40:37 +00001142 logging.debug('Running %s done.' % function_name)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001143 if not (isinstance(result, types.TupleType) or
1144 isinstance(result, types.ListType)):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001145 raise PresubmitFailure(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001146 'Presubmit functions must return a tuple or list')
1147 for item in result:
1148 if not isinstance(item, OutputApi.PresubmitResult):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001149 raise PresubmitFailure(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001150 'All presubmit results must be of types derived from '
1151 'output_api.PresubmitResult')
1152 else:
1153 result = () # no error since the script doesn't care about current event.
1154
chase@chromium.org8e416c82009-10-06 04:30:44 +00001155 # Return the process to the original working directory.
1156 os.chdir(main_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001157 return result
1158
dpranke@chromium.org5ac21012011-03-16 02:58:25 +00001159
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001160def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001161 committing,
1162 verbose,
1163 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001164 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001165 default_presubmit,
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001166 may_prompt,
maruel@chromium.org239f4112011-06-03 20:08:23 +00001167 rietveld_obj):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001168 """Runs all presubmit checks that apply to the files in the change.
1169
1170 This finds all PRESUBMIT.py files in directories enclosing the files in the
1171 change (up to the repository root) and calls the relevant entrypoint function
1172 depending on whether the change is being committed or uploaded.
1173
1174 Prints errors, warnings and notifications. Prompts the user for warnings
1175 when needed.
1176
1177 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001178 change: The Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001179 committing: True if 'gcl commit' is running, False if 'gcl upload' is.
1180 verbose: Prints debug info.
1181 output_stream: A stream to write output from presubmit tests to.
1182 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001183 default_presubmit: A default presubmit script to execute in any case.
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001184 may_prompt: Enable (y/n) questions on warning or error.
maruel@chromium.org239f4112011-06-03 20:08:23 +00001185 rietveld_obj: rietveld.Rietveld object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001186
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001187 Warning:
1188 If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream
1189 SHOULD be sys.stdin.
1190
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001191 Return:
dpranke@chromium.org5ac21012011-03-16 02:58:25 +00001192 A PresubmitOutput object. Use output.should_continue() to figure out
1193 if there were errors or warnings and the caller should abort.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001194 """
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001195 old_environ = os.environ
1196 try:
1197 # Make sure python subprocesses won't generate .pyc files.
1198 os.environ = os.environ.copy()
1199 os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001200
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001201 output = PresubmitOutput(input_stream, output_stream)
1202 if committing:
1203 output.write("Running presubmit commit checks ...\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001204 else:
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001205 output.write("Running presubmit upload checks ...\n")
1206 start_time = time.time()
1207 presubmit_files = ListRelevantPresubmitFiles(
1208 change.AbsoluteLocalPaths(True), change.RepositoryRoot())
1209 if not presubmit_files and verbose:
maruel@chromium.orgfae707b2011-09-15 18:57:58 +00001210 output.write("Warning, no PRESUBMIT.py found.\n")
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001211 results = []
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +00001212 executer = PresubmitExecuter(change, committing, rietveld_obj, verbose)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001213 if default_presubmit:
1214 if verbose:
1215 output.write("Running default presubmit script.\n")
1216 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
1217 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
1218 for filename in presubmit_files:
1219 filename = os.path.abspath(filename)
1220 if verbose:
1221 output.write("Running %s\n" % filename)
1222 # Accept CRLF presubmit script.
1223 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1224 results += executer.ExecPresubmitScript(presubmit_script, filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001225
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001226 errors = []
1227 notifications = []
1228 warnings = []
1229 for result in results:
1230 if result.fatal:
1231 errors.append(result)
1232 elif result.should_prompt:
1233 warnings.append(result)
1234 else:
1235 notifications.append(result)
pam@chromium.orged9a0832009-09-09 22:48:55 +00001236
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001237 output.write('\n')
1238 for name, items in (('Messages', notifications),
1239 ('Warnings', warnings),
1240 ('ERRORS', errors)):
1241 if items:
1242 output.write('** Presubmit %s **\n' % name)
1243 for item in items:
1244 item.handle(output)
1245 output.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001246
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001247 total_time = time.time() - start_time
1248 if total_time > 1.0:
1249 output.write("Presubmit checks took %.1fs to calculate.\n\n" % total_time)
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001250
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001251 if not errors:
1252 if not warnings:
1253 output.write('Presubmit checks passed.\n')
1254 elif may_prompt:
1255 output.prompt_yes_no('There were presubmit warnings. '
1256 'Are you sure you wish to continue? (y/N): ')
1257 else:
1258 output.fail()
1259
1260 global _ASKED_FOR_FEEDBACK
1261 # Ask for feedback one time out of 5.
1262 if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
1263 output.write("Was the presubmit check useful? Please send feedback "
1264 "& hate mail to maruel@chromium.org!\n")
1265 _ASKED_FOR_FEEDBACK = True
1266 return output
1267 finally:
1268 os.environ = old_environ
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001269
1270
1271def ScanSubDirs(mask, recursive):
1272 if not recursive:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001273 return [x for x in glob.glob(mask) if '.svn' not in x and '.git' not in x]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001274 else:
1275 results = []
1276 for root, dirs, files in os.walk('.'):
1277 if '.svn' in dirs:
1278 dirs.remove('.svn')
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001279 if '.git' in dirs:
1280 dirs.remove('.git')
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001281 for name in files:
1282 if fnmatch.fnmatch(name, mask):
1283 results.append(os.path.join(root, name))
1284 return results
1285
1286
1287def ParseFiles(args, recursive):
maruel@chromium.org7444c502011-02-09 14:02:11 +00001288 logging.debug('Searching for %s' % args)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001289 files = []
1290 for arg in args:
maruel@chromium.orge3608df2009-11-10 20:22:57 +00001291 files.extend([('M', f) for f in ScanSubDirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001292 return files
1293
1294
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001295def load_files(options, args):
1296 """Tries to determine the SCM."""
1297 change_scm = scm.determine_scm(options.root)
1298 files = []
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001299 if args:
1300 files = ParseFiles(args, options.recursive)
maruel@chromium.org9b31f162012-01-26 19:02:31 +00001301 if change_scm == 'svn':
1302 change_class = SvnChange
1303 if not files:
1304 files = scm.SVN.CaptureStatus([], options.root)
1305 elif change_scm == 'git':
1306 change_class = GitChange
1307 # TODO(maruel): Get upstream.
1308 if not files:
1309 files = scm.GIT.CaptureStatus([], options.root, None)
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001310 else:
maruel@chromium.org9b31f162012-01-26 19:02:31 +00001311 logging.info('Doesn\'t seem under source control. Got %d files' % len(args))
1312 if not files:
1313 return None, None
1314 change_class = Change
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001315 return change_class, files
1316
1317
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001318class NonexistantCannedCheckFilter(Exception):
1319 pass
1320
1321
1322@contextlib.contextmanager
1323def canned_check_filter(method_names):
1324 filtered = {}
1325 try:
1326 for method_name in method_names:
1327 if not hasattr(presubmit_canned_checks, method_name):
1328 raise NonexistantCannedCheckFilter(method_name)
1329 filtered[method_name] = getattr(presubmit_canned_checks, method_name)
1330 setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
1331 yield
1332 finally:
1333 for name, method in filtered.iteritems():
1334 setattr(presubmit_canned_checks, name, method)
1335
ilevy@chromium.orgbc117312013-04-20 03:57:56 +00001336def CallCommand(cmd_data):
1337 # multiprocessing needs a top level function with a single argument.
1338 cmd_data.kwargs['stdout'] = subprocess.PIPE
1339 cmd_data.kwargs['stderr'] = subprocess.STDOUT
1340 try:
1341 (out, _), code = subprocess.communicate(cmd_data.cmd, **cmd_data.kwargs)
1342 if code != 0:
1343 return cmd_data.message('%s failed\n%s' % (cmd_data.name, out))
1344 except OSError as e:
1345 return cmd_data.message(
ilevy@chromium.orge7ceaad2013-04-27 00:58:45 +00001346 '%s exec failure\n %s' % (cmd_data.name, e))
ilevy@chromium.orgbc117312013-04-20 03:57:56 +00001347
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001348
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001349def Main(argv):
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001350 parser = optparse.OptionParser(usage="%prog [options] <files...>",
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001351 version="%prog " + str(__version__))
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001352 parser.add_option("-c", "--commit", action="store_true", default=False,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001353 help="Use commit instead of upload checks")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001354 parser.add_option("-u", "--upload", action="store_false", dest='commit',
1355 help="Use upload instead of commit checks")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001356 parser.add_option("-r", "--recursive", action="store_true",
1357 help="Act recursively")
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001358 parser.add_option("-v", "--verbose", action="count", default=0,
1359 help="Use 2 times for more debug info")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001360 parser.add_option("--name", default='no name')
maruel@chromium.org58407af2011-04-12 23:15:57 +00001361 parser.add_option("--author")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001362 parser.add_option("--description", default='')
1363 parser.add_option("--issue", type='int', default=0)
1364 parser.add_option("--patchset", type='int', default=0)
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001365 parser.add_option("--root", default=os.getcwd(),
1366 help="Search for PRESUBMIT.py up to this directory. "
1367 "If inherit-review-settings-ok is present in this "
1368 "directory, parent directories up to the root file "
1369 "system directories will also be searched.")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001370 parser.add_option("--default_presubmit")
1371 parser.add_option("--may_prompt", action='store_true', default=False)
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001372 parser.add_option("--skip_canned", action='append', default=[],
1373 help="A list of checks to skip which appear in "
1374 "presubmit_canned_checks. Can be provided multiple times "
1375 "to skip multiple canned checks.")
maruel@chromium.org239f4112011-06-03 20:08:23 +00001376 parser.add_option("--rietveld_url", help=optparse.SUPPRESS_HELP)
1377 parser.add_option("--rietveld_email", help=optparse.SUPPRESS_HELP)
1378 parser.add_option("--rietveld_password", help=optparse.SUPPRESS_HELP)
iannucci@chromium.org720fd7a2013-04-24 04:13:50 +00001379 parser.add_option("--rietveld_fetch", action='store_true', default=False,
1380 help=optparse.SUPPRESS_HELP)
maruel@chromium.org82e5f282011-03-17 14:08:55 +00001381 options, args = parser.parse_args(argv)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001382 if options.verbose >= 2:
maruel@chromium.org7444c502011-02-09 14:02:11 +00001383 logging.basicConfig(level=logging.DEBUG)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001384 elif options.verbose:
1385 logging.basicConfig(level=logging.INFO)
1386 else:
1387 logging.basicConfig(level=logging.ERROR)
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001388 change_class, files = load_files(options, args)
1389 if not change_class:
1390 parser.error('For unversioned directory, <files> is not optional.')
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001391 logging.info('Found %d file(s).' % len(files))
maruel@chromium.org239f4112011-06-03 20:08:23 +00001392 rietveld_obj = None
1393 if options.rietveld_url:
maruel@chromium.org4bac4b52012-11-27 20:33:52 +00001394 rietveld_obj = rietveld.CachingRietveld(
maruel@chromium.org239f4112011-06-03 20:08:23 +00001395 options.rietveld_url,
1396 options.rietveld_email,
1397 options.rietveld_password)
iannucci@chromium.org720fd7a2013-04-24 04:13:50 +00001398 if options.rietveld_fetch:
1399 assert options.issue
1400 props = rietveld_obj.get_issue_properties(options.issue, False)
1401 options.author = props['owner_email']
1402 options.description = props['description']
1403 logging.info('Got author: "%s"', options.author)
1404 logging.info('Got description: """\n%s\n"""', options.description)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001405 try:
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001406 with canned_check_filter(options.skip_canned):
1407 results = DoPresubmitChecks(
1408 change_class(options.name,
1409 options.description,
1410 options.root,
1411 files,
1412 options.issue,
1413 options.patchset,
1414 options.author),
1415 options.commit,
1416 options.verbose,
1417 sys.stdout,
1418 sys.stdin,
1419 options.default_presubmit,
1420 options.may_prompt,
1421 rietveld_obj)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001422 return not results.should_continue()
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001423 except NonexistantCannedCheckFilter, e:
1424 print >> sys.stderr, (
1425 'Attempted to skip nonexistent canned presubmit check: %s' % e.message)
1426 return 2
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001427 except PresubmitFailure, e:
1428 print >> sys.stderr, e
1429 print >> sys.stderr, 'Maybe your depot_tools is out of date?'
1430 print >> sys.stderr, 'If all fails, contact maruel@'
1431 return 2
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001432
1433
1434if __name__ == '__main__':
maruel@chromium.org35625c72011-03-23 17:34:02 +00001435 fix_encoding.fix_encoding()
maruel@chromium.org82e5f282011-03-17 14:08:55 +00001436 sys.exit(Main(None))