blob: 1fa3c9fce843a84c83f1fbc421b1f6c89ee9b42d [file] [log] [blame]
maruel@chromium.org725f1c32011-04-01 20:24:54 +00001#!/usr/bin/env python
maruel@chromium.org3bbf2942012-01-10 16:52:06 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
Raul Tambre80ee78e2019-05-06 22:41:05 +00009from __future__ import print_function
10
stip@chromium.orgf7d31f52014-01-03 20:14:46 +000011__version__ = '1.8.0'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000012
13# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
14# caching (between all different invocations of presubmit scripts for a given
15# change). We should add it as our presubmit scripts start feeling slow.
16
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +090017import ast # Exposed through the API.
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +000018import contextlib
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +000019import cPickle # Exposed through the API.
20import cpplint
21import cStringIO # Exposed through the API.
dcheng091b7db2016-06-16 01:27:51 -070022import fnmatch # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000023import glob
asvitkine@chromium.org15169952011-09-27 14:30:53 +000024import inspect
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +000025import itertools
maruel@chromium.org4f6852c2012-04-20 20:39:20 +000026import json # Exposed through the API.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000027import logging
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000028import marshal # Exposed through the API.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000029import multiprocessing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import optparse
31import os # Somewhat exposed through the API.
32import pickle # Exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000033import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000034import re # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040035import signal
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000036import sys # Parts exposed through API.
37import tempfile # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040038import threading
jam@chromium.org2a891dc2009-08-20 20:33:37 +000039import time
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +000040import traceback # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000041import types
maruel@chromium.org1487d532009-06-06 00:22:57 +000042import unittest # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000043import urllib2 # Exposed through the API.
tandrii@chromium.org015ebae2016-04-25 19:37:22 +000044import urlparse
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000045from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000046
47# Local imports.
maruel@chromium.org35625c72011-03-23 17:34:02 +000048import fix_encoding
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +000049import gclient_paths # Exposed through the API
50import gclient_utils
Aaron Gableb584c4f2017-04-26 16:28:08 -070051import git_footers
tandrii@chromium.org015ebae2016-04-25 19:37:22 +000052import gerrit_util
dpranke@chromium.org2a009622011-03-01 02:43:31 +000053import owners
Jochen Eisinger76f5fc62017-04-07 16:27:46 +020054import owners_finder
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000055import presubmit_canned_checks
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000056import scm
maruel@chromium.org84f4fe32011-04-06 13:26:45 +000057import subprocess2 as subprocess # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000058
59
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000060# Ask for feedback only once in program lifetime.
61_ASKED_FOR_FEEDBACK = False
62
63
maruel@chromium.org899e1c12011-04-07 17:03:18 +000064class PresubmitFailure(Exception):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000065 pass
66
67
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000068class CommandData(object):
69 def __init__(self, name, cmd, kwargs, message):
70 self.name = name
71 self.cmd = cmd
Edward Lesmes8e282792018-04-03 18:50:29 -040072 self.stdin = kwargs.get('stdin', None)
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000073 self.kwargs = kwargs
Edward Lesmes8e282792018-04-03 18:50:29 -040074 self.kwargs['stdout'] = subprocess.PIPE
75 self.kwargs['stderr'] = subprocess.STDOUT
76 self.kwargs['stdin'] = subprocess.PIPE
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000077 self.message = message
78 self.info = None
79
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000080
Edward Lesmes8e282792018-04-03 18:50:29 -040081# Adapted from
82# https://github.com/google/gtest-parallel/blob/master/gtest_parallel.py#L37
83#
84# An object that catches SIGINT sent to the Python process and notices
85# if processes passed to wait() die by SIGINT (we need to look for
86# both of those cases, because pressing Ctrl+C can result in either
87# the main process or one of the subprocesses getting the signal).
88#
89# Before a SIGINT is seen, wait(p) will simply call p.wait() and
90# return the result. Once a SIGINT has been seen (in the main process
91# or a subprocess, including the one the current call is waiting for),
92# wait(p) will call p.terminate() and raise ProcessWasInterrupted.
93class SigintHandler(object):
94 class ProcessWasInterrupted(Exception):
95 pass
96
97 sigint_returncodes = {-signal.SIGINT, # Unix
98 -1073741510, # Windows
99 }
100 def __init__(self):
101 self.__lock = threading.Lock()
102 self.__processes = set()
103 self.__got_sigint = False
104 signal.signal(signal.SIGINT, lambda signal_num, frame: self.interrupt())
105
106 def __on_sigint(self):
107 self.__got_sigint = True
108 while self.__processes:
109 try:
110 self.__processes.pop().terminate()
111 except OSError:
112 pass
113
114 def interrupt(self):
115 with self.__lock:
116 self.__on_sigint()
117
118 def got_sigint(self):
119 with self.__lock:
120 return self.__got_sigint
121
122 def wait(self, p, stdin):
123 with self.__lock:
124 if self.__got_sigint:
125 p.terminate()
126 self.__processes.add(p)
127 stdout, stderr = p.communicate(stdin)
128 code = p.returncode
129 with self.__lock:
130 self.__processes.discard(p)
131 if code in self.sigint_returncodes:
132 self.__on_sigint()
133 if self.__got_sigint:
134 raise self.ProcessWasInterrupted
135 return stdout, stderr
136
137sigint_handler = SigintHandler()
138
139
140class ThreadPool(object):
141 def __init__(self, pool_size=None):
142 self._pool_size = pool_size or multiprocessing.cpu_count()
143 self._messages = []
144 self._messages_lock = threading.Lock()
145 self._tests = []
146 self._tests_lock = threading.Lock()
147 self._nonparallel_tests = []
148
149 def CallCommand(self, test):
150 """Runs an external program.
151
152 This function converts invocation of .py files and invocations of "python"
153 to vpython invocations.
154 """
155 vpython = 'vpython.bat' if sys.platform == 'win32' else 'vpython'
156
157 cmd = test.cmd
158 if cmd[0] == 'python':
159 cmd = list(cmd)
160 cmd[0] = vpython
161 elif cmd[0].endswith('.py'):
162 cmd = [vpython] + cmd
163
164 try:
165 start = time.time()
166 p = subprocess.Popen(cmd, **test.kwargs)
167 stdout, _ = sigint_handler.wait(p, test.stdin)
168 duration = time.time() - start
169 except OSError as e:
170 duration = time.time() - start
171 return test.message(
172 '%s exec failure (%4.2fs)\n %s' % (test.name, duration, e))
173 if p.returncode != 0:
174 return test.message(
175 '%s (%4.2fs) failed\n%s' % (test.name, duration, stdout))
176 if test.info:
177 return test.info('%s (%4.2fs)' % (test.name, duration))
178
179 def AddTests(self, tests, parallel=True):
180 if parallel:
181 self._tests.extend(tests)
182 else:
183 self._nonparallel_tests.extend(tests)
184
185 def RunAsync(self):
186 self._messages = []
187
188 def _WorkerFn():
189 while True:
190 test = None
191 with self._tests_lock:
192 if not self._tests:
193 break
194 test = self._tests.pop()
195 result = self.CallCommand(test)
196 if result:
197 with self._messages_lock:
198 self._messages.append(result)
199
200 def _StartDaemon():
201 t = threading.Thread(target=_WorkerFn)
202 t.daemon = True
203 t.start()
204 return t
205
206 while self._nonparallel_tests:
207 test = self._nonparallel_tests.pop()
208 result = self.CallCommand(test)
209 if result:
210 self._messages.append(result)
211
212 if self._tests:
213 threads = [_StartDaemon() for _ in range(self._pool_size)]
214 for worker in threads:
215 worker.join()
216
217 return self._messages
218
219
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000220def normpath(path):
221 '''Version of os.path.normpath that also changes backward slashes to
222 forward slashes when not running on Windows.
223 '''
224 # This is safe to always do because the Windows version of os.path.normpath
225 # will replace forward slashes with backward slashes.
226 path = path.replace(os.sep, '/')
227 return os.path.normpath(path)
228
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000229
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000230def _RightHandSideLinesImpl(affected_files):
231 """Implements RightHandSideLines for InputApi and GclChange."""
232 for af in affected_files:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000233 lines = af.ChangedContents()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000234 for line in lines:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000235 yield (af, line[0], line[1])
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000236
237
dpranke@chromium.org5ac21012011-03-16 02:58:25 +0000238class PresubmitOutput(object):
239 def __init__(self, input_stream=None, output_stream=None):
240 self.input_stream = input_stream
241 self.output_stream = output_stream
242 self.reviewers = []
Daniel Cheng7227d212017-11-17 08:12:37 -0800243 self.more_cc = []
dpranke@chromium.org5ac21012011-03-16 02:58:25 +0000244 self.written_output = []
245 self.error_count = 0
246
247 def prompt_yes_no(self, prompt_string):
248 self.write(prompt_string)
249 if self.input_stream:
250 response = self.input_stream.readline().strip().lower()
251 if response not in ('y', 'yes'):
252 self.fail()
253 else:
254 self.fail()
255
256 def fail(self):
257 self.error_count += 1
258
259 def should_continue(self):
260 return not self.error_count
261
262 def write(self, s):
263 self.written_output.append(s)
264 if self.output_stream:
265 self.output_stream.write(s)
266
267 def getvalue(self):
268 return ''.join(self.written_output)
269
270
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000271# Top level object so multiprocessing can pickle
272# Public access through OutputApi object.
273class _PresubmitResult(object):
274 """Base class for result objects."""
275 fatal = False
276 should_prompt = False
277
278 def __init__(self, message, items=None, long_text=''):
279 """
280 message: A short one-line message to indicate errors.
281 items: A list of short strings to indicate where errors occurred.
282 long_text: multi-line text output, e.g. from another tool
283 """
284 self._message = message
285 self._items = items or []
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000286 self._long_text = long_text.rstrip()
287
288 def handle(self, output):
289 output.write(self._message)
290 output.write('\n')
291 for index, item in enumerate(self._items):
292 output.write(' ')
293 # Write separately in case it's unicode.
294 output.write(str(item))
295 if index < len(self._items) - 1:
296 output.write(' \\')
297 output.write('\n')
298 if self._long_text:
299 output.write('\n***************\n')
300 # Write separately in case it's unicode.
301 output.write(self._long_text)
302 output.write('\n***************\n')
303 if self.fatal:
304 output.fail()
305
306
307# Top level object so multiprocessing can pickle
308# Public access through OutputApi object.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000309class _PresubmitError(_PresubmitResult):
310 """A hard presubmit error."""
311 fatal = True
312
313
314# Top level object so multiprocessing can pickle
315# Public access through OutputApi object.
316class _PresubmitPromptWarning(_PresubmitResult):
317 """An warning that prompts the user if they want to continue."""
318 should_prompt = True
319
320
321# Top level object so multiprocessing can pickle
322# Public access through OutputApi object.
323class _PresubmitNotifyResult(_PresubmitResult):
324 """Just print something to the screen -- but it's not even a warning."""
325 pass
326
327
328# Top level object so multiprocessing can pickle
329# Public access through OutputApi object.
330class _MailTextResult(_PresubmitResult):
331 """A warning that should be included in the review request email."""
332 def __init__(self, *args, **kwargs):
333 super(_MailTextResult, self).__init__()
334 raise NotImplementedError()
335
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000336class GerritAccessor(object):
337 """Limited Gerrit functionality for canned presubmit checks to work.
338
339 To avoid excessive Gerrit calls, caches the results.
340 """
341
342 def __init__(self, host):
343 self.host = host
344 self.cache = {}
345
346 def _FetchChangeDetail(self, issue):
347 # Separate function to be easily mocked in tests.
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100348 try:
349 return gerrit_util.GetChangeDetail(
350 self.host, str(issue),
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700351 ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS'])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100352 except gerrit_util.GerritError as e:
353 if e.http_status == 404:
354 raise Exception('Either Gerrit issue %s doesn\'t exist, or '
355 'no credentials to fetch issue details' % issue)
356 raise
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000357
358 def GetChangeInfo(self, issue):
359 """Returns labels and all revisions (patchsets) for this issue.
360
361 The result is a dictionary according to Gerrit REST Api.
362 https://gerrit-review.googlesource.com/Documentation/rest-api.html
363
364 However, API isn't very clear what's inside, so see tests for example.
365 """
366 assert issue
367 cache_key = int(issue)
368 if cache_key not in self.cache:
369 self.cache[cache_key] = self._FetchChangeDetail(issue)
370 return self.cache[cache_key]
371
372 def GetChangeDescription(self, issue, patchset=None):
373 """If patchset is none, fetches current patchset."""
374 info = self.GetChangeInfo(issue)
375 # info is a reference to cache. We'll modify it here adding description to
376 # it to the right patchset, if it is not yet there.
377
378 # Find revision info for the patchset we want.
379 if patchset is not None:
380 for rev, rev_info in info['revisions'].iteritems():
381 if str(rev_info['_number']) == str(patchset):
382 break
383 else:
384 raise Exception('patchset %s doesn\'t exist in issue %s' % (
385 patchset, issue))
386 else:
387 rev = info['current_revision']
388 rev_info = info['revisions'][rev]
389
Andrii Shyshkalov9c3a4642017-01-24 17:41:22 +0100390 return rev_info['commit']['message']
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000391
Mun Yong Jang603d01e2017-12-19 16:38:30 -0800392 def GetDestRef(self, issue):
393 ref = self.GetChangeInfo(issue)['branch']
394 if not ref.startswith('refs/'):
395 # NOTE: it is possible to create 'refs/x' branch,
396 # aka 'refs/heads/refs/x'. However, this is ill-advised.
397 ref = 'refs/heads/%s' % ref
398 return ref
399
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000400 def GetChangeOwner(self, issue):
401 return self.GetChangeInfo(issue)['owner']['email']
402
403 def GetChangeReviewers(self, issue, approving_only=True):
Aaron Gable8b478f02017-07-31 15:33:19 -0700404 changeinfo = self.GetChangeInfo(issue)
405 if approving_only:
406 labelinfo = changeinfo.get('labels', {}).get('Code-Review', {})
407 values = labelinfo.get('values', {}).keys()
408 try:
409 max_value = max(int(v) for v in values)
410 reviewers = [r for r in labelinfo.get('all', [])
411 if r.get('value', 0) == max_value]
412 except ValueError: # values is the empty list
413 reviewers = []
414 else:
415 reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', [])
416 return [r.get('email') for r in reviewers]
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000417
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000418
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000419class OutputApi(object):
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000420 """An instance of OutputApi gets passed to presubmit scripts so that they
421 can output various types of results.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000422 """
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000423 PresubmitResult = _PresubmitResult
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000424 PresubmitError = _PresubmitError
425 PresubmitPromptWarning = _PresubmitPromptWarning
426 PresubmitNotifyResult = _PresubmitNotifyResult
427 MailTextResult = _MailTextResult
428
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000429 def __init__(self, is_committing):
430 self.is_committing = is_committing
Daniel Cheng7227d212017-11-17 08:12:37 -0800431 self.more_cc = []
432
433 def AppendCC(self, cc):
434 """Appends a user to cc for this change."""
435 self.more_cc.append(cc)
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000436
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000437 def PresubmitPromptOrNotify(self, *args, **kwargs):
438 """Warn the user when uploading, but only notify if committing."""
439 if self.is_committing:
440 return self.PresubmitNotifyResult(*args, **kwargs)
441 return self.PresubmitPromptWarning(*args, **kwargs)
442
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000443
444class InputApi(object):
445 """An instance of this object is passed to presubmit scripts so they can
446 know stuff about the change they're looking at.
447 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000448 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800449 # pylint: disable=no-self-use
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000450
maruel@chromium.org3410d912009-06-09 20:56:16 +0000451 # File extensions that are considered source files from a style guide
452 # perspective. Don't modify this list from a presubmit script!
maruel@chromium.orgc33455a2011-06-24 19:14:18 +0000453 #
454 # Files without an extension aren't included in the list. If you want to
455 # filter them as source files, add r"(^|.*?[\\\/])[^.]+$" to the white list.
456 # Note that ALL CAPS files are black listed in DEFAULT_BLACK_LIST below.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000457 DEFAULT_WHITE_LIST = (
458 # C++ and friends
maruel@chromium.orgfe1211a2011-05-28 18:54:17 +0000459 r".+\.c$", r".+\.cc$", r".+\.cpp$", r".+\.h$", r".+\.m$", r".+\.mm$",
460 r".+\.inl$", r".+\.asm$", r".+\.hxx$", r".+\.hpp$", r".+\.s$", r".+\.S$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000461 # Scripts
maruel@chromium.orgfe1211a2011-05-28 18:54:17 +0000462 r".+\.js$", r".+\.py$", r".+\.sh$", r".+\.rb$", r".+\.pl$", r".+\.pm$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000463 # Other
Sergey Ulanov166bc4c2018-04-30 17:03:38 -0700464 r".+\.java$", r".+\.mk$", r".+\.am$", r".+\.css$", r".+\.mojom$",
465 r".+\.fidl$"
maruel@chromium.org3410d912009-06-09 20:56:16 +0000466 )
467
468 # Path regexp that should be excluded from being considered containing source
469 # files. Don't modify this list from a presubmit script!
470 DEFAULT_BLACK_LIST = (
gavinp@chromium.org656326d2012-08-13 00:43:57 +0000471 r"testing_support[\\\/]google_appengine[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000472 r".*\bexperimental[\\\/].*",
Kent Tamura179dd1e2018-04-26 15:07:41 +0900473 # Exclude third_party/.* but NOT third_party/{WebKit,blink}
474 # (crbug.com/539768 and crbug.com/836555).
475 r".*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000476 # Output directories (just in case)
477 r".*\bDebug[\\\/].*",
478 r".*\bRelease[\\\/].*",
479 r".*\bxcodebuild[\\\/].*",
thakis@chromium.orgc1c96352013-10-09 19:50:27 +0000480 r".*\bout[\\\/].*",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000481 # All caps files like README and LICENCE.
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000482 r".*\b[A-Z0-9_]{2,}$",
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000483 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
maruel@chromium.org5d0dc432011-01-03 02:40:37 +0000484 r"(|.*[\\\/])\.git[\\\/].*",
485 r"(|.*[\\\/])\.svn[\\\/].*",
maruel@chromium.org7ccb4bb2011-11-07 20:26:20 +0000486 # There is no point in processing a patch file.
487 r".+\.diff$",
488 r".+\.patch$",
maruel@chromium.org3410d912009-06-09 20:56:16 +0000489 )
490
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000491 def __init__(self, change, presubmit_path, is_committing,
Edward Lesmes8e282792018-04-03 18:50:29 -0400492 verbose, gerrit_obj, dry_run=None, thread_pool=None, parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000493 """Builds an InputApi object.
494
495 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000496 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000497 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000498 is_committing: True if the change is about to be committed.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000499 gerrit_obj: provides basic Gerrit codereview functionality.
500 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -0400501 parallel: if true, all tests reported via input_api.RunTests for all
502 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000503 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000504 # Version number of the presubmit_support script.
505 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000506 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000507 self.is_committing = is_committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000508 self.gerrit = gerrit_obj
tandrii@chromium.org57bafac2016-04-28 05:09:03 +0000509 self.dry_run = dry_run
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000510
Edward Lesmes8e282792018-04-03 18:50:29 -0400511 self.parallel = parallel
512 self.thread_pool = thread_pool or ThreadPool()
513
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000514 # We expose various modules and functions as attributes of the input_api
515 # so that presubmit scripts don't have to import them.
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +0900516 self.ast = ast
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000517 self.basename = os.path.basename
518 self.cPickle = cPickle
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000519 self.cpplint = cpplint
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000520 self.cStringIO = cStringIO
dcheng091b7db2016-06-16 01:27:51 -0700521 self.fnmatch = fnmatch
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +0000522 self.gclient_paths = gclient_paths
Yoshisato Yanagisawa57dd17b2019-03-22 09:10:29 +0000523 # TODO(yyanagisawa): stop exposing this when python3 become default.
524 # Since python3's tempfile has TemporaryDirectory, we do not need this.
525 self.temporary_directory = gclient_utils.temporary_directory
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000526 self.glob = glob.glob
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000527 self.json = json
maruel@chromium.org6fba34d2011-06-02 13:45:12 +0000528 self.logging = logging.getLogger('PRESUBMIT')
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +0000529 self.marshal = marshal
maruel@chromium.org2b5ce562011-03-31 16:15:44 +0000530 self.os_listdir = os.listdir
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000531 self.os_path = os.path
pgervais@chromium.orgbd0cace2014-10-02 23:23:46 +0000532 self.os_stat = os.stat
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +0000533 self.os_walk = os.walk
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000534 self.pickle = pickle
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000535 self.re = re
536 self.subprocess = subprocess
537 self.tempfile = tempfile
dpranke@chromium.org0d1bdea2011-03-24 22:54:38 +0000538 self.time = time
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000539 self.traceback = traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +0000540 self.unittest = unittest
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000541 self.urllib2 = urllib2
542
Robert Iannucci50258932018-03-19 10:30:59 -0700543 self.is_windows = sys.platform == 'win32'
544
545 # Set python_executable to 'python'. This is interpreted in CallCommand to
546 # convert to vpython in order to allow scripts in other repos (e.g. src.git)
547 # to automatically pick up that repo's .vpython file, instead of inheriting
548 # the one in depot_tools.
549 self.python_executable = 'python'
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000550 self.environ = os.environ
551
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000552 # InputApi.platform is the platform you're currently running on.
553 self.platform = sys.platform
554
iannucci@chromium.org0af3bb32015-06-12 20:44:35 +0000555 self.cpu_count = multiprocessing.cpu_count()
556
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000557 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000558 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000559
560 # We carry the canned checks so presubmit scripts can easily use them.
561 self.canned_checks = presubmit_canned_checks
562
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100563 # Temporary files we must manually remove at the end of a run.
564 self._named_temporary_files = []
Jochen Eisinger72606f82017-04-04 10:44:18 +0200565
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000566 # TODO(dpranke): figure out a list of all approved owners for a repo
567 # in order to be able to handle wildcard OWNERS files?
568 self.owners_db = owners.Database(change.RepositoryRoot(),
Jochen Eisinger72606f82017-04-04 10:44:18 +0200569 fopen=file, os_path=self.os_path)
Jochen Eisinger76f5fc62017-04-07 16:27:46 +0200570 self.owners_finder = owners_finder.OwnersFinder
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000571 self.verbose = verbose
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000572 self.Command = CommandData
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000573
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000574 # Replace <hash_map> and <hash_set> as headers that need to be included
danakj@chromium.org18278522013-06-11 22:42:32 +0000575 # with "base/containers/hash_tables.h" instead.
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000576 # Access to a protected member _XX of a client class
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800577 # pylint: disable=protected-access
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000578 self.cpplint._re_pattern_templates = [
danakj@chromium.org18278522013-06-11 22:42:32 +0000579 (a, b, 'base/containers/hash_tables.h')
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000580 if header in ('<hash_map>', '<hash_set>') else (a, b, header)
581 for (a, b, header) in cpplint._re_pattern_templates
582 ]
583
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000584 def PresubmitLocalPath(self):
585 """Returns the local path of the presubmit script currently being run.
586
587 This is useful if you don't want to hard-code absolute paths in the
588 presubmit script. For example, It can be used to find another file
589 relative to the PRESUBMIT.py script, so the whole tree can be branched and
590 the presubmit script still works, without editing its content.
591 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000592 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000593
agable0b65e732016-11-22 09:25:46 -0800594 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000595 """Same as input_api.change.AffectedFiles() except only lists files
596 (and optionally directories) in the same directory as the current presubmit
597 script, or subdirectories thereof.
598 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000599 dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000600 if len(dir_with_slash) == 1:
601 dir_with_slash = ''
sail@chromium.org5538e022011-05-12 17:53:16 +0000602
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000603 return filter(
604 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
agable0b65e732016-11-22 09:25:46 -0800605 self.change.AffectedFiles(include_deletes, file_filter))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000606
agable0b65e732016-11-22 09:25:46 -0800607 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000608 """Returns local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800609 paths = [af.LocalPath() for af in self.AffectedFiles()]
pgervais@chromium.org2f64f782014-04-25 00:12:33 +0000610 logging.debug("LocalPaths: %s", paths)
611 return paths
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000612
agable0b65e732016-11-22 09:25:46 -0800613 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000614 """Returns absolute local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800615 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000616
John Budorick16162372018-04-18 10:39:53 -0700617 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
agable0b65e732016-11-22 09:25:46 -0800618 """Same as input_api.change.AffectedTestableFiles() except only lists files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000619 in the same directory as the current presubmit script, or subdirectories
620 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000621 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000622 if include_deletes is not None:
agable0b65e732016-11-22 09:25:46 -0800623 warn("AffectedTestableFiles(include_deletes=%s)"
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000624 " is deprecated and ignored" % str(include_deletes),
625 category=DeprecationWarning,
626 stacklevel=2)
agable0b65e732016-11-22 09:25:46 -0800627 return filter(lambda x: x.IsTestableFile(),
John Budorick16162372018-04-18 10:39:53 -0700628 self.AffectedFiles(include_deletes=False, **kwargs))
agable0b65e732016-11-22 09:25:46 -0800629
630 def AffectedTextFiles(self, include_deletes=None):
631 """An alias to AffectedTestableFiles for backwards compatibility."""
632 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000633
maruel@chromium.org3410d912009-06-09 20:56:16 +0000634 def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
635 """Filters out files that aren't considered "source file".
636
637 If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
638 and InputApi.DEFAULT_BLACK_LIST is used respectively.
639
640 The lists will be compiled as regular expression and
641 AffectedFile.LocalPath() needs to pass both list.
642
643 Note: Copy-paste this function to suit your needs or use a lambda function.
644 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000645 def Find(affected_file, items):
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000646 local_path = affected_file.LocalPath()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000647 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000648 if self.re.match(item, local_path):
maruel@chromium.org3410d912009-06-09 20:56:16 +0000649 return True
650 return False
651 return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
652 not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))
653
654 def AffectedSourceFiles(self, source_file):
agable0b65e732016-11-22 09:25:46 -0800655 """Filter the list of AffectedTestableFiles by the function source_file.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000656
657 If source_file is None, InputApi.FilterSourceFile() is used.
658 """
659 if not source_file:
660 source_file = self.FilterSourceFile
agable0b65e732016-11-22 09:25:46 -0800661 return filter(source_file, self.AffectedTestableFiles())
maruel@chromium.org3410d912009-06-09 20:56:16 +0000662
663 def RightHandSideLines(self, source_file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000664 """An iterator over all text lines in "new" version of changed files.
665
666 Only lists lines from new or modified text files in the change that are
667 contained by the directory of the currently executing presubmit script.
668
669 This is useful for doing line-by-line regex checks, like checking for
670 trailing whitespace.
671
672 Yields:
673 a 3 tuple:
674 the AffectedFile instance of the current file;
675 integer line number (1-based); and
676 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000677
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000678 Note: The carriage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000679 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000680 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000681 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000682
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000683 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000684 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000685
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000686 Deny reading anything outside the repository.
687 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000688 if isinstance(file_item, AffectedFile):
689 file_item = file_item.AbsoluteLocalPath()
690 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000691 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000692 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000693
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100694 def CreateTemporaryFile(self, **kwargs):
695 """Returns a named temporary file that must be removed with a call to
696 RemoveTemporaryFiles().
697
698 All keyword arguments are forwarded to tempfile.NamedTemporaryFile(),
699 except for |delete|, which is always set to False.
700
701 Presubmit checks that need to create a temporary file and pass it for
702 reading should use this function instead of NamedTemporaryFile(), as
703 Windows fails to open a file that is already open for writing.
704
705 with input_api.CreateTemporaryFile() as f:
706 f.write('xyz')
707 f.close()
708 input_api.subprocess.check_output(['script-that', '--reads-from',
709 f.name])
710
711
712 Note that callers of CreateTemporaryFile() should not worry about removing
713 any temporary file; this is done transparently by the presubmit handling
714 code.
715 """
716 if 'delete' in kwargs:
717 # Prevent users from passing |delete|; we take care of file deletion
718 # ourselves and this prevents unintuitive error messages when we pass
719 # delete=False and 'delete' is also in kwargs.
720 raise TypeError('CreateTemporaryFile() does not take a "delete" '
721 'argument, file deletion is handled automatically by '
722 'the same presubmit_support code that creates InputApi '
723 'objects.')
724 temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs)
725 self._named_temporary_files.append(temp_file.name)
726 return temp_file
727
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000728 @property
729 def tbr(self):
730 """Returns if a change is TBR'ed."""
Jeremy Romandce22502017-06-20 15:37:29 -0400731 return 'TBR' in self.change.tags or self.change.TBRsFromDescription()
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000732
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000733 def RunTests(self, tests_mix, parallel=True):
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000734 tests = []
735 msgs = []
736 for t in tests_mix:
Edward Lesmes8e282792018-04-03 18:50:29 -0400737 if isinstance(t, OutputApi.PresubmitResult) and t:
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000738 msgs.append(t)
739 else:
740 assert issubclass(t.message, _PresubmitResult)
741 tests.append(t)
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000742 if self.verbose:
743 t.info = _PresubmitNotifyResult
Edward Lemur1037c742018-05-01 18:56:04 -0400744 if not t.kwargs.get('cwd'):
745 t.kwargs['cwd'] = self.PresubmitLocalPath()
Edward Lesmes8e282792018-04-03 18:50:29 -0400746 self.thread_pool.AddTests(tests, parallel)
Edward Lemur21000eb2019-05-24 23:25:58 +0000747 # When self.parallel is True (i.e. --parallel is passed as an option)
748 # RunTests doesn't actually run tests. It adds them to a ThreadPool that
749 # will run all tests once all PRESUBMIT files are processed.
750 # Otherwise, it will run them and return the results.
751 if not self.parallel:
Edward Lesmes8e282792018-04-03 18:50:29 -0400752 msgs.extend(self.thread_pool.RunAsync())
753 return msgs
scottmg86099d72016-09-01 09:16:51 -0700754
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000755
nick@chromium.orgff526192013-06-10 19:30:26 +0000756class _DiffCache(object):
757 """Caches diffs retrieved from a particular SCM."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000758 def __init__(self, upstream=None):
759 """Stores the upstream revision against which all diffs will be computed."""
760 self._upstream = upstream
nick@chromium.orgff526192013-06-10 19:30:26 +0000761
762 def GetDiff(self, path, local_root):
763 """Get the diff for a particular path."""
764 raise NotImplementedError()
765
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700766 def GetOldContents(self, path, local_root):
767 """Get the old version for a particular path."""
768 raise NotImplementedError()
769
nick@chromium.orgff526192013-06-10 19:30:26 +0000770
nick@chromium.orgff526192013-06-10 19:30:26 +0000771class _GitDiffCache(_DiffCache):
772 """DiffCache implementation for git; gets all file diffs at once."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000773 def __init__(self, upstream):
774 super(_GitDiffCache, self).__init__(upstream=upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000775 self._diffs_by_file = None
776
777 def GetDiff(self, path, local_root):
778 if not self._diffs_by_file:
779 # Compute a single diff for all files and parse the output; should
780 # with git this is much faster than computing one diff for each file.
781 diffs = {}
782
783 # Don't specify any filenames below, because there are command line length
784 # limits on some platforms and GenerateDiff would fail.
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000785 unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True,
786 branch=self._upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000787
788 # This regex matches the path twice, separated by a space. Note that
789 # filename itself may contain spaces.
790 file_marker = re.compile('^diff --git (?P<filename>.*) (?P=filename)$')
791 current_diff = []
792 keep_line_endings = True
793 for x in unified_diff.splitlines(keep_line_endings):
794 match = file_marker.match(x)
795 if match:
796 # Marks the start of a new per-file section.
797 diffs[match.group('filename')] = current_diff = [x]
798 elif x.startswith('diff --git'):
799 raise PresubmitFailure('Unexpected diff line: %s' % x)
800 else:
801 current_diff.append(x)
802
803 self._diffs_by_file = dict(
804 (normpath(path), ''.join(diff)) for path, diff in diffs.items())
805
806 if path not in self._diffs_by_file:
807 raise PresubmitFailure(
808 'Unified diff did not contain entry for file %s' % path)
809
810 return self._diffs_by_file[path]
811
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700812 def GetOldContents(self, path, local_root):
813 return scm.GIT.GetOldContents(local_root, path, branch=self._upstream)
814
nick@chromium.orgff526192013-06-10 19:30:26 +0000815
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000816class AffectedFile(object):
817 """Representation of a file in a change."""
nick@chromium.orgff526192013-06-10 19:30:26 +0000818
819 DIFF_CACHE = _DiffCache
820
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000821 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800822 # pylint: disable=no-self-use
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000823 def __init__(self, path, action, repository_root, diff_cache):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000824 self._path = path
825 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000826 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000827 self._is_directory = None
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000828 self._cached_changed_contents = None
829 self._cached_new_contents = None
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000830 self._diff_cache = diff_cache
tobiasjs2836bcf2016-08-16 04:08:16 -0700831 logging.debug('%s(%s)', self.__class__.__name__, self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000832
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000833 def LocalPath(self):
834 """Returns the path of this file on the local disk relative to client root.
Andrew Grieve92b8b992017-11-02 09:42:24 -0400835
836 This should be used for error messages but not for accessing files,
837 because presubmit checks are run with CWD=PresubmitLocalPath() (which is
838 often != client root).
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000839 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000840 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000841
842 def AbsoluteLocalPath(self):
843 """Returns the absolute path of this file on the local disk.
844 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000845 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000846
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000847 def Action(self):
848 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000849 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000850
agable0b65e732016-11-22 09:25:46 -0800851 def IsTestableFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000852 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000853
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000854 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000855 raise NotImplementedError() # Implement when needed
856
agable0b65e732016-11-22 09:25:46 -0800857 def IsTextFile(self):
858 """An alias to IsTestableFile for backwards compatibility."""
859 return self.IsTestableFile()
860
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700861 def OldContents(self):
862 """Returns an iterator over the lines in the old version of file.
863
Daniel Cheng2da34fe2017-03-21 20:42:12 -0700864 The old version is the file before any modifications in the user's
865 workspace, i.e. the "left hand side".
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700866
867 Contents will be empty if the file is a directory or does not exist.
868 Note: The carriage returns (LF or CR) are stripped off.
869 """
870 return self._diff_cache.GetOldContents(self.LocalPath(),
871 self._local_root).splitlines()
872
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000873 def NewContents(self):
874 """Returns an iterator over the lines in the new version of file.
875
876 The new version is the file in the user's workspace, i.e. the "right hand
877 side".
878
879 Contents will be empty if the file is a directory or does not exist.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000880 Note: The carriage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000881 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000882 if self._cached_new_contents is None:
883 self._cached_new_contents = []
agable0b65e732016-11-22 09:25:46 -0800884 try:
885 self._cached_new_contents = gclient_utils.FileRead(
886 self.AbsoluteLocalPath(), 'rU').splitlines()
887 except IOError:
888 pass # File not found? That's fine; maybe it was deleted.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000889 return self._cached_new_contents[:]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000890
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000891 def ChangedContents(self):
892 """Returns a list of tuples (line number, line text) of all new lines.
893
894 This relies on the scm diff output describing each changed code section
895 with a line of the form
896
897 ^@@ <old line num>,<old size> <new line num>,<new size> @@$
898 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000899 if self._cached_changed_contents is not None:
900 return self._cached_changed_contents[:]
901 self._cached_changed_contents = []
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000902 line_num = 0
903
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000904 for line in self.GenerateScmDiff().splitlines():
905 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
906 if m:
907 line_num = int(m.groups(1)[0])
908 continue
909 if line.startswith('+') and not line.startswith('++'):
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000910 self._cached_changed_contents.append((line_num, line[1:]))
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000911 if not line.startswith('-'):
912 line_num += 1
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000913 return self._cached_changed_contents[:]
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000914
maruel@chromium.org5de13972009-06-10 18:16:06 +0000915 def __str__(self):
916 return self.LocalPath()
917
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000918 def GenerateScmDiff(self):
nick@chromium.orgff526192013-06-10 19:30:26 +0000919 return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000920
maruel@chromium.org58407af2011-04-12 23:15:57 +0000921
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000922class GitAffectedFile(AffectedFile):
923 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000924 # Method 'NNN' is abstract in class 'NNN' but is not overridden
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800925 # pylint: disable=abstract-method
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000926
nick@chromium.orgff526192013-06-10 19:30:26 +0000927 DIFF_CACHE = _GitDiffCache
928
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000929 def __init__(self, *args, **kwargs):
930 AffectedFile.__init__(self, *args, **kwargs)
931 self._server_path = None
agable0b65e732016-11-22 09:25:46 -0800932 self._is_testable_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000933
agable0b65e732016-11-22 09:25:46 -0800934 def IsTestableFile(self):
935 if self._is_testable_file is None:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000936 if self.Action() == 'D':
agable0b65e732016-11-22 09:25:46 -0800937 # A deleted file is not testable.
938 self._is_testable_file = False
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000939 else:
agable0b65e732016-11-22 09:25:46 -0800940 self._is_testable_file = os.path.isfile(self.AbsoluteLocalPath())
941 return self._is_testable_file
maruel@chromium.orgc70a2202009-06-17 12:55:10 +0000942
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000943
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000944class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000945 """Describe a change.
946
947 Used directly by the presubmit scripts to query the current change being
948 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000949
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000950 Instance members:
nick@chromium.orgff526192013-06-10 19:30:26 +0000951 tags: Dictionary of KEY=VALUE pairs found in the change description.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000952 self.KEY: equivalent to tags['KEY']
953 """
954
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000955 _AFFECTED_FILES = AffectedFile
956
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000957 # Matches key/value (or "tag") lines in changelist descriptions.
maruel@chromium.org428342a2011-11-10 15:46:33 +0000958 TAG_LINE_RE = re.compile(
maruel@chromium.orgc6f60e82013-04-19 17:01:57 +0000959 '^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
maruel@chromium.orgc1938752011-04-12 23:11:13 +0000960 scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000961
maruel@chromium.org58407af2011-04-12 23:15:57 +0000962 def __init__(
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000963 self, name, description, local_root, files, issue, patchset, author,
964 upstream=None):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000965 if files is None:
966 files = []
967 self._name = name
chase@chromium.org8e416c82009-10-06 04:30:44 +0000968 # Convert root into an absolute path.
969 self._local_root = os.path.abspath(local_root)
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000970 self._upstream = upstream
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000971 self.issue = issue
972 self.patchset = patchset
maruel@chromium.org58407af2011-04-12 23:15:57 +0000973 self.author_email = author
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000974
isherman@chromium.orgb5cded62014-03-25 17:47:57 +0000975 self._full_description = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000976 self.tags = {}
isherman@chromium.orgb5cded62014-03-25 17:47:57 +0000977 self._description_without_tags = ''
978 self.SetDescriptionText(description)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000979
maruel@chromium.orge085d812011-10-10 19:49:15 +0000980 assert all(
981 (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files
982
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000983 diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream)
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000984 self._affected_files = [
nick@chromium.orgff526192013-06-10 19:30:26 +0000985 self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache)
986 for action, path in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +0000987 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000988
maruel@chromium.org92022ec2009-06-11 01:59:28 +0000989 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000990 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +0000991 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000992
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000993 def DescriptionText(self):
994 """Returns the user-entered changelist description, minus tags.
995
996 Any line in the user-provided description starting with e.g. "FOO="
997 (whitespace permitted before and around) is considered a tag line. Such
998 lines are stripped out of the description this function returns.
999 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001000 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001001
1002 def FullDescriptionText(self):
1003 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001004 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001005
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001006 def SetDescriptionText(self, description):
1007 """Sets the full description text (including tags) to |description|.
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001008
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001009 Also updates the list of tags."""
1010 self._full_description = description
1011
1012 # From the description text, build up a dictionary of key/value pairs
1013 # plus the description minus all key/value or "tag" lines.
1014 description_without_tags = []
1015 self.tags = {}
1016 for line in self._full_description.splitlines():
1017 m = self.TAG_LINE_RE.match(line)
1018 if m:
1019 self.tags[m.group('key')] = m.group('value')
1020 else:
1021 description_without_tags.append(line)
1022
1023 # Change back to text and remove whitespace at end.
1024 self._description_without_tags = (
1025 '\n'.join(description_without_tags).rstrip())
1026
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001027 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001028 """Returns the repository (checkout) root directory for this change,
1029 as an absolute path.
1030 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001031 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001032
1033 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001034 """Return tags directly as attributes on the object."""
1035 if not re.match(r"^[A-Z_]*$", attr):
1036 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +00001037 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001038
Aaron Gablefc03e672017-05-15 14:09:42 -07001039 def BugsFromDescription(self):
1040 """Returns all bugs referenced in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001041 tags = [b.strip() for b in self.tags.get('BUG', '').split(',') if b.strip()]
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001042 footers = []
1043 unsplit_footers = git_footers.parse_footers(self._full_description).get(
1044 'Bug', [])
1045 for unsplit_footer in unsplit_footers:
1046 footers += [b.strip() for b in unsplit_footer.split(',')]
Aaron Gable12ef5012017-05-15 14:29:00 -07001047 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001048
1049 def ReviewersFromDescription(self):
1050 """Returns all reviewers listed in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001051 # We don't support a "R:" git-footer for reviewers; that is in metadata.
1052 tags = [r.strip() for r in self.tags.get('R', '').split(',') if r.strip()]
1053 return sorted(set(tags))
Aaron Gablefc03e672017-05-15 14:09:42 -07001054
1055 def TBRsFromDescription(self):
1056 """Returns all TBR reviewers listed in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001057 tags = [r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()]
1058 # TODO(agable): Remove support for 'Tbr:' when TBRs are programmatically
1059 # determined by self-CR+1s.
1060 footers = git_footers.parse_footers(self._full_description).get('Tbr', [])
1061 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001062
1063 # TODO(agable): Delete these once we're sure they're unused.
1064 @property
1065 def BUG(self):
1066 return ','.join(self.BugsFromDescription())
1067 @property
1068 def R(self):
1069 return ','.join(self.ReviewersFromDescription())
1070 @property
1071 def TBR(self):
1072 return ','.join(self.TBRsFromDescription())
1073
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001074 def AllFiles(self, root=None):
1075 """List all files under source control in the repo."""
1076 raise NotImplementedError()
1077
agable0b65e732016-11-22 09:25:46 -08001078 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001079 """Returns a list of AffectedFile instances for all files in the change.
1080
1081 Args:
1082 include_deletes: If false, deleted files will be filtered out.
sail@chromium.org5538e022011-05-12 17:53:16 +00001083 file_filter: An additional filter to apply.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001084
1085 Returns:
1086 [AffectedFile(path, action), AffectedFile(path, action)]
1087 """
agable0b65e732016-11-22 09:25:46 -08001088 affected = filter(file_filter, self._affected_files)
sail@chromium.org5538e022011-05-12 17:53:16 +00001089
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001090 if include_deletes:
1091 return affected
Lei Zhang9611c4c2017-04-04 01:41:56 -07001092 return filter(lambda x: x.Action() != 'D', affected)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001093
John Budorick16162372018-04-18 10:39:53 -07001094 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001095 """Return a list of the existing text files in a change."""
1096 if include_deletes is not None:
agable0b65e732016-11-22 09:25:46 -08001097 warn("AffectedTeestableFiles(include_deletes=%s)"
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001098 " is deprecated and ignored" % str(include_deletes),
1099 category=DeprecationWarning,
1100 stacklevel=2)
agable0b65e732016-11-22 09:25:46 -08001101 return filter(lambda x: x.IsTestableFile(),
John Budorick16162372018-04-18 10:39:53 -07001102 self.AffectedFiles(include_deletes=False, **kwargs))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001103
agable0b65e732016-11-22 09:25:46 -08001104 def AffectedTextFiles(self, include_deletes=None):
1105 """An alias to AffectedTestableFiles for backwards compatibility."""
1106 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001107
agable0b65e732016-11-22 09:25:46 -08001108 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001109 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001110 return [af.LocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001111
agable0b65e732016-11-22 09:25:46 -08001112 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001113 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001114 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001115
1116 def RightHandSideLines(self):
1117 """An iterator over all text lines in "new" version of changed files.
1118
1119 Lists lines from new or modified text files in the change.
1120
1121 This is useful for doing line-by-line regex checks, like checking for
1122 trailing whitespace.
1123
1124 Yields:
1125 a 3 tuple:
1126 the AffectedFile instance of the current file;
1127 integer line number (1-based); and
1128 the contents of the line as a string.
1129 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001130 return _RightHandSideLinesImpl(
1131 x for x in self.AffectedFiles(include_deletes=False)
agable0b65e732016-11-22 09:25:46 -08001132 if x.IsTestableFile())
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001133
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001134 def OriginalOwnersFiles(self):
1135 """A map from path names of affected OWNERS files to their old content."""
1136 def owners_file_filter(f):
1137 return 'OWNERS' in os.path.split(f.LocalPath())[1]
1138 files = self.AffectedFiles(file_filter=owners_file_filter)
1139 return dict([(f.LocalPath(), f.OldContents()) for f in files])
1140
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001141
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001142class GitChange(Change):
1143 _AFFECTED_FILES = GitAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001144 scm = 'git'
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001145
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001146 def AllFiles(self, root=None):
1147 """List all files under source control in the repo."""
1148 root = root or self.RepositoryRoot()
1149 return subprocess.check_output(
Aaron Gable7817f022017-12-12 09:43:17 -08001150 ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'],
1151 cwd=root).splitlines()
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001152
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001153
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001154def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001155 """Finds all presubmit files that apply to a given set of source files.
1156
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001157 If inherit-review-settings-ok is present right under root, looks for
1158 PRESUBMIT.py in directories enclosing root.
1159
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001160 Args:
1161 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001162 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001163
1164 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001165 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001166 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001167 files = [normpath(os.path.join(root, f)) for f in files]
1168
1169 # List all the individual directories containing files.
1170 directories = set([os.path.dirname(f) for f in files])
1171
1172 # Ignore root if inherit-review-settings-ok is present.
1173 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
1174 root = None
1175
1176 # Collect all unique directories that may contain PRESUBMIT.py.
1177 candidates = set()
1178 for directory in directories:
1179 while True:
1180 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001181 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001182 candidates.add(directory)
1183 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001184 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001185 parent_dir = os.path.dirname(directory)
1186 if parent_dir == directory:
1187 # We hit the system root directory.
1188 break
1189 directory = parent_dir
1190
1191 # Look for PRESUBMIT.py in all candidate directories.
1192 results = []
1193 for directory in sorted(list(candidates)):
tobiasjsff061c02016-08-17 03:23:57 -07001194 try:
1195 for f in os.listdir(directory):
1196 p = os.path.join(directory, f)
1197 if os.path.isfile(p) and re.match(
1198 r'PRESUBMIT.*\.py$', f) and not f.startswith('PRESUBMIT_test'):
1199 results.append(p)
1200 except OSError:
1201 pass
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001202
tobiasjs2836bcf2016-08-16 04:08:16 -07001203 logging.debug('Presubmit files: %s', ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001204 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001205
1206
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001207class GetTryMastersExecuter(object):
1208 @staticmethod
1209 def ExecPresubmitScript(script_text, presubmit_path, project, change):
1210 """Executes GetPreferredTryMasters() from a single presubmit script.
1211
1212 Args:
1213 script_text: The text of the presubmit script.
1214 presubmit_path: Project script to run.
1215 project: Project name to pass to presubmit script for bot selection.
1216
1217 Return:
1218 A map of try masters to map of builders to set of tests.
1219 """
1220 context = {}
1221 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001222 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1223 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001224 except Exception as e:
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001225 raise PresubmitFailure('"%s" had an exception.\n%s'
1226 % (presubmit_path, e))
1227
1228 function_name = 'GetPreferredTryMasters'
1229 if function_name not in context:
1230 return {}
1231 get_preferred_try_masters = context[function_name]
1232 if not len(inspect.getargspec(get_preferred_try_masters)[0]) == 2:
1233 raise PresubmitFailure(
1234 'Expected function "GetPreferredTryMasters" to take two arguments.')
1235 return get_preferred_try_masters(project, change)
1236
1237
rmistry@google.com5626a922015-02-26 14:03:30 +00001238class GetPostUploadExecuter(object):
1239 @staticmethod
1240 def ExecPresubmitScript(script_text, presubmit_path, cl, change):
1241 """Executes PostUploadHook() from a single presubmit script.
1242
1243 Args:
1244 script_text: The text of the presubmit script.
1245 presubmit_path: Project script to run.
1246 cl: The Changelist object.
1247 change: The Change object.
1248
1249 Return:
1250 A list of results objects.
1251 """
1252 context = {}
1253 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001254 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1255 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001256 except Exception as e:
rmistry@google.com5626a922015-02-26 14:03:30 +00001257 raise PresubmitFailure('"%s" had an exception.\n%s'
1258 % (presubmit_path, e))
1259
1260 function_name = 'PostUploadHook'
1261 if function_name not in context:
1262 return {}
1263 post_upload_hook = context[function_name]
1264 if not len(inspect.getargspec(post_upload_hook)[0]) == 3:
1265 raise PresubmitFailure(
1266 'Expected function "PostUploadHook" to take three arguments.')
1267 return post_upload_hook(cl, change, OutputApi(False))
1268
1269
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001270def _MergeMasters(masters1, masters2):
1271 """Merges two master maps. Merges also the tests of each builder."""
1272 result = {}
1273 for (master, builders) in itertools.chain(masters1.iteritems(),
1274 masters2.iteritems()):
1275 new_builders = result.setdefault(master, {})
1276 for (builder, tests) in builders.iteritems():
1277 new_builders.setdefault(builder, set([])).update(tests)
1278 return result
1279
1280
1281def DoGetTryMasters(change,
1282 changed_files,
1283 repository_root,
1284 default_presubmit,
1285 project,
1286 verbose,
1287 output_stream):
1288 """Get the list of try masters from the presubmit scripts.
1289
1290 Args:
1291 changed_files: List of modified files.
1292 repository_root: The repository root.
1293 default_presubmit: A default presubmit script to execute in any case.
1294 project: Optional name of a project used in selecting trybots.
1295 verbose: Prints debug info.
1296 output_stream: A stream to write debug output to.
1297
1298 Return:
1299 Map of try masters to map of builders to set of tests.
1300 """
1301 presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
1302 if not presubmit_files and verbose:
1303 output_stream.write("Warning, no PRESUBMIT.py found.\n")
1304 results = {}
1305 executer = GetTryMastersExecuter()
1306
1307 if default_presubmit:
1308 if verbose:
1309 output_stream.write("Running default presubmit script.\n")
1310 fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
1311 results = _MergeMasters(results, executer.ExecPresubmitScript(
1312 default_presubmit, fake_path, project, change))
1313 for filename in presubmit_files:
1314 filename = os.path.abspath(filename)
1315 if verbose:
1316 output_stream.write("Running %s\n" % filename)
1317 # Accept CRLF presubmit script.
1318 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1319 results = _MergeMasters(results, executer.ExecPresubmitScript(
1320 presubmit_script, filename, project, change))
1321
1322 # Make sets to lists again for later JSON serialization.
1323 for builders in results.itervalues():
1324 for builder in builders:
1325 builders[builder] = list(builders[builder])
1326
1327 if results and verbose:
1328 output_stream.write('%s\n' % str(results))
1329 return results
1330
1331
rmistry@google.com5626a922015-02-26 14:03:30 +00001332def DoPostUploadExecuter(change,
1333 cl,
1334 repository_root,
1335 verbose,
1336 output_stream):
1337 """Execute the post upload hook.
1338
1339 Args:
1340 change: The Change object.
1341 cl: The Changelist object.
1342 repository_root: The repository root.
1343 verbose: Prints debug info.
1344 output_stream: A stream to write debug output to.
1345 """
1346 presubmit_files = ListRelevantPresubmitFiles(
1347 change.LocalPaths(), repository_root)
1348 if not presubmit_files and verbose:
1349 output_stream.write("Warning, no PRESUBMIT.py found.\n")
1350 results = []
1351 executer = GetPostUploadExecuter()
1352 # The root presubmit file should be executed after the ones in subdirectories.
1353 # i.e. the specific post upload hooks should run before the general ones.
1354 # Thus, reverse the order provided by ListRelevantPresubmitFiles.
1355 presubmit_files.reverse()
1356
1357 for filename in presubmit_files:
1358 filename = os.path.abspath(filename)
1359 if verbose:
1360 output_stream.write("Running %s\n" % filename)
1361 # Accept CRLF presubmit script.
1362 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1363 results.extend(executer.ExecPresubmitScript(
1364 presubmit_script, filename, cl, change))
1365 output_stream.write('\n')
1366 if results:
1367 output_stream.write('** Post Upload Hook Messages **\n')
1368 for result in results:
1369 result.handle(output_stream)
1370 output_stream.write('\n')
1371
1372 return results
1373
1374
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001375class PresubmitExecuter(object):
Aaron Gable668c1d82018-04-03 10:19:16 -07001376 def __init__(self, change, committing, verbose,
Edward Lesmes8e282792018-04-03 18:50:29 -04001377 gerrit_obj, dry_run=None, thread_pool=None, parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001378 """
1379 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001380 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001381 committing: True if 'git cl land' is running, False if 'git cl upload' is.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001382 gerrit_obj: provides basic Gerrit codereview functionality.
1383 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001384 parallel: if true, all tests reported via input_api.RunTests for all
1385 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001386 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001387 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001388 self.committing = committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001389 self.gerrit = gerrit_obj
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001390 self.verbose = verbose
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001391 self.dry_run = dry_run
Daniel Cheng7227d212017-11-17 08:12:37 -08001392 self.more_cc = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001393 self.thread_pool = thread_pool
1394 self.parallel = parallel
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001395
1396 def ExecPresubmitScript(self, script_text, presubmit_path):
1397 """Executes a single presubmit script.
1398
1399 Args:
1400 script_text: The text of the presubmit script.
1401 presubmit_path: The path to the presubmit file (this will be reported via
1402 input_api.PresubmitLocalPath()).
1403
1404 Return:
1405 A list of result objects, empty if no problems.
1406 """
thakis@chromium.orgc6ef53a2014-11-04 00:13:54 +00001407
chase@chromium.org8e416c82009-10-06 04:30:44 +00001408 # Change to the presubmit file's directory to support local imports.
1409 main_path = os.getcwd()
1410 os.chdir(os.path.dirname(presubmit_path))
1411
1412 # Load the presubmit script into context.
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001413 input_api = InputApi(self.change, presubmit_path, self.committing,
Aaron Gable668c1d82018-04-03 10:19:16 -07001414 self.verbose, gerrit_obj=self.gerrit,
Edward Lesmes8e282792018-04-03 18:50:29 -04001415 dry_run=self.dry_run, thread_pool=self.thread_pool,
1416 parallel=self.parallel)
Daniel Cheng7227d212017-11-17 08:12:37 -08001417 output_api = OutputApi(self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001418 context = {}
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001419 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001420 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1421 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001422 except Exception as e:
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001423 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001424
1425 # These function names must change if we make substantial changes to
1426 # the presubmit API that are not backwards compatible.
1427 if self.committing:
1428 function_name = 'CheckChangeOnCommit'
1429 else:
1430 function_name = 'CheckChangeOnUpload'
1431 if function_name in context:
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +01001432 try:
Daniel Cheng7227d212017-11-17 08:12:37 -08001433 context['__args'] = (input_api, output_api)
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +01001434 logging.debug('Running %s in %s', function_name, presubmit_path)
1435 result = eval(function_name + '(*__args)', context)
1436 logging.debug('Running %s done.', function_name)
Daniel Chengd36fce42017-11-21 21:52:52 -08001437 self.more_cc.extend(output_api.more_cc)
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +01001438 finally:
1439 map(os.remove, input_api._named_temporary_files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001440 if not (isinstance(result, types.TupleType) or
1441 isinstance(result, types.ListType)):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001442 raise PresubmitFailure(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001443 'Presubmit functions must return a tuple or list')
1444 for item in result:
1445 if not isinstance(item, OutputApi.PresubmitResult):
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001446 raise PresubmitFailure(
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001447 'All presubmit results must be of types derived from '
1448 'output_api.PresubmitResult')
1449 else:
1450 result = () # no error since the script doesn't care about current event.
1451
chase@chromium.org8e416c82009-10-06 04:30:44 +00001452 # Return the process to the original working directory.
1453 os.chdir(main_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001454 return result
1455
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001456def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001457 committing,
1458 verbose,
1459 output_stream,
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001460 input_stream,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001461 default_presubmit,
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001462 may_prompt,
Aaron Gable668c1d82018-04-03 10:19:16 -07001463 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001464 dry_run=None,
1465 parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001466 """Runs all presubmit checks that apply to the files in the change.
1467
1468 This finds all PRESUBMIT.py files in directories enclosing the files in the
1469 change (up to the repository root) and calls the relevant entrypoint function
1470 depending on whether the change is being committed or uploaded.
1471
1472 Prints errors, warnings and notifications. Prompts the user for warnings
1473 when needed.
1474
1475 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001476 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001477 committing: True if 'git cl land' is running, False if 'git cl upload' is.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001478 verbose: Prints debug info.
1479 output_stream: A stream to write output from presubmit tests to.
1480 input_stream: A stream to read input from the user.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001481 default_presubmit: A default presubmit script to execute in any case.
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001482 may_prompt: Enable (y/n) questions on warning or error. If False,
1483 any questions are answered with yes by default.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001484 gerrit_obj: provides basic Gerrit codereview functionality.
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001485 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001486 parallel: if true, all tests specified by input_api.RunTests in all
1487 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001488
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001489 Warning:
1490 If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream
1491 SHOULD be sys.stdin.
1492
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001493 Return:
dpranke@chromium.org5ac21012011-03-16 02:58:25 +00001494 A PresubmitOutput object. Use output.should_continue() to figure out
1495 if there were errors or warnings and the caller should abort.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001496 """
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001497 old_environ = os.environ
1498 try:
1499 # Make sure python subprocesses won't generate .pyc files.
1500 os.environ = os.environ.copy()
1501 os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001502
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001503 output = PresubmitOutput(input_stream, output_stream)
1504 if committing:
1505 output.write("Running presubmit commit checks ...\n")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001506 else:
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001507 output.write("Running presubmit upload checks ...\n")
1508 start_time = time.time()
1509 presubmit_files = ListRelevantPresubmitFiles(
agable0b65e732016-11-22 09:25:46 -08001510 change.AbsoluteLocalPaths(), change.RepositoryRoot())
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001511 if not presubmit_files and verbose:
maruel@chromium.orgfae707b2011-09-15 18:57:58 +00001512 output.write("Warning, no PRESUBMIT.py found.\n")
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001513 results = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001514 thread_pool = ThreadPool()
Edward Lemur7e3c67f2018-07-20 20:52:49 +00001515 executer = PresubmitExecuter(change, committing, verbose, gerrit_obj,
1516 dry_run, thread_pool, parallel)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001517 if default_presubmit:
1518 if verbose:
1519 output.write("Running default presubmit script.\n")
1520 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
1521 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
1522 for filename in presubmit_files:
1523 filename = os.path.abspath(filename)
1524 if verbose:
1525 output.write("Running %s\n" % filename)
1526 # Accept CRLF presubmit script.
1527 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1528 results += executer.ExecPresubmitScript(presubmit_script, filename)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001529
Edward Lesmes8e282792018-04-03 18:50:29 -04001530 results += thread_pool.RunAsync()
1531
Daniel Cheng7227d212017-11-17 08:12:37 -08001532 output.more_cc.extend(executer.more_cc)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001533 errors = []
1534 notifications = []
1535 warnings = []
1536 for result in results:
1537 if result.fatal:
1538 errors.append(result)
1539 elif result.should_prompt:
1540 warnings.append(result)
1541 else:
1542 notifications.append(result)
pam@chromium.orged9a0832009-09-09 22:48:55 +00001543
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001544 output.write('\n')
1545 for name, items in (('Messages', notifications),
1546 ('Warnings', warnings),
1547 ('ERRORS', errors)):
1548 if items:
1549 output.write('** Presubmit %s **\n' % name)
1550 for item in items:
1551 item.handle(output)
1552 output.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001553
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001554 total_time = time.time() - start_time
1555 if total_time > 1.0:
1556 output.write("Presubmit checks took %.1fs to calculate.\n\n" % total_time)
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001557
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001558 if errors:
1559 output.fail()
1560 elif warnings:
1561 output.write('There were presubmit warnings. ')
1562 if may_prompt:
1563 output.prompt_yes_no('Are you sure you wish to continue? (y/N): ')
1564 else:
1565 output.write('Presubmit checks passed.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001566
1567 global _ASKED_FOR_FEEDBACK
1568 # Ask for feedback one time out of 5.
1569 if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
maruel@chromium.org1ce8e662014-01-14 15:23:00 +00001570 output.write(
1571 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n'
1572 'to figure out which PRESUBMIT.py was run, then run git blame\n'
1573 'on the file to figure out who to ask for help.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001574 _ASKED_FOR_FEEDBACK = True
1575 return output
1576 finally:
1577 os.environ = old_environ
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001578
1579
1580def ScanSubDirs(mask, recursive):
1581 if not recursive:
pgervais@chromium.orge57b09d2014-05-07 00:58:13 +00001582 return [x for x in glob.glob(mask) if x not in ('.svn', '.git')]
Lei Zhang9611c4c2017-04-04 01:41:56 -07001583
1584 results = []
1585 for root, dirs, files in os.walk('.'):
1586 if '.svn' in dirs:
1587 dirs.remove('.svn')
1588 if '.git' in dirs:
1589 dirs.remove('.git')
1590 for name in files:
1591 if fnmatch.fnmatch(name, mask):
1592 results.append(os.path.join(root, name))
1593 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001594
1595
1596def ParseFiles(args, recursive):
tobiasjs2836bcf2016-08-16 04:08:16 -07001597 logging.debug('Searching for %s', args)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001598 files = []
1599 for arg in args:
maruel@chromium.orge3608df2009-11-10 20:22:57 +00001600 files.extend([('M', f) for f in ScanSubDirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001601 return files
1602
1603
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001604def load_files(options, args):
1605 """Tries to determine the SCM."""
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001606 files = []
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001607 if args:
1608 files = ParseFiles(args, options.recursive)
agable0b65e732016-11-22 09:25:46 -08001609 change_scm = scm.determine_scm(options.root)
1610 if change_scm == 'git':
maruel@chromium.org9b31f162012-01-26 19:02:31 +00001611 change_class = GitChange
agable@chromium.org2da1ade2014-04-30 17:40:45 +00001612 upstream = options.upstream or None
maruel@chromium.org9b31f162012-01-26 19:02:31 +00001613 if not files:
agable@chromium.org2da1ade2014-04-30 17:40:45 +00001614 files = scm.GIT.CaptureStatus([], options.root, upstream)
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001615 else:
tobiasjs2836bcf2016-08-16 04:08:16 -07001616 logging.info('Doesn\'t seem under source control. Got %d files', len(args))
maruel@chromium.org9b31f162012-01-26 19:02:31 +00001617 if not files:
1618 return None, None
1619 change_class = Change
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001620 return change_class, files
1621
1622
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001623@contextlib.contextmanager
1624def canned_check_filter(method_names):
1625 filtered = {}
1626 try:
1627 for method_name in method_names:
1628 if not hasattr(presubmit_canned_checks, method_name):
Aaron Gableecee74c2018-04-02 15:13:08 -07001629 logging.warn('Skipping unknown "canned" check %s' % method_name)
1630 continue
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001631 filtered[method_name] = getattr(presubmit_canned_checks, method_name)
1632 setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
1633 yield
1634 finally:
1635 for name, method in filtered.iteritems():
1636 setattr(presubmit_canned_checks, name, method)
1637
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +00001638
sbc@chromium.org013731e2015-02-26 18:28:43 +00001639def main(argv=None):
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001640 parser = optparse.OptionParser(usage="%prog [options] <files...>",
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001641 version="%prog " + str(__version__))
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001642 parser.add_option("-c", "--commit", action="store_true", default=False,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001643 help="Use commit instead of upload checks")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001644 parser.add_option("-u", "--upload", action="store_false", dest='commit',
1645 help="Use upload instead of commit checks")
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001646 parser.add_option("-r", "--recursive", action="store_true",
1647 help="Act recursively")
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001648 parser.add_option("-v", "--verbose", action="count", default=0,
1649 help="Use 2 times for more debug info")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001650 parser.add_option("--name", default='no name')
maruel@chromium.org58407af2011-04-12 23:15:57 +00001651 parser.add_option("--author")
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001652 parser.add_option("--description", default='')
1653 parser.add_option("--issue", type='int', default=0)
1654 parser.add_option("--patchset", type='int', default=0)
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001655 parser.add_option("--root", default=os.getcwd(),
1656 help="Search for PRESUBMIT.py up to this directory. "
1657 "If inherit-review-settings-ok is present in this "
1658 "directory, parent directories up to the root file "
1659 "system directories will also be searched.")
agable@chromium.org2da1ade2014-04-30 17:40:45 +00001660 parser.add_option("--upstream",
1661 help="Git only: the base ref or upstream branch against "
1662 "which the diff should be computed.")
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001663 parser.add_option("--default_presubmit")
1664 parser.add_option("--may_prompt", action='store_true', default=False)
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001665 parser.add_option("--skip_canned", action='append', default=[],
1666 help="A list of checks to skip which appear in "
1667 "presubmit_canned_checks. Can be provided multiple times "
1668 "to skip multiple canned checks.")
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001669 parser.add_option("--dry_run", action='store_true',
1670 help=optparse.SUPPRESS_HELP)
tandrii@chromium.org015ebae2016-04-25 19:37:22 +00001671 parser.add_option("--gerrit_url", help=optparse.SUPPRESS_HELP)
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001672 parser.add_option("--gerrit_fetch", action='store_true',
1673 help=optparse.SUPPRESS_HELP)
Edward Lesmes8e282792018-04-03 18:50:29 -04001674 parser.add_option('--parallel', action='store_true',
1675 help='Run all tests specified by input_api.RunTests in all '
1676 'PRESUBMIT files in parallel.')
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001677
maruel@chromium.org82e5f282011-03-17 14:08:55 +00001678 options, args = parser.parse_args(argv)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001679
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001680 if options.verbose >= 2:
maruel@chromium.org7444c502011-02-09 14:02:11 +00001681 logging.basicConfig(level=logging.DEBUG)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001682 elif options.verbose:
1683 logging.basicConfig(level=logging.INFO)
1684 else:
1685 logging.basicConfig(level=logging.ERROR)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001686
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001687 change_class, files = load_files(options, args)
1688 if not change_class:
1689 parser.error('For unversioned directory, <files> is not optional.')
tobiasjs2836bcf2016-08-16 04:08:16 -07001690 logging.info('Found %d file(s).', len(files))
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001691
Aaron Gable668c1d82018-04-03 10:19:16 -07001692 gerrit_obj = None
tandrii@chromium.org015ebae2016-04-25 19:37:22 +00001693 if options.gerrit_url and options.gerrit_fetch:
tandrii@chromium.org83b1b232016-04-29 16:33:19 +00001694 assert options.issue and options.patchset
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001695 gerrit_obj = GerritAccessor(urlparse.urlparse(options.gerrit_url).netloc)
1696 options.author = gerrit_obj.GetChangeOwner(options.issue)
1697 options.description = gerrit_obj.GetChangeDescription(options.issue,
1698 options.patchset)
tandrii@chromium.org015ebae2016-04-25 19:37:22 +00001699 logging.info('Got author: "%s"', options.author)
1700 logging.info('Got description: """\n%s\n"""', options.description)
1701
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001702 try:
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001703 with canned_check_filter(options.skip_canned):
1704 results = DoPresubmitChecks(
1705 change_class(options.name,
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001706 options.description,
1707 options.root,
1708 files,
1709 options.issue,
1710 options.patchset,
1711 options.author,
1712 upstream=options.upstream),
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001713 options.commit,
1714 options.verbose,
1715 sys.stdout,
1716 sys.stdin,
1717 options.default_presubmit,
1718 options.may_prompt,
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001719 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001720 options.dry_run,
1721 options.parallel)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001722 return not results.should_continue()
Raul Tambre7c938462019-05-24 16:35:35 +00001723 except PresubmitFailure as e:
Raul Tambre80ee78e2019-05-06 22:41:05 +00001724 print(e, file=sys.stderr)
1725 print('Maybe your depot_tools is out of date?', file=sys.stderr)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001726 return 2
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001727
1728
1729if __name__ == '__main__':
maruel@chromium.org35625c72011-03-23 17:34:02 +00001730 fix_encoding.fix_encoding()
sbc@chromium.org013731e2015-02-26 18:28:43 +00001731 try:
1732 sys.exit(main())
1733 except KeyboardInterrupt:
1734 sys.stderr.write('interrupted\n')
sergiybf8a3b382016-07-05 11:21:30 -07001735 sys.exit(2)