blob: 84d5711fc9736bd18dd12861baa7b5c928d7ef07 [file] [log] [blame]
maruel@chromium.org725f1c32011-04-01 20:24:54 +00001#!/usr/bin/env python
maruel@chromium.org3bbf2942012-01-10 16:52:06 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
Raul Tambre80ee78e2019-05-06 22:41:05 +00009from __future__ import print_function
10
Saagar Sanghavi99816902020-08-11 22:41:25 +000011__version__ = '2.0.0'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000012
13# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
14# caching (between all different invocations of presubmit scripts for a given
15# change). We should add it as our presubmit scripts start feeling slow.
16
Edward Lemura5799e32020-01-17 19:26:51 +000017import argparse
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +090018import ast # Exposed through the API.
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +000019import contextlib
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +000020import cpplint
dcheng091b7db2016-06-16 01:27:51 -070021import fnmatch # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000022import glob
asvitkine@chromium.org15169952011-09-27 14:30:53 +000023import inspect
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +000024import itertools
maruel@chromium.org4f6852c2012-04-20 20:39:20 +000025import json # Exposed through the API.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000026import logging
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000027import multiprocessing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000028import os # Somewhat exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000029import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import re # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040031import signal
Allen Webbfe7d7092021-05-18 02:05:49 +000032import six
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033import sys # Parts exposed through API.
34import tempfile # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040035import threading
jam@chromium.org2a891dc2009-08-20 20:33:37 +000036import time
Edward Lemurde9e3ca2019-10-24 21:13:31 +000037import traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +000038import unittest # Exposed through the API.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000039from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040
41# Local imports.
maruel@chromium.org35625c72011-03-23 17:34:02 +000042import fix_encoding
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +000043import gclient_paths # Exposed through the API
44import gclient_utils
Aaron Gableb584c4f2017-04-26 16:28:08 -070045import git_footers
tandrii@chromium.org015ebae2016-04-25 19:37:22 +000046import gerrit_util
Edward Lesmes9ce03f82021-01-12 20:13:31 +000047import owners as owners_db
48import owners_client
Jochen Eisinger76f5fc62017-04-07 16:27:46 +020049import owners_finder
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000050import presubmit_canned_checks
Saagar Sanghavi9949ab72020-07-20 20:56:40 +000051import rdb_wrapper
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000052import scm
maruel@chromium.org84f4fe32011-04-06 13:26:45 +000053import subprocess2 as subprocess # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000054
Edward Lemur16af3562019-10-17 22:11:33 +000055if sys.version_info.major == 2:
56 # TODO(1009814): Expose urllib2 only through urllib_request and urllib_error
57 import urllib2 # Exposed through the API.
58 import urlparse
59 import urllib2 as urllib_request
60 import urllib2 as urllib_error
61else:
62 import urllib.parse as urlparse
63 import urllib.request as urllib_request
64 import urllib.error as urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000065
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000066# Ask for feedback only once in program lifetime.
67_ASKED_FOR_FEEDBACK = False
68
Edward Lemurecc27072020-01-06 16:42:34 +000069def time_time():
70 # Use this so that it can be mocked in tests without interfering with python
71 # system machinery.
72 return time.time()
73
74
maruel@chromium.org899e1c12011-04-07 17:03:18 +000075class PresubmitFailure(Exception):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000076 pass
77
78
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000079class CommandData(object):
Edward Lemur940c2822019-08-23 00:34:25 +000080 def __init__(self, name, cmd, kwargs, message, python3=False):
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000081 self.name = name
82 self.cmd = cmd
Edward Lesmes8e282792018-04-03 18:50:29 -040083 self.stdin = kwargs.get('stdin', None)
Edward Lemur2d6b67c2019-08-23 22:25:41 +000084 self.kwargs = kwargs.copy()
Edward Lesmes8e282792018-04-03 18:50:29 -040085 self.kwargs['stdout'] = subprocess.PIPE
86 self.kwargs['stderr'] = subprocess.STDOUT
87 self.kwargs['stdin'] = subprocess.PIPE
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000088 self.message = message
89 self.info = None
Edward Lemur940c2822019-08-23 00:34:25 +000090 self.python3 = python3
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000091
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000092
Edward Lesmes8e282792018-04-03 18:50:29 -040093# Adapted from
94# https://github.com/google/gtest-parallel/blob/master/gtest_parallel.py#L37
95#
96# An object that catches SIGINT sent to the Python process and notices
97# if processes passed to wait() die by SIGINT (we need to look for
98# both of those cases, because pressing Ctrl+C can result in either
99# the main process or one of the subprocesses getting the signal).
100#
101# Before a SIGINT is seen, wait(p) will simply call p.wait() and
102# return the result. Once a SIGINT has been seen (in the main process
103# or a subprocess, including the one the current call is waiting for),
Edward Lemur9a5bb612019-09-26 02:01:52 +0000104# wait(p) will call p.terminate().
Edward Lesmes8e282792018-04-03 18:50:29 -0400105class SigintHandler(object):
Edward Lesmes8e282792018-04-03 18:50:29 -0400106 sigint_returncodes = {-signal.SIGINT, # Unix
107 -1073741510, # Windows
108 }
109 def __init__(self):
110 self.__lock = threading.Lock()
111 self.__processes = set()
112 self.__got_sigint = False
Edward Lemur9a5bb612019-09-26 02:01:52 +0000113 self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt)
Edward Lesmes8e282792018-04-03 18:50:29 -0400114
115 def __on_sigint(self):
116 self.__got_sigint = True
117 while self.__processes:
118 try:
119 self.__processes.pop().terminate()
120 except OSError:
121 pass
122
Edward Lemur9a5bb612019-09-26 02:01:52 +0000123 def interrupt(self, signal_num, frame):
Edward Lesmes8e282792018-04-03 18:50:29 -0400124 with self.__lock:
125 self.__on_sigint()
Edward Lemur9a5bb612019-09-26 02:01:52 +0000126 self.__previous_signal(signal_num, frame)
Edward Lesmes8e282792018-04-03 18:50:29 -0400127
128 def got_sigint(self):
129 with self.__lock:
130 return self.__got_sigint
131
132 def wait(self, p, stdin):
133 with self.__lock:
134 if self.__got_sigint:
135 p.terminate()
136 self.__processes.add(p)
137 stdout, stderr = p.communicate(stdin)
138 code = p.returncode
139 with self.__lock:
140 self.__processes.discard(p)
141 if code in self.sigint_returncodes:
142 self.__on_sigint()
Edward Lesmes8e282792018-04-03 18:50:29 -0400143 return stdout, stderr
144
145sigint_handler = SigintHandler()
146
147
Edward Lemurecc27072020-01-06 16:42:34 +0000148class Timer(object):
149 def __init__(self, timeout, fn):
150 self.completed = False
151 self._fn = fn
152 self._timer = threading.Timer(timeout, self._onTimer) if timeout else None
153
154 def __enter__(self):
155 if self._timer:
156 self._timer.start()
157 return self
158
159 def __exit__(self, _type, _value, _traceback):
160 if self._timer:
161 self._timer.cancel()
162
163 def _onTimer(self):
164 self._fn()
165 self.completed = True
166
167
Edward Lesmes8e282792018-04-03 18:50:29 -0400168class ThreadPool(object):
Edward Lemurecc27072020-01-06 16:42:34 +0000169 def __init__(self, pool_size=None, timeout=None):
170 self.timeout = timeout
Edward Lesmes8e282792018-04-03 18:50:29 -0400171 self._pool_size = pool_size or multiprocessing.cpu_count()
172 self._messages = []
173 self._messages_lock = threading.Lock()
174 self._tests = []
175 self._tests_lock = threading.Lock()
176 self._nonparallel_tests = []
177
Edward Lemurecc27072020-01-06 16:42:34 +0000178 def _GetCommand(self, test):
Edward Lemur940c2822019-08-23 00:34:25 +0000179 vpython = 'vpython'
180 if test.python3:
181 vpython += '3'
182 if sys.platform == 'win32':
183 vpython += '.bat'
Edward Lesmes8e282792018-04-03 18:50:29 -0400184
185 cmd = test.cmd
186 if cmd[0] == 'python':
187 cmd = list(cmd)
188 cmd[0] = vpython
189 elif cmd[0].endswith('.py'):
190 cmd = [vpython] + cmd
191
Edward Lemur336e51f2019-11-14 21:42:04 +0000192 # On Windows, scripts on the current directory take precedence over PATH, so
193 # that when testing depot_tools on Windows, calling `vpython.bat` will
194 # execute the copy of vpython of the depot_tools under test instead of the
195 # one in the bot.
196 # As a workaround, we run the tests from the parent directory instead.
197 if (cmd[0] == vpython and
198 'cwd' in test.kwargs and
199 os.path.basename(test.kwargs['cwd']) == 'depot_tools'):
200 test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd'])
201 cmd[1] = os.path.join('depot_tools', cmd[1])
202
Edward Lemurecc27072020-01-06 16:42:34 +0000203 return cmd
204
205 def _RunWithTimeout(self, cmd, stdin, kwargs):
206 p = subprocess.Popen(cmd, **kwargs)
207 with Timer(self.timeout, p.terminate) as timer:
208 stdout, _ = sigint_handler.wait(p, stdin)
Josip Sokcevic6635baf2021-11-19 02:04:39 +0000209 stdout = stdout.decode('utf-8', 'ignore')
Edward Lemurecc27072020-01-06 16:42:34 +0000210 if timer.completed:
211 stdout = 'Process timed out after %ss\n%s' % (self.timeout, stdout)
Josip Sokcevic6635baf2021-11-19 02:04:39 +0000212 return p.returncode, stdout
Edward Lemurecc27072020-01-06 16:42:34 +0000213
214 def CallCommand(self, test):
215 """Runs an external program.
216
Edward Lemura5799e32020-01-17 19:26:51 +0000217 This function converts invocation of .py files and invocations of 'python'
Edward Lemurecc27072020-01-06 16:42:34 +0000218 to vpython invocations.
219 """
220 cmd = self._GetCommand(test)
Edward Lesmes8e282792018-04-03 18:50:29 -0400221 try:
Edward Lemurecc27072020-01-06 16:42:34 +0000222 start = time_time()
223 returncode, stdout = self._RunWithTimeout(cmd, test.stdin, test.kwargs)
224 duration = time_time() - start
Edward Lemur7cf94382019-11-15 22:36:41 +0000225 except Exception:
Edward Lemurecc27072020-01-06 16:42:34 +0000226 duration = time_time() - start
Edward Lesmes8e282792018-04-03 18:50:29 -0400227 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000228 '%s\n%s exec failure (%4.2fs)\n%s' % (
229 test.name, ' '.join(cmd), duration, traceback.format_exc()))
Edward Lemurde9e3ca2019-10-24 21:13:31 +0000230
Edward Lemurecc27072020-01-06 16:42:34 +0000231 if returncode != 0:
Edward Lesmes8e282792018-04-03 18:50:29 -0400232 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000233 '%s\n%s (%4.2fs) failed\n%s' % (
234 test.name, ' '.join(cmd), duration, stdout))
Edward Lemurecc27072020-01-06 16:42:34 +0000235
Edward Lesmes8e282792018-04-03 18:50:29 -0400236 if test.info:
Edward Lemur7cf94382019-11-15 22:36:41 +0000237 return test.info('%s\n%s (%4.2fs)' % (test.name, ' '.join(cmd), duration))
Edward Lesmes8e282792018-04-03 18:50:29 -0400238
239 def AddTests(self, tests, parallel=True):
240 if parallel:
241 self._tests.extend(tests)
242 else:
243 self._nonparallel_tests.extend(tests)
244
245 def RunAsync(self):
246 self._messages = []
247
248 def _WorkerFn():
249 while True:
250 test = None
251 with self._tests_lock:
252 if not self._tests:
253 break
254 test = self._tests.pop()
255 result = self.CallCommand(test)
256 if result:
257 with self._messages_lock:
258 self._messages.append(result)
259
260 def _StartDaemon():
261 t = threading.Thread(target=_WorkerFn)
262 t.daemon = True
263 t.start()
264 return t
265
266 while self._nonparallel_tests:
267 test = self._nonparallel_tests.pop()
268 result = self.CallCommand(test)
269 if result:
270 self._messages.append(result)
271
272 if self._tests:
273 threads = [_StartDaemon() for _ in range(self._pool_size)]
274 for worker in threads:
275 worker.join()
276
277 return self._messages
278
279
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000280def normpath(path):
281 '''Version of os.path.normpath that also changes backward slashes to
282 forward slashes when not running on Windows.
283 '''
284 # This is safe to always do because the Windows version of os.path.normpath
285 # will replace forward slashes with backward slashes.
286 path = path.replace(os.sep, '/')
287 return os.path.normpath(path)
288
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000289
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000290def _RightHandSideLinesImpl(affected_files):
291 """Implements RightHandSideLines for InputApi and GclChange."""
292 for af in affected_files:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000293 lines = af.ChangedContents()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000294 for line in lines:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000295 yield (af, line[0], line[1])
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000296
297
Edward Lemur6eb1d322020-02-27 22:20:15 +0000298def prompt_should_continue(prompt_string):
299 sys.stdout.write(prompt_string)
Dirk Pranke7288f882021-06-03 18:01:30 +0000300 sys.stdout.flush()
Edward Lemur6eb1d322020-02-27 22:20:15 +0000301 response = sys.stdin.readline().strip().lower()
302 return response in ('y', 'yes')
dpranke@chromium.org5ac21012011-03-16 02:58:25 +0000303
304
Josip Sokcevice293d3d2022-02-16 22:52:15 +0000305def _ShouldRunPresubmit(script_text, use_python3):
306 """Try to figure out whether these presubmit checks should be run under
307 python2 or python3. We need to do this without actually trying to
308 compile the text, since the text might compile in one but not the
309 other.
310
311 Args:
312 script_text: The text of the presubmit script.
313 use_python3: if true, will use python3 instead of python2 by default
314 if USE_PYTHON3 is not specified.
315
316 Return:
317 A boolean if presubmit should be executed
318 """
319 m = re.search('^USE_PYTHON3 = (True|False)$', script_text, flags=re.MULTILINE)
320 if m:
321 use_python3 = m.group(1) == 'True'
322
323 return ((sys.version_info.major == 2) and not use_python3) or \
324 ((sys.version_info.major == 3) and use_python3)
325
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000326# Top level object so multiprocessing can pickle
327# Public access through OutputApi object.
328class _PresubmitResult(object):
329 """Base class for result objects."""
330 fatal = False
331 should_prompt = False
332
333 def __init__(self, message, items=None, long_text=''):
334 """
335 message: A short one-line message to indicate errors.
336 items: A list of short strings to indicate where errors occurred.
337 long_text: multi-line text output, e.g. from another tool
338 """
339 self._message = message
340 self._items = items or []
Tom McKee61c72652021-07-20 11:56:32 +0000341 self._long_text = _PresubmitResult._ensure_str(long_text.rstrip())
342
343 @staticmethod
344 def _ensure_str(val):
345 """
346 val: A "stringish" value. Can be any of str, unicode or bytes.
347 returns: A str after applying encoding/decoding as needed.
348 Assumes/uses UTF-8 for relevant inputs/outputs.
349
350 We'd prefer to use six.ensure_str but our copy of six is old :(
351 """
352 if isinstance(val, str):
353 return val
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000354
Tom McKee61c72652021-07-20 11:56:32 +0000355 if six.PY2 and isinstance(val, unicode):
356 return val.encode()
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000357
358 if six.PY3 and isinstance(val, bytes):
Tom McKee61c72652021-07-20 11:56:32 +0000359 return val.decode()
360 raise ValueError("Unknown string type %s" % type(val))
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000361
Edward Lemur6eb1d322020-02-27 22:20:15 +0000362 def handle(self):
363 sys.stdout.write(self._message)
364 sys.stdout.write('\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000365 for index, item in enumerate(self._items):
Edward Lemur6eb1d322020-02-27 22:20:15 +0000366 sys.stdout.write(' ')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000367 # Write separately in case it's unicode.
Edward Lemur6eb1d322020-02-27 22:20:15 +0000368 sys.stdout.write(str(item))
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000369 if index < len(self._items) - 1:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000370 sys.stdout.write(' \\')
371 sys.stdout.write('\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000372 if self._long_text:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000373 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000374 # Write separately in case it's unicode.
Tom McKee61c72652021-07-20 11:56:32 +0000375 sys.stdout.write(self._long_text)
Edward Lemur6eb1d322020-02-27 22:20:15 +0000376 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000377
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000378 def json_format(self):
379 return {
380 'message': self._message,
Debrian Figueroa6095d402019-06-28 18:47:18 +0000381 'items': [str(item) for item in self._items],
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000382 'long_text': self._long_text,
383 'fatal': self.fatal
384 }
385
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000386
387# Top level object so multiprocessing can pickle
388# Public access through OutputApi object.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000389class _PresubmitError(_PresubmitResult):
390 """A hard presubmit error."""
391 fatal = True
392
393
394# Top level object so multiprocessing can pickle
395# Public access through OutputApi object.
396class _PresubmitPromptWarning(_PresubmitResult):
397 """An warning that prompts the user if they want to continue."""
398 should_prompt = True
399
400
401# Top level object so multiprocessing can pickle
402# Public access through OutputApi object.
403class _PresubmitNotifyResult(_PresubmitResult):
404 """Just print something to the screen -- but it's not even a warning."""
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000405
406
407# Top level object so multiprocessing can pickle
408# Public access through OutputApi object.
409class _MailTextResult(_PresubmitResult):
410 """A warning that should be included in the review request email."""
411 def __init__(self, *args, **kwargs):
412 super(_MailTextResult, self).__init__()
413 raise NotImplementedError()
414
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000415class GerritAccessor(object):
416 """Limited Gerrit functionality for canned presubmit checks to work.
417
418 To avoid excessive Gerrit calls, caches the results.
419 """
420
Edward Lesmeseb1bd622021-03-01 19:54:07 +0000421 def __init__(self, url=None, project=None, branch=None):
422 self.host = urlparse.urlparse(url).netloc if url else None
423 self.project = project
424 self.branch = branch
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000425 self.cache = {}
Edward Lesmes8170c292021-03-19 20:04:43 +0000426 self.code_owners_enabled = None
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000427
428 def _FetchChangeDetail(self, issue):
429 # Separate function to be easily mocked in tests.
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100430 try:
431 return gerrit_util.GetChangeDetail(
432 self.host, str(issue),
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700433 ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS'])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100434 except gerrit_util.GerritError as e:
435 if e.http_status == 404:
436 raise Exception('Either Gerrit issue %s doesn\'t exist, or '
437 'no credentials to fetch issue details' % issue)
438 raise
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000439
440 def GetChangeInfo(self, issue):
441 """Returns labels and all revisions (patchsets) for this issue.
442
443 The result is a dictionary according to Gerrit REST Api.
444 https://gerrit-review.googlesource.com/Documentation/rest-api.html
445
446 However, API isn't very clear what's inside, so see tests for example.
447 """
448 assert issue
449 cache_key = int(issue)
450 if cache_key not in self.cache:
451 self.cache[cache_key] = self._FetchChangeDetail(issue)
452 return self.cache[cache_key]
453
454 def GetChangeDescription(self, issue, patchset=None):
455 """If patchset is none, fetches current patchset."""
456 info = self.GetChangeInfo(issue)
457 # info is a reference to cache. We'll modify it here adding description to
458 # it to the right patchset, if it is not yet there.
459
460 # Find revision info for the patchset we want.
461 if patchset is not None:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +0000462 for rev, rev_info in info['revisions'].items():
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000463 if str(rev_info['_number']) == str(patchset):
464 break
465 else:
466 raise Exception('patchset %s doesn\'t exist in issue %s' % (
467 patchset, issue))
468 else:
469 rev = info['current_revision']
470 rev_info = info['revisions'][rev]
471
Andrii Shyshkalov9c3a4642017-01-24 17:41:22 +0100472 return rev_info['commit']['message']
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000473
Mun Yong Jang603d01e2017-12-19 16:38:30 -0800474 def GetDestRef(self, issue):
475 ref = self.GetChangeInfo(issue)['branch']
476 if not ref.startswith('refs/'):
477 # NOTE: it is possible to create 'refs/x' branch,
478 # aka 'refs/heads/refs/x'. However, this is ill-advised.
479 ref = 'refs/heads/%s' % ref
480 return ref
481
Edward Lesmes02d4b822020-11-11 00:37:35 +0000482 def _GetApproversForLabel(self, issue, label):
483 change_info = self.GetChangeInfo(issue)
484 label_info = change_info.get('labels', {}).get(label, {})
485 values = label_info.get('values', {}).keys()
486 if not values:
487 return []
488 max_value = max(int(v) for v in values)
489 return [v for v in label_info.get('all', [])
490 if v.get('value', 0) == max_value]
491
Edward Lesmesc4566172021-03-19 16:55:13 +0000492 def IsBotCommitApproved(self, issue):
493 return bool(self._GetApproversForLabel(issue, 'Bot-Commit'))
494
Edward Lesmescf49cb82020-11-11 01:08:36 +0000495 def IsOwnersOverrideApproved(self, issue):
496 return bool(self._GetApproversForLabel(issue, 'Owners-Override'))
497
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000498 def GetChangeOwner(self, issue):
499 return self.GetChangeInfo(issue)['owner']['email']
500
501 def GetChangeReviewers(self, issue, approving_only=True):
Aaron Gable8b478f02017-07-31 15:33:19 -0700502 changeinfo = self.GetChangeInfo(issue)
503 if approving_only:
Edward Lesmes02d4b822020-11-11 00:37:35 +0000504 reviewers = self._GetApproversForLabel(issue, 'Code-Review')
Aaron Gable8b478f02017-07-31 15:33:19 -0700505 else:
506 reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', [])
507 return [r.get('email') for r in reviewers]
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000508
Edward Lemure4d329c2020-02-03 20:41:18 +0000509 def UpdateDescription(self, description, issue):
510 gerrit_util.SetCommitMessage(self.host, issue, description, notify='NONE')
511
Edward Lesmes8170c292021-03-19 20:04:43 +0000512 def IsCodeOwnersEnabledOnRepo(self):
513 if self.code_owners_enabled is None:
514 self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo(
Edward Lesmes392c4072021-03-19 21:58:45 +0000515 self.host, self.project)
Edward Lesmes8170c292021-03-19 20:04:43 +0000516 return self.code_owners_enabled
517
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000518
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000519class OutputApi(object):
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000520 """An instance of OutputApi gets passed to presubmit scripts so that they
521 can output various types of results.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000522 """
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000523 PresubmitResult = _PresubmitResult
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000524 PresubmitError = _PresubmitError
525 PresubmitPromptWarning = _PresubmitPromptWarning
526 PresubmitNotifyResult = _PresubmitNotifyResult
527 MailTextResult = _MailTextResult
528
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000529 def __init__(self, is_committing):
530 self.is_committing = is_committing
Daniel Cheng7227d212017-11-17 08:12:37 -0800531 self.more_cc = []
532
533 def AppendCC(self, cc):
534 """Appends a user to cc for this change."""
535 self.more_cc.append(cc)
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000536
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000537 def PresubmitPromptOrNotify(self, *args, **kwargs):
538 """Warn the user when uploading, but only notify if committing."""
539 if self.is_committing:
540 return self.PresubmitNotifyResult(*args, **kwargs)
541 return self.PresubmitPromptWarning(*args, **kwargs)
542
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000543
544class InputApi(object):
545 """An instance of this object is passed to presubmit scripts so they can
546 know stuff about the change they're looking at.
547 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000548 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800549 # pylint: disable=no-self-use
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000550
maruel@chromium.org3410d912009-06-09 20:56:16 +0000551 # File extensions that are considered source files from a style guide
552 # perspective. Don't modify this list from a presubmit script!
maruel@chromium.orgc33455a2011-06-24 19:14:18 +0000553 #
554 # Files without an extension aren't included in the list. If you want to
local_bot30f774e2020-06-25 18:23:34 +0000555 # filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list.
local_bot64021412020-07-08 21:05:39 +0000556 # Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below.
557 DEFAULT_FILES_TO_CHECK = (
maruel@chromium.org3410d912009-06-09 20:56:16 +0000558 # C++ and friends
Edward Lemura5799e32020-01-17 19:26:51 +0000559 r'.+\.c$', r'.+\.cc$', r'.+\.cpp$', r'.+\.h$', r'.+\.m$', r'.+\.mm$',
560 r'.+\.inl$', r'.+\.asm$', r'.+\.hxx$', r'.+\.hpp$', r'.+\.s$', r'.+\.S$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000561 # Scripts
Edward Lemura5799e32020-01-17 19:26:51 +0000562 r'.+\.js$', r'.+\.py$', r'.+\.sh$', r'.+\.rb$', r'.+\.pl$', r'.+\.pm$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000563 # Other
Edward Lemura5799e32020-01-17 19:26:51 +0000564 r'.+\.java$', r'.+\.mk$', r'.+\.am$', r'.+\.css$', r'.+\.mojom$',
565 r'.+\.fidl$'
maruel@chromium.org3410d912009-06-09 20:56:16 +0000566 )
567
568 # Path regexp that should be excluded from being considered containing source
569 # files. Don't modify this list from a presubmit script!
local_bot64021412020-07-08 21:05:39 +0000570 DEFAULT_FILES_TO_SKIP = (
Edward Lemura5799e32020-01-17 19:26:51 +0000571 r'testing_support[\\\/]google_appengine[\\\/].*',
572 r'.*\bexperimental[\\\/].*',
Kent Tamura179dd1e2018-04-26 15:07:41 +0900573 # Exclude third_party/.* but NOT third_party/{WebKit,blink}
574 # (crbug.com/539768 and crbug.com/836555).
Edward Lemura5799e32020-01-17 19:26:51 +0000575 r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000576 # Output directories (just in case)
Edward Lemura5799e32020-01-17 19:26:51 +0000577 r'.*\bDebug[\\\/].*',
578 r'.*\bRelease[\\\/].*',
579 r'.*\bxcodebuild[\\\/].*',
580 r'.*\bout[\\\/].*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000581 # All caps files like README and LICENCE.
Edward Lemura5799e32020-01-17 19:26:51 +0000582 r'.*\b[A-Z0-9_]{2,}$',
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000583 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
Edward Lemura5799e32020-01-17 19:26:51 +0000584 r'(|.*[\\\/])\.git[\\\/].*',
585 r'(|.*[\\\/])\.svn[\\\/].*',
maruel@chromium.org7ccb4bb2011-11-07 20:26:20 +0000586 # There is no point in processing a patch file.
Edward Lemura5799e32020-01-17 19:26:51 +0000587 r'.+\.diff$',
588 r'.+\.patch$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000589 )
590
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000591 def __init__(self, change, presubmit_path, is_committing,
Edward Lesmes8e282792018-04-03 18:50:29 -0400592 verbose, gerrit_obj, dry_run=None, thread_pool=None, parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000593 """Builds an InputApi object.
594
595 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000596 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000597 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000598 is_committing: True if the change is about to be committed.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000599 gerrit_obj: provides basic Gerrit codereview functionality.
600 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -0400601 parallel: if true, all tests reported via input_api.RunTests for all
602 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000603 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000604 # Version number of the presubmit_support script.
605 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000606 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000607 self.is_committing = is_committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000608 self.gerrit = gerrit_obj
tandrii@chromium.org57bafac2016-04-28 05:09:03 +0000609 self.dry_run = dry_run
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000610
Edward Lesmes8e282792018-04-03 18:50:29 -0400611 self.parallel = parallel
612 self.thread_pool = thread_pool or ThreadPool()
613
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000614 # We expose various modules and functions as attributes of the input_api
615 # so that presubmit scripts don't have to import them.
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +0900616 self.ast = ast
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000617 self.basename = os.path.basename
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000618 self.cpplint = cpplint
dcheng091b7db2016-06-16 01:27:51 -0700619 self.fnmatch = fnmatch
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +0000620 self.gclient_paths = gclient_paths
Yoshisato Yanagisawa57dd17b2019-03-22 09:10:29 +0000621 # TODO(yyanagisawa): stop exposing this when python3 become default.
622 # Since python3's tempfile has TemporaryDirectory, we do not need this.
623 self.temporary_directory = gclient_utils.temporary_directory
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000624 self.glob = glob.glob
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000625 self.json = json
maruel@chromium.org6fba34d2011-06-02 13:45:12 +0000626 self.logging = logging.getLogger('PRESUBMIT')
maruel@chromium.org2b5ce562011-03-31 16:15:44 +0000627 self.os_listdir = os.listdir
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000628 self.os_path = os.path
pgervais@chromium.orgbd0cace2014-10-02 23:23:46 +0000629 self.os_stat = os.stat
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +0000630 self.os_walk = os.walk
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000631 self.re = re
632 self.subprocess = subprocess
Edward Lemurb9830242019-10-30 22:19:20 +0000633 self.sys = sys
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000634 self.tempfile = tempfile
dpranke@chromium.org0d1bdea2011-03-24 22:54:38 +0000635 self.time = time
maruel@chromium.org1487d532009-06-06 00:22:57 +0000636 self.unittest = unittest
Edward Lemurb9830242019-10-30 22:19:20 +0000637 if sys.version_info.major == 2:
638 self.urllib2 = urllib2
Edward Lemur16af3562019-10-17 22:11:33 +0000639 self.urllib_request = urllib_request
640 self.urllib_error = urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000641
Robert Iannucci50258932018-03-19 10:30:59 -0700642 self.is_windows = sys.platform == 'win32'
643
Edward Lemurb9646622019-10-25 20:57:35 +0000644 # Set python_executable to 'vpython' in order to allow scripts in other
645 # repos (e.g. src.git) to automatically pick up that repo's .vpython file,
646 # instead of inheriting the one in depot_tools.
647 self.python_executable = 'vpython'
Erik Staab69135d12021-05-14 22:31:57 +0000648 # Offer a python 3 executable for use during the migration off of python 2.
649 self.python3_executable = 'vpython3'
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000650 self.environ = os.environ
651
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000652 # InputApi.platform is the platform you're currently running on.
653 self.platform = sys.platform
654
iannucci@chromium.org0af3bb32015-06-12 20:44:35 +0000655 self.cpu_count = multiprocessing.cpu_count()
656
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000657 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000658 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000659
660 # We carry the canned checks so presubmit scripts can easily use them.
661 self.canned_checks = presubmit_canned_checks
662
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100663 # Temporary files we must manually remove at the end of a run.
664 self._named_temporary_files = []
Jochen Eisinger72606f82017-04-04 10:44:18 +0200665
Edward Lesmesdc11c6b2021-03-19 19:22:05 +0000666 self.owners_client = None
667 if self.gerrit:
668 self.owners_client = owners_client.GetCodeOwnersClient(
669 root=change.RepositoryRoot(),
670 upstream=change.UpstreamBranch(),
671 host=self.gerrit.host,
672 project=self.gerrit.project,
673 branch=self.gerrit.branch)
Edward Lesmes9ce03f82021-01-12 20:13:31 +0000674 self.owners_db = owners_db.Database(
675 change.RepositoryRoot(), fopen=open, os_path=self.os_path)
Jochen Eisinger76f5fc62017-04-07 16:27:46 +0200676 self.owners_finder = owners_finder.OwnersFinder
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000677 self.verbose = verbose
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000678 self.Command = CommandData
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000679
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000680 # Replace <hash_map> and <hash_set> as headers that need to be included
Edward Lemura5799e32020-01-17 19:26:51 +0000681 # with 'base/containers/hash_tables.h' instead.
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000682 # Access to a protected member _XX of a client class
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800683 # pylint: disable=protected-access
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000684 self.cpplint._re_pattern_templates = [
danakj@chromium.org18278522013-06-11 22:42:32 +0000685 (a, b, 'base/containers/hash_tables.h')
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000686 if header in ('<hash_map>', '<hash_set>') else (a, b, header)
687 for (a, b, header) in cpplint._re_pattern_templates
688 ]
689
Edward Lemurecc27072020-01-06 16:42:34 +0000690 def SetTimeout(self, timeout):
691 self.thread_pool.timeout = timeout
692
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000693 def PresubmitLocalPath(self):
694 """Returns the local path of the presubmit script currently being run.
695
696 This is useful if you don't want to hard-code absolute paths in the
697 presubmit script. For example, It can be used to find another file
698 relative to the PRESUBMIT.py script, so the whole tree can be branched and
699 the presubmit script still works, without editing its content.
700 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000701 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000702
agable0b65e732016-11-22 09:25:46 -0800703 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000704 """Same as input_api.change.AffectedFiles() except only lists files
705 (and optionally directories) in the same directory as the current presubmit
Bruce Dawson7a0b07a2020-04-23 17:14:40 +0000706 script, or subdirectories thereof. Note that files are listed using the OS
707 path separator, so backslashes are used as separators on Windows.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000708 """
Edward Lemura5799e32020-01-17 19:26:51 +0000709 dir_with_slash = normpath('%s/' % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000710 if len(dir_with_slash) == 1:
711 dir_with_slash = ''
sail@chromium.org5538e022011-05-12 17:53:16 +0000712
Edward Lemurb9830242019-10-30 22:19:20 +0000713 return list(filter(
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000714 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
Edward Lemurb9830242019-10-30 22:19:20 +0000715 self.change.AffectedFiles(include_deletes, file_filter)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000716
agable0b65e732016-11-22 09:25:46 -0800717 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000718 """Returns local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800719 paths = [af.LocalPath() for af in self.AffectedFiles()]
Edward Lemura5799e32020-01-17 19:26:51 +0000720 logging.debug('LocalPaths: %s', paths)
pgervais@chromium.org2f64f782014-04-25 00:12:33 +0000721 return paths
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000722
agable0b65e732016-11-22 09:25:46 -0800723 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000724 """Returns absolute local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800725 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000726
John Budorick16162372018-04-18 10:39:53 -0700727 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
agable0b65e732016-11-22 09:25:46 -0800728 """Same as input_api.change.AffectedTestableFiles() except only lists files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000729 in the same directory as the current presubmit script, or subdirectories
730 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000731 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000732 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +0000733 warn('AffectedTestableFiles(include_deletes=%s)'
734 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000735 category=DeprecationWarning,
736 stacklevel=2)
Edward Lemurb9830242019-10-30 22:19:20 +0000737 return list(filter(
738 lambda x: x.IsTestableFile(),
739 self.AffectedFiles(include_deletes=False, **kwargs)))
agable0b65e732016-11-22 09:25:46 -0800740
741 def AffectedTextFiles(self, include_deletes=None):
742 """An alias to AffectedTestableFiles for backwards compatibility."""
743 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000744
Josip Sokcevic8c955952021-02-01 21:32:57 +0000745 def FilterSourceFile(self,
746 affected_file,
747 files_to_check=None,
748 files_to_skip=None,
749 allow_list=None,
750 block_list=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000751 """Filters out files that aren't considered 'source file'.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000752
local_bot64021412020-07-08 21:05:39 +0000753 If files_to_check or files_to_skip is None, InputApi.DEFAULT_FILES_TO_CHECK
754 and InputApi.DEFAULT_FILES_TO_SKIP is used respectively.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000755
756 The lists will be compiled as regular expression and
757 AffectedFile.LocalPath() needs to pass both list.
758
759 Note: Copy-paste this function to suit your needs or use a lambda function.
760 """
local_bot64021412020-07-08 21:05:39 +0000761 if files_to_check is None:
762 files_to_check = self.DEFAULT_FILES_TO_CHECK
763 if files_to_skip is None:
764 files_to_skip = self.DEFAULT_FILES_TO_SKIP
local_bot30f774e2020-06-25 18:23:34 +0000765
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000766 def Find(affected_file, items):
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000767 local_path = affected_file.LocalPath()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000768 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000769 if self.re.match(item, local_path):
maruel@chromium.org3410d912009-06-09 20:56:16 +0000770 return True
771 return False
local_bot64021412020-07-08 21:05:39 +0000772 return (Find(affected_file, files_to_check) and
773 not Find(affected_file, files_to_skip))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000774
775 def AffectedSourceFiles(self, source_file):
agable0b65e732016-11-22 09:25:46 -0800776 """Filter the list of AffectedTestableFiles by the function source_file.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000777
778 If source_file is None, InputApi.FilterSourceFile() is used.
779 """
780 if not source_file:
781 source_file = self.FilterSourceFile
Edward Lemurb9830242019-10-30 22:19:20 +0000782 return list(filter(source_file, self.AffectedTestableFiles()))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000783
784 def RightHandSideLines(self, source_file_filter=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000785 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000786
787 Only lists lines from new or modified text files in the change that are
788 contained by the directory of the currently executing presubmit script.
789
790 This is useful for doing line-by-line regex checks, like checking for
791 trailing whitespace.
792
793 Yields:
794 a 3 tuple:
795 the AffectedFile instance of the current file;
796 integer line number (1-based); and
797 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000798
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000799 Note: The carriage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000800 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000801 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000802 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000803
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000804 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000805 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000806
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000807 Deny reading anything outside the repository.
808 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000809 if isinstance(file_item, AffectedFile):
810 file_item = file_item.AbsoluteLocalPath()
811 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000812 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000813 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000814
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100815 def CreateTemporaryFile(self, **kwargs):
816 """Returns a named temporary file that must be removed with a call to
817 RemoveTemporaryFiles().
818
819 All keyword arguments are forwarded to tempfile.NamedTemporaryFile(),
820 except for |delete|, which is always set to False.
821
822 Presubmit checks that need to create a temporary file and pass it for
823 reading should use this function instead of NamedTemporaryFile(), as
824 Windows fails to open a file that is already open for writing.
825
826 with input_api.CreateTemporaryFile() as f:
827 f.write('xyz')
828 f.close()
829 input_api.subprocess.check_output(['script-that', '--reads-from',
830 f.name])
831
832
833 Note that callers of CreateTemporaryFile() should not worry about removing
834 any temporary file; this is done transparently by the presubmit handling
835 code.
836 """
837 if 'delete' in kwargs:
838 # Prevent users from passing |delete|; we take care of file deletion
839 # ourselves and this prevents unintuitive error messages when we pass
840 # delete=False and 'delete' is also in kwargs.
841 raise TypeError('CreateTemporaryFile() does not take a "delete" '
842 'argument, file deletion is handled automatically by '
843 'the same presubmit_support code that creates InputApi '
844 'objects.')
845 temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs)
846 self._named_temporary_files.append(temp_file.name)
847 return temp_file
848
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000849 @property
850 def tbr(self):
851 """Returns if a change is TBR'ed."""
Jeremy Romandce22502017-06-20 15:37:29 -0400852 return 'TBR' in self.change.tags or self.change.TBRsFromDescription()
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000853
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000854 def RunTests(self, tests_mix, parallel=True):
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000855 tests = []
856 msgs = []
857 for t in tests_mix:
Edward Lesmes8e282792018-04-03 18:50:29 -0400858 if isinstance(t, OutputApi.PresubmitResult) and t:
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000859 msgs.append(t)
860 else:
861 assert issubclass(t.message, _PresubmitResult)
862 tests.append(t)
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000863 if self.verbose:
864 t.info = _PresubmitNotifyResult
Edward Lemur1037c742018-05-01 18:56:04 -0400865 if not t.kwargs.get('cwd'):
866 t.kwargs['cwd'] = self.PresubmitLocalPath()
Edward Lesmes8e282792018-04-03 18:50:29 -0400867 self.thread_pool.AddTests(tests, parallel)
Edward Lemur21000eb2019-05-24 23:25:58 +0000868 # When self.parallel is True (i.e. --parallel is passed as an option)
869 # RunTests doesn't actually run tests. It adds them to a ThreadPool that
870 # will run all tests once all PRESUBMIT files are processed.
871 # Otherwise, it will run them and return the results.
872 if not self.parallel:
Edward Lesmes8e282792018-04-03 18:50:29 -0400873 msgs.extend(self.thread_pool.RunAsync())
874 return msgs
scottmg86099d72016-09-01 09:16:51 -0700875
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000876
nick@chromium.orgff526192013-06-10 19:30:26 +0000877class _DiffCache(object):
878 """Caches diffs retrieved from a particular SCM."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000879 def __init__(self, upstream=None):
880 """Stores the upstream revision against which all diffs will be computed."""
881 self._upstream = upstream
nick@chromium.orgff526192013-06-10 19:30:26 +0000882
883 def GetDiff(self, path, local_root):
884 """Get the diff for a particular path."""
885 raise NotImplementedError()
886
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700887 def GetOldContents(self, path, local_root):
888 """Get the old version for a particular path."""
889 raise NotImplementedError()
890
nick@chromium.orgff526192013-06-10 19:30:26 +0000891
nick@chromium.orgff526192013-06-10 19:30:26 +0000892class _GitDiffCache(_DiffCache):
893 """DiffCache implementation for git; gets all file diffs at once."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000894 def __init__(self, upstream):
895 super(_GitDiffCache, self).__init__(upstream=upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000896 self._diffs_by_file = None
897
898 def GetDiff(self, path, local_root):
899 if not self._diffs_by_file:
900 # Compute a single diff for all files and parse the output; should
901 # with git this is much faster than computing one diff for each file.
902 diffs = {}
903
904 # Don't specify any filenames below, because there are command line length
905 # limits on some platforms and GenerateDiff would fail.
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000906 unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True,
907 branch=self._upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000908
909 # This regex matches the path twice, separated by a space. Note that
910 # filename itself may contain spaces.
911 file_marker = re.compile('^diff --git (?P<filename>.*) (?P=filename)$')
912 current_diff = []
913 keep_line_endings = True
914 for x in unified_diff.splitlines(keep_line_endings):
915 match = file_marker.match(x)
916 if match:
917 # Marks the start of a new per-file section.
918 diffs[match.group('filename')] = current_diff = [x]
919 elif x.startswith('diff --git'):
920 raise PresubmitFailure('Unexpected diff line: %s' % x)
921 else:
922 current_diff.append(x)
923
924 self._diffs_by_file = dict(
925 (normpath(path), ''.join(diff)) for path, diff in diffs.items())
926
927 if path not in self._diffs_by_file:
928 raise PresubmitFailure(
929 'Unified diff did not contain entry for file %s' % path)
930
931 return self._diffs_by_file[path]
932
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700933 def GetOldContents(self, path, local_root):
934 return scm.GIT.GetOldContents(local_root, path, branch=self._upstream)
935
nick@chromium.orgff526192013-06-10 19:30:26 +0000936
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000937class AffectedFile(object):
938 """Representation of a file in a change."""
nick@chromium.orgff526192013-06-10 19:30:26 +0000939
940 DIFF_CACHE = _DiffCache
941
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000942 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800943 # pylint: disable=no-self-use
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000944 def __init__(self, path, action, repository_root, diff_cache):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000945 self._path = path
946 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000947 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000948 self._is_directory = None
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000949 self._cached_changed_contents = None
950 self._cached_new_contents = None
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000951 self._diff_cache = diff_cache
tobiasjs2836bcf2016-08-16 04:08:16 -0700952 logging.debug('%s(%s)', self.__class__.__name__, self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000953
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000954 def LocalPath(self):
955 """Returns the path of this file on the local disk relative to client root.
Andrew Grieve92b8b992017-11-02 09:42:24 -0400956
957 This should be used for error messages but not for accessing files,
958 because presubmit checks are run with CWD=PresubmitLocalPath() (which is
959 often != client root).
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000960 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000961 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000962
963 def AbsoluteLocalPath(self):
964 """Returns the absolute path of this file on the local disk.
965 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000966 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000967
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000968 def Action(self):
969 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000970 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000971
agable0b65e732016-11-22 09:25:46 -0800972 def IsTestableFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000973 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000974
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000975 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000976 raise NotImplementedError() # Implement when needed
977
agable0b65e732016-11-22 09:25:46 -0800978 def IsTextFile(self):
979 """An alias to IsTestableFile for backwards compatibility."""
980 return self.IsTestableFile()
981
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700982 def OldContents(self):
983 """Returns an iterator over the lines in the old version of file.
984
Daniel Cheng2da34fe2017-03-21 20:42:12 -0700985 The old version is the file before any modifications in the user's
Edward Lemura5799e32020-01-17 19:26:51 +0000986 workspace, i.e. the 'left hand side'.
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700987
988 Contents will be empty if the file is a directory or does not exist.
989 Note: The carriage returns (LF or CR) are stripped off.
990 """
991 return self._diff_cache.GetOldContents(self.LocalPath(),
992 self._local_root).splitlines()
993
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000994 def NewContents(self):
995 """Returns an iterator over the lines in the new version of file.
996
Edward Lemura5799e32020-01-17 19:26:51 +0000997 The new version is the file in the user's workspace, i.e. the 'right hand
998 side'.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000999
1000 Contents will be empty if the file is a directory or does not exist.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001001 Note: The carriage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001002 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001003 if self._cached_new_contents is None:
1004 self._cached_new_contents = []
agable0b65e732016-11-22 09:25:46 -08001005 try:
1006 self._cached_new_contents = gclient_utils.FileRead(
1007 self.AbsoluteLocalPath(), 'rU').splitlines()
1008 except IOError:
1009 pass # File not found? That's fine; maybe it was deleted.
Greg Thompson30cde452021-06-01 16:38:47 +00001010 except UnicodeDecodeError as e:
Dirk Pranke6fc394f2021-05-26 23:25:14 +00001011 # log the filename since we're probably trying to read a binary
1012 # file, and shouldn't be.
1013 print('Error reading %s: %s' % (self.AbsoluteLocalPath(), e))
1014 raise
1015
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001016 return self._cached_new_contents[:]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001017
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001018 def ChangedContents(self, keeplinebreaks=False):
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001019 """Returns a list of tuples (line number, line text) of all new lines.
1020
1021 This relies on the scm diff output describing each changed code section
1022 with a line of the form
1023
1024 ^@@ <old line num>,<old size> <new line num>,<new size> @@$
1025 """
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001026 # Don't return cached results when line breaks are requested.
1027 if not keeplinebreaks and self._cached_changed_contents is not None:
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001028 return self._cached_changed_contents[:]
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001029 result = []
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001030 line_num = 0
1031
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001032 # The keeplinebreaks parameter to splitlines must be True or else the
1033 # CheckForWindowsLineEndings presubmit will be a NOP.
1034 for line in self.GenerateScmDiff().splitlines(keeplinebreaks):
Edward Lemurac5c55f2020-02-29 00:17:16 +00001035 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
1036 if m:
1037 line_num = int(m.groups(1)[0])
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001038 continue
1039 if line.startswith('+') and not line.startswith('++'):
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001040 result.append((line_num, line[1:]))
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001041 if not line.startswith('-'):
1042 line_num += 1
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001043 # Don't cache results with line breaks.
1044 if keeplinebreaks:
1045 return result;
1046 self._cached_changed_contents = result
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001047 return self._cached_changed_contents[:]
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001048
maruel@chromium.org5de13972009-06-10 18:16:06 +00001049 def __str__(self):
1050 return self.LocalPath()
1051
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001052 def GenerateScmDiff(self):
nick@chromium.orgff526192013-06-10 19:30:26 +00001053 return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001054
maruel@chromium.org58407af2011-04-12 23:15:57 +00001055
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001056class GitAffectedFile(AffectedFile):
1057 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +00001058 # Method 'NNN' is abstract in class 'NNN' but is not overridden
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001059 # pylint: disable=abstract-method
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001060
nick@chromium.orgff526192013-06-10 19:30:26 +00001061 DIFF_CACHE = _GitDiffCache
1062
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001063 def __init__(self, *args, **kwargs):
1064 AffectedFile.__init__(self, *args, **kwargs)
1065 self._server_path = None
agable0b65e732016-11-22 09:25:46 -08001066 self._is_testable_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001067
agable0b65e732016-11-22 09:25:46 -08001068 def IsTestableFile(self):
1069 if self._is_testable_file is None:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001070 if self.Action() == 'D':
agable0b65e732016-11-22 09:25:46 -08001071 # A deleted file is not testable.
1072 self._is_testable_file = False
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001073 else:
agable0b65e732016-11-22 09:25:46 -08001074 self._is_testable_file = os.path.isfile(self.AbsoluteLocalPath())
1075 return self._is_testable_file
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001076
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001077
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001078class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001079 """Describe a change.
1080
1081 Used directly by the presubmit scripts to query the current change being
1082 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001083
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001084 Instance members:
nick@chromium.orgff526192013-06-10 19:30:26 +00001085 tags: Dictionary of KEY=VALUE pairs found in the change description.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001086 self.KEY: equivalent to tags['KEY']
1087 """
1088
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001089 _AFFECTED_FILES = AffectedFile
1090
Edward Lemura5799e32020-01-17 19:26:51 +00001091 # Matches key/value (or 'tag') lines in changelist descriptions.
maruel@chromium.org428342a2011-11-10 15:46:33 +00001092 TAG_LINE_RE = re.compile(
maruel@chromium.orgc6f60e82013-04-19 17:01:57 +00001093 '^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001094 scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001095
maruel@chromium.org58407af2011-04-12 23:15:57 +00001096 def __init__(
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001097 self, name, description, local_root, files, issue, patchset, author,
1098 upstream=None):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001099 if files is None:
1100 files = []
1101 self._name = name
chase@chromium.org8e416c82009-10-06 04:30:44 +00001102 # Convert root into an absolute path.
1103 self._local_root = os.path.abspath(local_root)
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001104 self._upstream = upstream
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001105 self.issue = issue
1106 self.patchset = patchset
maruel@chromium.org58407af2011-04-12 23:15:57 +00001107 self.author_email = author
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001108
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001109 self._full_description = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001110 self.tags = {}
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001111 self._description_without_tags = ''
1112 self.SetDescriptionText(description)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001113
maruel@chromium.orge085d812011-10-10 19:49:15 +00001114 assert all(
1115 (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files
1116
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001117 diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream)
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001118 self._affected_files = [
nick@chromium.orgff526192013-06-10 19:30:26 +00001119 self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache)
1120 for action, path in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +00001121 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001122
Edward Lesmes9ce03f82021-01-12 20:13:31 +00001123 def UpstreamBranch(self):
1124 """Returns the upstream branch for the change."""
1125 return self._upstream
1126
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001127 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001128 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001129 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001130
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001131 def DescriptionText(self):
1132 """Returns the user-entered changelist description, minus tags.
1133
Edward Lemura5799e32020-01-17 19:26:51 +00001134 Any line in the user-provided description starting with e.g. 'FOO='
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001135 (whitespace permitted before and around) is considered a tag line. Such
1136 lines are stripped out of the description this function returns.
1137 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001138 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001139
1140 def FullDescriptionText(self):
1141 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001142 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001143
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001144 def SetDescriptionText(self, description):
1145 """Sets the full description text (including tags) to |description|.
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001146
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001147 Also updates the list of tags."""
1148 self._full_description = description
1149
1150 # From the description text, build up a dictionary of key/value pairs
Edward Lemura5799e32020-01-17 19:26:51 +00001151 # plus the description minus all key/value or 'tag' lines.
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001152 description_without_tags = []
1153 self.tags = {}
1154 for line in self._full_description.splitlines():
1155 m = self.TAG_LINE_RE.match(line)
1156 if m:
1157 self.tags[m.group('key')] = m.group('value')
1158 else:
1159 description_without_tags.append(line)
1160
1161 # Change back to text and remove whitespace at end.
1162 self._description_without_tags = (
1163 '\n'.join(description_without_tags).rstrip())
1164
Edward Lemur69bb8be2020-02-03 20:37:38 +00001165 def AddDescriptionFooter(self, key, value):
1166 """Adds the given footer to the change description.
1167
1168 Args:
1169 key: A string with the key for the git footer. It must conform to
1170 the git footers format (i.e. 'List-Of-Tokens') and will be case
1171 normalized so that each token is title-cased.
1172 value: A string with the value for the git footer.
1173 """
1174 description = git_footers.add_footer(
1175 self.FullDescriptionText(), git_footers.normalize_name(key), value)
1176 self.SetDescriptionText(description)
1177
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001178 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001179 """Returns the repository (checkout) root directory for this change,
1180 as an absolute path.
1181 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001182 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001183
1184 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001185 """Return tags directly as attributes on the object."""
Edward Lemura5799e32020-01-17 19:26:51 +00001186 if not re.match(r'^[A-Z_]*$', attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001187 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +00001188 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001189
Edward Lemur69bb8be2020-02-03 20:37:38 +00001190 def GitFootersFromDescription(self):
1191 """Return the git footers present in the description.
1192
1193 Returns:
1194 footers: A dict of {footer: [values]} containing a multimap of the footers
1195 in the change description.
1196 """
1197 return git_footers.parse_footers(self.FullDescriptionText())
1198
Aaron Gablefc03e672017-05-15 14:09:42 -07001199 def BugsFromDescription(self):
1200 """Returns all bugs referenced in the commit description."""
Sean McAllister1e509c52021-10-25 17:54:25 +00001201 bug_tags = ['BUG', 'FIXED']
1202
1203 tags = []
1204 for tag in bug_tags:
1205 values = self.tags.get(tag)
1206 if values:
1207 tags += [value.strip() for value in values.split(',')]
1208
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001209 footers = []
Edward Lemur69bb8be2020-02-03 20:37:38 +00001210 parsed = self.GitFootersFromDescription()
Dan Beam62954042019-10-03 21:20:33 +00001211 unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', [])
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001212 for unsplit_footer in unsplit_footers:
1213 footers += [b.strip() for b in unsplit_footer.split(',')]
Aaron Gable12ef5012017-05-15 14:29:00 -07001214 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001215
1216 def ReviewersFromDescription(self):
1217 """Returns all reviewers listed in the commit description."""
Edward Lemura5799e32020-01-17 19:26:51 +00001218 # We don't support a 'R:' git-footer for reviewers; that is in metadata.
Aaron Gable12ef5012017-05-15 14:29:00 -07001219 tags = [r.strip() for r in self.tags.get('R', '').split(',') if r.strip()]
1220 return sorted(set(tags))
Aaron Gablefc03e672017-05-15 14:09:42 -07001221
1222 def TBRsFromDescription(self):
1223 """Returns all TBR reviewers listed in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001224 tags = [r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()]
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001225 # TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are
1226 # programmatically determined by self-CR+1s.
Edward Lemur69bb8be2020-02-03 20:37:38 +00001227 footers = self.GitFootersFromDescription().get('Tbr', [])
Aaron Gable12ef5012017-05-15 14:29:00 -07001228 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001229
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001230 # TODO(crbug.com/753425): Delete these once we're sure they're unused.
Aaron Gablefc03e672017-05-15 14:09:42 -07001231 @property
1232 def BUG(self):
1233 return ','.join(self.BugsFromDescription())
1234 @property
1235 def R(self):
1236 return ','.join(self.ReviewersFromDescription())
1237 @property
1238 def TBR(self):
1239 return ','.join(self.TBRsFromDescription())
1240
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001241 def AllFiles(self, root=None):
1242 """List all files under source control in the repo."""
1243 raise NotImplementedError()
1244
agable0b65e732016-11-22 09:25:46 -08001245 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001246 """Returns a list of AffectedFile instances for all files in the change.
1247
1248 Args:
1249 include_deletes: If false, deleted files will be filtered out.
sail@chromium.org5538e022011-05-12 17:53:16 +00001250 file_filter: An additional filter to apply.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001251
1252 Returns:
1253 [AffectedFile(path, action), AffectedFile(path, action)]
1254 """
Edward Lemurb9830242019-10-30 22:19:20 +00001255 affected = list(filter(file_filter, self._affected_files))
sail@chromium.org5538e022011-05-12 17:53:16 +00001256
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001257 if include_deletes:
1258 return affected
Edward Lemurb9830242019-10-30 22:19:20 +00001259 return list(filter(lambda x: x.Action() != 'D', affected))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001260
John Budorick16162372018-04-18 10:39:53 -07001261 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001262 """Return a list of the existing text files in a change."""
1263 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +00001264 warn('AffectedTeestableFiles(include_deletes=%s)'
1265 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001266 category=DeprecationWarning,
1267 stacklevel=2)
Edward Lemurb9830242019-10-30 22:19:20 +00001268 return list(filter(
1269 lambda x: x.IsTestableFile(),
1270 self.AffectedFiles(include_deletes=False, **kwargs)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001271
agable0b65e732016-11-22 09:25:46 -08001272 def AffectedTextFiles(self, include_deletes=None):
1273 """An alias to AffectedTestableFiles for backwards compatibility."""
1274 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001275
agable0b65e732016-11-22 09:25:46 -08001276 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001277 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001278 return [af.LocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001279
agable0b65e732016-11-22 09:25:46 -08001280 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001281 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001282 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001283
1284 def RightHandSideLines(self):
Edward Lemura5799e32020-01-17 19:26:51 +00001285 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001286
1287 Lists lines from new or modified text files in the change.
1288
1289 This is useful for doing line-by-line regex checks, like checking for
1290 trailing whitespace.
1291
1292 Yields:
1293 a 3 tuple:
1294 the AffectedFile instance of the current file;
1295 integer line number (1-based); and
1296 the contents of the line as a string.
1297 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001298 return _RightHandSideLinesImpl(
1299 x for x in self.AffectedFiles(include_deletes=False)
agable0b65e732016-11-22 09:25:46 -08001300 if x.IsTestableFile())
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001301
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001302 def OriginalOwnersFiles(self):
1303 """A map from path names of affected OWNERS files to their old content."""
1304 def owners_file_filter(f):
1305 return 'OWNERS' in os.path.split(f.LocalPath())[1]
1306 files = self.AffectedFiles(file_filter=owners_file_filter)
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001307 return {f.LocalPath(): f.OldContents() for f in files}
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001308
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001309
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001310class GitChange(Change):
1311 _AFFECTED_FILES = GitAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001312 scm = 'git'
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001313
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001314 def AllFiles(self, root=None):
1315 """List all files under source control in the repo."""
1316 root = root or self.RepositoryRoot()
1317 return subprocess.check_output(
Aaron Gable7817f022017-12-12 09:43:17 -08001318 ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'],
Josip Sokcevic0b123462021-06-08 20:41:32 +00001319 cwd=root).decode('utf-8', 'ignore').splitlines()
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001320
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001321
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001322def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001323 """Finds all presubmit files that apply to a given set of source files.
1324
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001325 If inherit-review-settings-ok is present right under root, looks for
1326 PRESUBMIT.py in directories enclosing root.
1327
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001328 Args:
1329 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001330 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001331
1332 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001333 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001334 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001335 files = [normpath(os.path.join(root, f)) for f in files]
1336
1337 # List all the individual directories containing files.
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001338 directories = {os.path.dirname(f) for f in files}
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001339
1340 # Ignore root if inherit-review-settings-ok is present.
1341 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
1342 root = None
1343
1344 # Collect all unique directories that may contain PRESUBMIT.py.
1345 candidates = set()
1346 for directory in directories:
1347 while True:
1348 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001349 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001350 candidates.add(directory)
1351 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001352 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001353 parent_dir = os.path.dirname(directory)
1354 if parent_dir == directory:
1355 # We hit the system root directory.
1356 break
1357 directory = parent_dir
1358
1359 # Look for PRESUBMIT.py in all candidate directories.
1360 results = []
1361 for directory in sorted(list(candidates)):
tobiasjsff061c02016-08-17 03:23:57 -07001362 try:
1363 for f in os.listdir(directory):
1364 p = os.path.join(directory, f)
1365 if os.path.isfile(p) and re.match(
1366 r'PRESUBMIT.*\.py$', f) and not f.startswith('PRESUBMIT_test'):
1367 results.append(p)
1368 except OSError:
1369 pass
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001370
tobiasjs2836bcf2016-08-16 04:08:16 -07001371 logging.debug('Presubmit files: %s', ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001372 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001373
1374
rmistry@google.com5626a922015-02-26 14:03:30 +00001375class GetPostUploadExecuter(object):
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001376 def __init__(self, use_python3):
1377 """
1378 Args:
1379 use_python3: if true, will use python3 instead of python2 by default
1380 if USE_PYTHON3 is not specified.
1381 """
1382 self.use_python3 = use_python3
1383
1384 def ExecPresubmitScript(self, script_text, presubmit_path, gerrit_obj,
1385 change):
rmistry@google.com5626a922015-02-26 14:03:30 +00001386 """Executes PostUploadHook() from a single presubmit script.
1387
1388 Args:
1389 script_text: The text of the presubmit script.
1390 presubmit_path: Project script to run.
Edward Lemur016a0872020-02-04 22:13:28 +00001391 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001392 change: The Change object.
1393
1394 Return:
1395 A list of results objects.
1396 """
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001397 if not _ShouldRunPresubmit(script_text, self.use_python3):
1398 return {}
1399
rmistry@google.com5626a922015-02-26 14:03:30 +00001400 context = {}
1401 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001402 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1403 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001404 except Exception as e:
rmistry@google.com5626a922015-02-26 14:03:30 +00001405 raise PresubmitFailure('"%s" had an exception.\n%s'
1406 % (presubmit_path, e))
1407
1408 function_name = 'PostUploadHook'
1409 if function_name not in context:
1410 return {}
1411 post_upload_hook = context[function_name]
1412 if not len(inspect.getargspec(post_upload_hook)[0]) == 3:
1413 raise PresubmitFailure(
1414 'Expected function "PostUploadHook" to take three arguments.')
Edward Lemur016a0872020-02-04 22:13:28 +00001415 return post_upload_hook(gerrit_obj, change, OutputApi(False))
rmistry@google.com5626a922015-02-26 14:03:30 +00001416
1417
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001418def _MergeMasters(masters1, masters2):
1419 """Merges two master maps. Merges also the tests of each builder."""
1420 result = {}
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001421 for (master, builders) in itertools.chain(masters1.items(),
1422 masters2.items()):
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001423 new_builders = result.setdefault(master, {})
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001424 for (builder, tests) in builders.items():
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001425 new_builders.setdefault(builder, set([])).update(tests)
1426 return result
1427
1428
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001429def DoPostUploadExecuter(change, gerrit_obj, verbose, use_python3=False):
rmistry@google.com5626a922015-02-26 14:03:30 +00001430 """Execute the post upload hook.
1431
1432 Args:
1433 change: The Change object.
Edward Lemur016a0872020-02-04 22:13:28 +00001434 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001435 verbose: Prints debug info.
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001436 use_python3: if true, default to using Python3 for presubmit checks
1437 rather than Python2.
rmistry@google.com5626a922015-02-26 14:03:30 +00001438 """
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001439 python_version = 'Python %s' % sys.version_info.major
1440 sys.stdout.write('Running %s post upload checks ...\n' % python_version)
rmistry@google.com5626a922015-02-26 14:03:30 +00001441 presubmit_files = ListRelevantPresubmitFiles(
Edward Lemur6eb1d322020-02-27 22:20:15 +00001442 change.LocalPaths(), change.RepositoryRoot())
rmistry@google.com5626a922015-02-26 14:03:30 +00001443 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001444 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
rmistry@google.com5626a922015-02-26 14:03:30 +00001445 results = []
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001446 executer = GetPostUploadExecuter(use_python3)
rmistry@google.com5626a922015-02-26 14:03:30 +00001447 # The root presubmit file should be executed after the ones in subdirectories.
1448 # i.e. the specific post upload hooks should run before the general ones.
1449 # Thus, reverse the order provided by ListRelevantPresubmitFiles.
1450 presubmit_files.reverse()
1451
1452 for filename in presubmit_files:
1453 filename = os.path.abspath(filename)
1454 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001455 sys.stdout.write('Running %s\n' % filename)
rmistry@google.com5626a922015-02-26 14:03:30 +00001456 # Accept CRLF presubmit script.
1457 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1458 results.extend(executer.ExecPresubmitScript(
Edward Lemur016a0872020-02-04 22:13:28 +00001459 presubmit_script, filename, gerrit_obj, change))
rmistry@google.com5626a922015-02-26 14:03:30 +00001460
Edward Lemur6eb1d322020-02-27 22:20:15 +00001461 if not results:
1462 return 0
1463
1464 sys.stdout.write('\n')
1465 sys.stdout.write('** Post Upload Hook Messages **\n')
1466
1467 exit_code = 0
1468 for result in results:
1469 if result.fatal:
1470 exit_code = 1
1471 result.handle()
1472 sys.stdout.write('\n')
1473
1474 return exit_code
rmistry@google.com5626a922015-02-26 14:03:30 +00001475
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001476class PresubmitExecuter(object):
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001477 def __init__(self, change, committing, verbose, gerrit_obj, dry_run=None,
Dirk Pranke6f0df682021-06-25 00:42:33 +00001478 thread_pool=None, parallel=False, use_python3=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001479 """
1480 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001481 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001482 committing: True if 'git cl land' is running, False if 'git cl upload' is.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001483 gerrit_obj: provides basic Gerrit codereview functionality.
1484 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001485 parallel: if true, all tests reported via input_api.RunTests for all
1486 PRESUBMIT files will be run in parallel.
Dirk Pranke6f0df682021-06-25 00:42:33 +00001487 use_python3: if true, will use python3 instead of python2 by default
1488 if USE_PYTHON3 is not specified.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001489 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001490 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001491 self.committing = committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001492 self.gerrit = gerrit_obj
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001493 self.verbose = verbose
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001494 self.dry_run = dry_run
Daniel Cheng7227d212017-11-17 08:12:37 -08001495 self.more_cc = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001496 self.thread_pool = thread_pool
1497 self.parallel = parallel
Dirk Pranke6f0df682021-06-25 00:42:33 +00001498 self.use_python3 = use_python3
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001499
1500 def ExecPresubmitScript(self, script_text, presubmit_path):
1501 """Executes a single presubmit script.
1502
1503 Args:
1504 script_text: The text of the presubmit script.
1505 presubmit_path: The path to the presubmit file (this will be reported via
1506 input_api.PresubmitLocalPath()).
1507
1508 Return:
1509 A list of result objects, empty if no problems.
1510 """
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001511 if not _ShouldRunPresubmit(script_text, self.use_python3):
1512 return []
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001513
chase@chromium.org8e416c82009-10-06 04:30:44 +00001514 # Change to the presubmit file's directory to support local imports.
1515 main_path = os.getcwd()
Saagar Sanghavi98b332f2020-07-31 17:19:15 +00001516 presubmit_dir = os.path.dirname(presubmit_path)
1517 os.chdir(presubmit_dir)
chase@chromium.org8e416c82009-10-06 04:30:44 +00001518
1519 # Load the presubmit script into context.
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001520 input_api = InputApi(self.change, presubmit_path, self.committing,
Aaron Gable668c1d82018-04-03 10:19:16 -07001521 self.verbose, gerrit_obj=self.gerrit,
Edward Lesmes8e282792018-04-03 18:50:29 -04001522 dry_run=self.dry_run, thread_pool=self.thread_pool,
1523 parallel=self.parallel)
Daniel Cheng7227d212017-11-17 08:12:37 -08001524 output_api = OutputApi(self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001525 context = {}
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001526
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001527 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001528 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1529 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001530 except Exception as e:
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001531 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001532
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001533 context['__args'] = (input_api, output_api)
Ben Pastene8351dc12020-08-06 05:01:35 +00001534
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001535 # Get path of presubmit directory relative to repository root.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001536 # Always use forward slashes, so that path is same in *nix and Windows
1537 root = input_api.change.RepositoryRoot()
1538 rel_path = os.path.relpath(presubmit_dir, root)
1539 rel_path = rel_path.replace(os.path.sep, '/')
Ben Pastene8351dc12020-08-06 05:01:35 +00001540
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001541 # Get the URL of git remote origin and use it to identify host and project
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001542 host = project = ''
1543 if self.gerrit:
1544 host = self.gerrit.host or ''
1545 project = self.gerrit.project or ''
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001546
1547 # Prefix for test names
1548 prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path)
1549
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001550 # Perform all the desired presubmit checks.
1551 results = []
Ben Pastene8351dc12020-08-06 05:01:35 +00001552
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001553 try:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001554 version = [
1555 int(x) for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.')
1556 ]
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001557
Scott Leecc2fe9b2020-11-19 19:38:06 +00001558 with rdb_wrapper.client(prefix) as sink:
1559 if version >= [2, 0, 0]:
Andrew Grievebdfb8f22022-02-02 21:56:47 +00001560 # Copy the keys to prevent "dictionary changed size during iteration"
1561 # exception if checks add globals to context. E.g. sometimes the
1562 # Python runtime will add __warningregistry__.
1563 for function_name in list(context.keys()):
Scott Leecc2fe9b2020-11-19 19:38:06 +00001564 if not function_name.startswith('Check'):
1565 continue
1566 if function_name.endswith('Commit') and not self.committing:
1567 continue
1568 if function_name.endswith('Upload') and self.committing:
1569 continue
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001570 logging.debug('Running %s in %s', function_name, presubmit_path)
1571 results.extend(
Scott Leecc2fe9b2020-11-19 19:38:06 +00001572 self._run_check_function(function_name, context, sink))
1573 logging.debug('Running %s done.', function_name)
1574 self.more_cc.extend(output_api.more_cc)
1575
1576 else: # Old format
1577 if self.committing:
1578 function_name = 'CheckChangeOnCommit'
1579 else:
1580 function_name = 'CheckChangeOnUpload'
Andrew Grievebdfb8f22022-02-02 21:56:47 +00001581 if function_name in list(context.keys()):
Scott Leecc2fe9b2020-11-19 19:38:06 +00001582 logging.debug('Running %s in %s', function_name, presubmit_path)
1583 results.extend(
1584 self._run_check_function(function_name, context, sink))
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001585 logging.debug('Running %s done.', function_name)
1586 self.more_cc.extend(output_api.more_cc)
1587
1588 finally:
1589 for f in input_api._named_temporary_files:
1590 os.remove(f)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001591
chase@chromium.org8e416c82009-10-06 04:30:44 +00001592 # Return the process to the original working directory.
1593 os.chdir(main_path)
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001594 return results
1595
Scott Leecc2fe9b2020-11-19 19:38:06 +00001596 def _run_check_function(self, function_name, context, sink=None):
1597 """Evaluates and returns the result of a given presubmit function.
1598
1599 If sink is given, the result of the presubmit function will be reported
1600 to the ResultSink.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001601
1602 Args:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001603 function_name: the name of the presubmit function to evaluate
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001604 context: a context dictionary in which the function will be evaluated
Scott Leecc2fe9b2020-11-19 19:38:06 +00001605 sink: an instance of ResultSink. None, by default.
1606 Returns:
1607 the result of the presubmit function call.
1608 """
1609 start_time = time_time()
1610 try:
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001611 result = eval(function_name + '(*__args)', context)
1612 self._check_result_type(result)
Allen Webbfe7d7092021-05-18 02:05:49 +00001613 except Exception:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001614 if sink:
1615 elapsed_time = time_time() - start_time
1616 sink.report(function_name, rdb_wrapper.STATUS_FAIL, elapsed_time)
Allen Webbfe7d7092021-05-18 02:05:49 +00001617 # TODO(crbug.com/953884): replace reraise with native py3:
Allen Webbb65bbfe2021-05-11 21:22:01 +00001618 # raise .. from e
Allen Webbfe7d7092021-05-18 02:05:49 +00001619 e_type, e_value, e_tb = sys.exc_info()
Dirk Pranke6fc394f2021-05-26 23:25:14 +00001620 print('Evaluation of %s failed: %s' % (function_name, e_value))
1621 six.reraise(e_type, e_value, e_tb)
Scott Leecc2fe9b2020-11-19 19:38:06 +00001622
Bruce Dawson2bdc49f2021-05-21 19:13:03 +00001623 elapsed_time = time_time() - start_time
1624 if elapsed_time > 10.0:
1625 sys.stdout.write(
1626 '%s took %.1fs to run.\n' % (function_name, elapsed_time))
Scott Leecc2fe9b2020-11-19 19:38:06 +00001627 if sink:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001628 status = rdb_wrapper.STATUS_PASS
1629 if any(r.fatal for r in result):
1630 status = rdb_wrapper.STATUS_FAIL
1631 sink.report(function_name, status, elapsed_time)
1632
1633 return result
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001634
Saagar Sanghavi9949ab72020-07-20 20:56:40 +00001635 def _check_result_type(self, result):
1636 """Helper function which ensures result is a list, and all elements are
1637 instances of OutputApi.PresubmitResult"""
1638 if not isinstance(result, (tuple, list)):
1639 raise PresubmitFailure('Presubmit functions must return a tuple or list')
1640 if not all(isinstance(res, OutputApi.PresubmitResult) for res in result):
1641 raise PresubmitFailure(
1642 'All presubmit results must be of types derived from '
1643 'output_api.PresubmitResult')
1644
1645
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001646def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001647 committing,
1648 verbose,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001649 default_presubmit,
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001650 may_prompt,
Aaron Gable668c1d82018-04-03 10:19:16 -07001651 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001652 dry_run=None,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00001653 parallel=False,
Dirk Pranke6f0df682021-06-25 00:42:33 +00001654 json_output=None,
1655 use_python3=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001656 """Runs all presubmit checks that apply to the files in the change.
1657
1658 This finds all PRESUBMIT.py files in directories enclosing the files in the
1659 change (up to the repository root) and calls the relevant entrypoint function
1660 depending on whether the change is being committed or uploaded.
1661
1662 Prints errors, warnings and notifications. Prompts the user for warnings
1663 when needed.
1664
1665 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001666 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001667 committing: True if 'git cl land' is running, False if 'git cl upload' is.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001668 verbose: Prints debug info.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001669 default_presubmit: A default presubmit script to execute in any case.
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001670 may_prompt: Enable (y/n) questions on warning or error. If False,
1671 any questions are answered with yes by default.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001672 gerrit_obj: provides basic Gerrit codereview functionality.
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001673 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001674 parallel: if true, all tests specified by input_api.RunTests in all
1675 PRESUBMIT files will be run in parallel.
Dirk Pranke6f0df682021-06-25 00:42:33 +00001676 use_python3: if true, default to using Python3 for presubmit checks
1677 rather than Python2.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001678 Return:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001679 1 if presubmit checks failed or 0 otherwise.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001680 """
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001681 old_environ = os.environ
1682 try:
1683 # Make sure python subprocesses won't generate .pyc files.
1684 os.environ = os.environ.copy()
Edward Lemurb9830242019-10-30 22:19:20 +00001685 os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001686
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001687 python_version = 'Python %s' % sys.version_info.major
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001688 if committing:
Dirk Pranke6fc394f2021-05-26 23:25:14 +00001689 sys.stdout.write('Running %s presubmit commit checks ...\n' %
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001690 python_version)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001691 else:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001692 sys.stdout.write('Running %s presubmit upload checks ...\n' %
1693 python_version)
Edward Lemurecc27072020-01-06 16:42:34 +00001694 start_time = time_time()
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001695 presubmit_files = ListRelevantPresubmitFiles(
agable0b65e732016-11-22 09:25:46 -08001696 change.AbsoluteLocalPaths(), change.RepositoryRoot())
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001697 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001698 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001699 results = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001700 thread_pool = ThreadPool()
Edward Lemur7e3c67f2018-07-20 20:52:49 +00001701 executer = PresubmitExecuter(change, committing, verbose, gerrit_obj,
Dirk Pranke6f0df682021-06-25 00:42:33 +00001702 dry_run, thread_pool, parallel, use_python3)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001703 if default_presubmit:
1704 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001705 sys.stdout.write('Running default presubmit script.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001706 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
1707 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
1708 for filename in presubmit_files:
1709 filename = os.path.abspath(filename)
1710 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001711 sys.stdout.write('Running %s\n' % filename)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001712 # Accept CRLF presubmit script.
1713 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1714 results += executer.ExecPresubmitScript(presubmit_script, filename)
Edward Lesmes8e282792018-04-03 18:50:29 -04001715 results += thread_pool.RunAsync()
1716
Edward Lemur6eb1d322020-02-27 22:20:15 +00001717 messages = {}
1718 should_prompt = False
1719 presubmits_failed = False
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001720 for result in results:
1721 if result.fatal:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001722 presubmits_failed = True
1723 messages.setdefault('ERRORS', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001724 elif result.should_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001725 should_prompt = True
1726 messages.setdefault('Warnings', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001727 else:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001728 messages.setdefault('Messages', []).append(result)
pam@chromium.orged9a0832009-09-09 22:48:55 +00001729
Edward Lemur6eb1d322020-02-27 22:20:15 +00001730 sys.stdout.write('\n')
1731 for name, items in messages.items():
1732 sys.stdout.write('** Presubmit %s **\n' % name)
1733 for item in items:
1734 item.handle()
1735 sys.stdout.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001736
Edward Lemurecc27072020-01-06 16:42:34 +00001737 total_time = time_time() - start_time
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001738 if total_time > 1.0:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001739 sys.stdout.write(
1740 'Presubmit checks took %.1fs to calculate.\n\n' % total_time)
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001741
Edward Lemur6eb1d322020-02-27 22:20:15 +00001742 if not should_prompt and not presubmits_failed:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001743 sys.stdout.write('%s presubmit checks passed.\n' % python_version)
Josip Sokcevic7592e0a2022-01-12 00:57:54 +00001744 elif should_prompt and not presubmits_failed:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001745 sys.stdout.write('There were %s presubmit warnings. ' % python_version)
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001746 if may_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001747 presubmits_failed = not prompt_should_continue(
1748 'Are you sure you wish to continue? (y/N): ')
Garrett Beatyacb807e2020-08-28 18:17:24 +00001749 else:
1750 sys.stdout.write('\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001751
Edward Lemur1dc66e12020-02-21 21:36:34 +00001752 if json_output:
1753 # Write the presubmit results to json output
1754 presubmit_results = {
1755 'errors': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001756 error.json_format()
1757 for error in messages.get('ERRORS', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001758 ],
1759 'notifications': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001760 notification.json_format()
1761 for notification in messages.get('Messages', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001762 ],
1763 'warnings': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001764 warning.json_format()
1765 for warning in messages.get('Warnings', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001766 ],
Edward Lemur6eb1d322020-02-27 22:20:15 +00001767 'more_cc': executer.more_cc,
Edward Lemur1dc66e12020-02-21 21:36:34 +00001768 }
1769
1770 gclient_utils.FileWrite(
1771 json_output, json.dumps(presubmit_results, sort_keys=True))
1772
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001773 global _ASKED_FOR_FEEDBACK
1774 # Ask for feedback one time out of 5.
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001775 if (results and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
Edward Lemur6eb1d322020-02-27 22:20:15 +00001776 sys.stdout.write(
maruel@chromium.org1ce8e662014-01-14 15:23:00 +00001777 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n'
1778 'to figure out which PRESUBMIT.py was run, then run git blame\n'
1779 'on the file to figure out who to ask for help.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001780 _ASKED_FOR_FEEDBACK = True
Edward Lemur6eb1d322020-02-27 22:20:15 +00001781
1782 return 1 if presubmits_failed else 0
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001783 finally:
1784 os.environ = old_environ
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001785
1786
Edward Lemur50984a62020-02-06 18:10:18 +00001787def _scan_sub_dirs(mask, recursive):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001788 if not recursive:
pgervais@chromium.orge57b09d2014-05-07 00:58:13 +00001789 return [x for x in glob.glob(mask) if x not in ('.svn', '.git')]
Lei Zhang9611c4c2017-04-04 01:41:56 -07001790
1791 results = []
1792 for root, dirs, files in os.walk('.'):
1793 if '.svn' in dirs:
1794 dirs.remove('.svn')
1795 if '.git' in dirs:
1796 dirs.remove('.git')
1797 for name in files:
1798 if fnmatch.fnmatch(name, mask):
1799 results.append(os.path.join(root, name))
1800 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001801
1802
Edward Lemur50984a62020-02-06 18:10:18 +00001803def _parse_files(args, recursive):
tobiasjs2836bcf2016-08-16 04:08:16 -07001804 logging.debug('Searching for %s', args)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001805 files = []
1806 for arg in args:
Edward Lemur50984a62020-02-06 18:10:18 +00001807 files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001808 return files
1809
1810
Edward Lemur50984a62020-02-06 18:10:18 +00001811def _parse_change(parser, options):
1812 """Process change options.
1813
1814 Args:
1815 parser: The parser used to parse the arguments from command line.
1816 options: The arguments parsed from command line.
1817 Returns:
1818 A GitChange if the change root is a git repository, or a Change otherwise.
1819 """
1820 if options.files and options.all_files:
1821 parser.error('<files> cannot be specified when --all-files is set.')
1822
Edward Lesmes44ea3ff2020-02-05 00:14:30 +00001823 change_scm = scm.determine_scm(options.root)
Edward Lemur50984a62020-02-06 18:10:18 +00001824 if change_scm != 'git' and not options.files:
1825 parser.error('<files> is not optional for unversioned directories.')
1826
1827 if options.files:
1828 change_files = _parse_files(options.files, options.recursive)
1829 elif options.all_files:
1830 change_files = [('M', f) for f in scm.GIT.GetAllFiles(options.root)]
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001831 else:
Edward Lemur50984a62020-02-06 18:10:18 +00001832 change_files = scm.GIT.CaptureStatus(
Edward Lemur7f6dec02020-02-06 20:23:58 +00001833 options.root, options.upstream or None)
Edward Lemur50984a62020-02-06 18:10:18 +00001834
1835 logging.info('Found %d file(s).', len(change_files))
1836
1837 change_class = GitChange if change_scm == 'git' else Change
1838 return change_class(
1839 options.name,
1840 options.description,
1841 options.root,
1842 change_files,
1843 options.issue,
1844 options.patchset,
1845 options.author,
1846 upstream=options.upstream)
1847
1848
1849def _parse_gerrit_options(parser, options):
1850 """Process gerrit options.
1851
1852 SIDE EFFECTS: Modifies options.author and options.description from Gerrit if
1853 options.gerrit_fetch is set.
1854
1855 Args:
1856 parser: The parser used to parse the arguments from command line.
1857 options: The arguments parsed from command line.
1858 Returns:
Stephen Martinisfb09de22021-02-25 03:41:13 +00001859 A GerritAccessor object if options.gerrit_url is set, or None otherwise.
Edward Lemur50984a62020-02-06 18:10:18 +00001860 """
1861 gerrit_obj = None
Stephen Martinisfb09de22021-02-25 03:41:13 +00001862 if options.gerrit_url:
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001863 gerrit_obj = GerritAccessor(
1864 url=options.gerrit_url,
1865 project=options.gerrit_project,
1866 branch=options.gerrit_branch)
Edward Lemur50984a62020-02-06 18:10:18 +00001867
1868 if not options.gerrit_fetch:
1869 return gerrit_obj
1870
1871 if not options.gerrit_url or not options.issue or not options.patchset:
1872 parser.error(
1873 '--gerrit_fetch requires --gerrit_url, --issue and --patchset.')
1874
1875 options.author = gerrit_obj.GetChangeOwner(options.issue)
1876 options.description = gerrit_obj.GetChangeDescription(
1877 options.issue, options.patchset)
1878
1879 logging.info('Got author: "%s"', options.author)
1880 logging.info('Got description: """\n%s\n"""', options.description)
1881
1882 return gerrit_obj
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001883
1884
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001885@contextlib.contextmanager
1886def canned_check_filter(method_names):
1887 filtered = {}
1888 try:
1889 for method_name in method_names:
1890 if not hasattr(presubmit_canned_checks, method_name):
Gavin Make6a62332020-12-04 21:57:10 +00001891 logging.warning('Skipping unknown "canned" check %s' % method_name)
Aaron Gableecee74c2018-04-02 15:13:08 -07001892 continue
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001893 filtered[method_name] = getattr(presubmit_canned_checks, method_name)
1894 setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
1895 yield
1896 finally:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001897 for name, method in filtered.items():
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001898 setattr(presubmit_canned_checks, name, method)
1899
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +00001900
sbc@chromium.org013731e2015-02-26 18:28:43 +00001901def main(argv=None):
Edward Lemura5799e32020-01-17 19:26:51 +00001902 parser = argparse.ArgumentParser(usage='%(prog)s [options] <files...>')
1903 hooks = parser.add_mutually_exclusive_group()
1904 hooks.add_argument('-c', '--commit', action='store_true',
1905 help='Use commit instead of upload checks.')
1906 hooks.add_argument('-u', '--upload', action='store_false', dest='commit',
1907 help='Use upload instead of commit checks.')
Edward Lemur75526302020-02-27 22:31:05 +00001908 hooks.add_argument('--post_upload', action='store_true',
1909 help='Run post-upload commit hooks.')
Edward Lemura5799e32020-01-17 19:26:51 +00001910 parser.add_argument('-r', '--recursive', action='store_true',
1911 help='Act recursively.')
1912 parser.add_argument('-v', '--verbose', action='count', default=0,
1913 help='Use 2 times for more debug info.')
1914 parser.add_argument('--name', default='no name')
1915 parser.add_argument('--author')
Edward Lemur227d5102020-02-25 23:45:35 +00001916 desc = parser.add_mutually_exclusive_group()
1917 desc.add_argument('--description', default='', help='The change description.')
1918 desc.add_argument('--description_file',
1919 help='File to read change description from.')
Edward Lemura5799e32020-01-17 19:26:51 +00001920 parser.add_argument('--issue', type=int, default=0)
1921 parser.add_argument('--patchset', type=int, default=0)
1922 parser.add_argument('--root', default=os.getcwd(),
1923 help='Search for PRESUBMIT.py up to this directory. '
1924 'If inherit-review-settings-ok is present in this '
1925 'directory, parent directories up to the root file '
1926 'system directories will also be searched.')
1927 parser.add_argument('--upstream',
1928 help='Git only: the base ref or upstream branch against '
1929 'which the diff should be computed.')
1930 parser.add_argument('--default_presubmit')
1931 parser.add_argument('--may_prompt', action='store_true', default=False)
1932 parser.add_argument('--skip_canned', action='append', default=[],
1933 help='A list of checks to skip which appear in '
1934 'presubmit_canned_checks. Can be provided multiple times '
1935 'to skip multiple canned checks.')
1936 parser.add_argument('--dry_run', action='store_true', help=argparse.SUPPRESS)
1937 parser.add_argument('--gerrit_url', help=argparse.SUPPRESS)
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001938 parser.add_argument('--gerrit_project', help=argparse.SUPPRESS)
1939 parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS)
Edward Lemura5799e32020-01-17 19:26:51 +00001940 parser.add_argument('--gerrit_fetch', action='store_true',
1941 help=argparse.SUPPRESS)
1942 parser.add_argument('--parallel', action='store_true',
1943 help='Run all tests specified by input_api.RunTests in '
1944 'all PRESUBMIT files in parallel.')
1945 parser.add_argument('--json_output',
1946 help='Write presubmit errors to json output.')
Edward Lemur1dc66e12020-02-21 21:36:34 +00001947 parser.add_argument('--all_files', action='store_true',
Edward Lemur50984a62020-02-06 18:10:18 +00001948 help='Mark all files under source control as modified.')
Edward Lemura5799e32020-01-17 19:26:51 +00001949 parser.add_argument('files', nargs='*',
1950 help='List of files to be marked as modified when '
1951 'executing presubmit or post-upload hooks. fnmatch '
1952 'wildcards can also be used.')
Dirk Pranke6f0df682021-06-25 00:42:33 +00001953 parser.add_argument('--use-python3', action='store_true',
1954 help='Use python3 for presubmit checks by default')
Edward Lemura5799e32020-01-17 19:26:51 +00001955 options = parser.parse_args(argv)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001956
Erik Staabcca5c492020-04-16 17:40:07 +00001957 log_level = logging.ERROR
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001958 if options.verbose >= 2:
Erik Staabcca5c492020-04-16 17:40:07 +00001959 log_level = logging.DEBUG
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001960 elif options.verbose:
Erik Staabcca5c492020-04-16 17:40:07 +00001961 log_level = logging.INFO
1962 log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d '
1963 '%(filename)s] %(message)s')
1964 logging.basicConfig(format=log_format, level=log_level)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001965
Edward Lemur227d5102020-02-25 23:45:35 +00001966 if options.description_file:
1967 options.description = gclient_utils.FileRead(options.description_file)
Edward Lemur50984a62020-02-06 18:10:18 +00001968 gerrit_obj = _parse_gerrit_options(parser, options)
1969 change = _parse_change(parser, options)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001970
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001971 try:
Edward Lemur75526302020-02-27 22:31:05 +00001972 if options.post_upload:
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001973 return DoPostUploadExecuter(change, gerrit_obj, options.verbose,
1974 options.use_python3)
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001975 with canned_check_filter(options.skip_canned):
Edward Lemur6eb1d322020-02-27 22:20:15 +00001976 return DoPresubmitChecks(
Edward Lemur50984a62020-02-06 18:10:18 +00001977 change,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001978 options.commit,
1979 options.verbose,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001980 options.default_presubmit,
1981 options.may_prompt,
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001982 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001983 options.dry_run,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00001984 options.parallel,
Dirk Pranke6f0df682021-06-25 00:42:33 +00001985 options.json_output,
1986 options.use_python3)
Raul Tambre7c938462019-05-24 16:35:35 +00001987 except PresubmitFailure as e:
Raul Tambre80ee78e2019-05-06 22:41:05 +00001988 print(e, file=sys.stderr)
1989 print('Maybe your depot_tools is out of date?', file=sys.stderr)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001990 return 2
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001991
1992
1993if __name__ == '__main__':
maruel@chromium.org35625c72011-03-23 17:34:02 +00001994 fix_encoding.fix_encoding()
sbc@chromium.org013731e2015-02-26 18:28:43 +00001995 try:
1996 sys.exit(main())
1997 except KeyboardInterrupt:
1998 sys.stderr.write('interrupted\n')
sergiybf8a3b382016-07-05 11:21:30 -07001999 sys.exit(2)