blob: 341c08e50aa0fd5dc8315a0cdaf851c53a4597a4 [file] [log] [blame]
Josip Sokcevic4de5dea2022-03-23 21:15:14 +00001#!/usr/bin/env python3
maruel@chromium.org3bbf2942012-01-10 16:52:06 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
Raul Tambre80ee78e2019-05-06 22:41:05 +00009from __future__ import print_function
10
Saagar Sanghavi99816902020-08-11 22:41:25 +000011__version__ = '2.0.0'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000012
13# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
14# caching (between all different invocations of presubmit scripts for a given
15# change). We should add it as our presubmit scripts start feeling slow.
16
Edward Lemura5799e32020-01-17 19:26:51 +000017import argparse
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +090018import ast # Exposed through the API.
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +000019import contextlib
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +000020import cpplint
dcheng091b7db2016-06-16 01:27:51 -070021import fnmatch # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000022import glob
asvitkine@chromium.org15169952011-09-27 14:30:53 +000023import inspect
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +000024import itertools
maruel@chromium.org4f6852c2012-04-20 20:39:20 +000025import json # Exposed through the API.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000026import logging
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000027import multiprocessing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000028import os # Somewhat exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000029import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import re # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040031import signal
Allen Webbfe7d7092021-05-18 02:05:49 +000032import six
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033import sys # Parts exposed through API.
34import tempfile # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040035import threading
jam@chromium.org2a891dc2009-08-20 20:33:37 +000036import time
Edward Lemurde9e3ca2019-10-24 21:13:31 +000037import traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +000038import unittest # Exposed through the API.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000039from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040
41# Local imports.
maruel@chromium.org35625c72011-03-23 17:34:02 +000042import fix_encoding
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +000043import gclient_paths # Exposed through the API
44import gclient_utils
Aaron Gableb584c4f2017-04-26 16:28:08 -070045import git_footers
tandrii@chromium.org015ebae2016-04-25 19:37:22 +000046import gerrit_util
Gavin Maka9677a52022-08-08 22:30:27 +000047import owners as owners_db
Edward Lesmes9ce03f82021-01-12 20:13:31 +000048import owners_client
Jochen Eisinger76f5fc62017-04-07 16:27:46 +020049import owners_finder
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000050import presubmit_canned_checks
Saagar Sanghavi9949ab72020-07-20 20:56:40 +000051import rdb_wrapper
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000052import scm
maruel@chromium.org84f4fe32011-04-06 13:26:45 +000053import subprocess2 as subprocess # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000054
Edward Lemur16af3562019-10-17 22:11:33 +000055if sys.version_info.major == 2:
56 # TODO(1009814): Expose urllib2 only through urllib_request and urllib_error
57 import urllib2 # Exposed through the API.
58 import urlparse
59 import urllib2 as urllib_request
60 import urllib2 as urllib_error
61else:
62 import urllib.parse as urlparse
63 import urllib.request as urllib_request
64 import urllib.error as urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000065
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000066# Ask for feedback only once in program lifetime.
67_ASKED_FOR_FEEDBACK = False
68
Edward Lemurecc27072020-01-06 16:42:34 +000069def time_time():
70 # Use this so that it can be mocked in tests without interfering with python
71 # system machinery.
72 return time.time()
73
74
maruel@chromium.org899e1c12011-04-07 17:03:18 +000075class PresubmitFailure(Exception):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000076 pass
77
78
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000079class CommandData(object):
Edward Lemur940c2822019-08-23 00:34:25 +000080 def __init__(self, name, cmd, kwargs, message, python3=False):
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000081 self.name = name
82 self.cmd = cmd
Edward Lesmes8e282792018-04-03 18:50:29 -040083 self.stdin = kwargs.get('stdin', None)
Edward Lemur2d6b67c2019-08-23 22:25:41 +000084 self.kwargs = kwargs.copy()
Edward Lesmes8e282792018-04-03 18:50:29 -040085 self.kwargs['stdout'] = subprocess.PIPE
86 self.kwargs['stderr'] = subprocess.STDOUT
87 self.kwargs['stdin'] = subprocess.PIPE
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000088 self.message = message
89 self.info = None
Edward Lemur940c2822019-08-23 00:34:25 +000090 self.python3 = python3
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000091
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000092
Edward Lesmes8e282792018-04-03 18:50:29 -040093# Adapted from
94# https://github.com/google/gtest-parallel/blob/master/gtest_parallel.py#L37
95#
96# An object that catches SIGINT sent to the Python process and notices
97# if processes passed to wait() die by SIGINT (we need to look for
98# both of those cases, because pressing Ctrl+C can result in either
99# the main process or one of the subprocesses getting the signal).
100#
101# Before a SIGINT is seen, wait(p) will simply call p.wait() and
102# return the result. Once a SIGINT has been seen (in the main process
103# or a subprocess, including the one the current call is waiting for),
Edward Lemur9a5bb612019-09-26 02:01:52 +0000104# wait(p) will call p.terminate().
Edward Lesmes8e282792018-04-03 18:50:29 -0400105class SigintHandler(object):
Edward Lesmes8e282792018-04-03 18:50:29 -0400106 sigint_returncodes = {-signal.SIGINT, # Unix
107 -1073741510, # Windows
108 }
109 def __init__(self):
110 self.__lock = threading.Lock()
111 self.__processes = set()
112 self.__got_sigint = False
Edward Lemur9a5bb612019-09-26 02:01:52 +0000113 self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt)
Edward Lesmes8e282792018-04-03 18:50:29 -0400114
115 def __on_sigint(self):
116 self.__got_sigint = True
117 while self.__processes:
118 try:
119 self.__processes.pop().terminate()
120 except OSError:
121 pass
122
Edward Lemur9a5bb612019-09-26 02:01:52 +0000123 def interrupt(self, signal_num, frame):
Edward Lesmes8e282792018-04-03 18:50:29 -0400124 with self.__lock:
125 self.__on_sigint()
Edward Lemur9a5bb612019-09-26 02:01:52 +0000126 self.__previous_signal(signal_num, frame)
Edward Lesmes8e282792018-04-03 18:50:29 -0400127
128 def got_sigint(self):
129 with self.__lock:
130 return self.__got_sigint
131
132 def wait(self, p, stdin):
133 with self.__lock:
134 if self.__got_sigint:
135 p.terminate()
136 self.__processes.add(p)
137 stdout, stderr = p.communicate(stdin)
138 code = p.returncode
139 with self.__lock:
140 self.__processes.discard(p)
141 if code in self.sigint_returncodes:
142 self.__on_sigint()
Edward Lesmes8e282792018-04-03 18:50:29 -0400143 return stdout, stderr
144
145sigint_handler = SigintHandler()
146
147
Edward Lemurecc27072020-01-06 16:42:34 +0000148class Timer(object):
149 def __init__(self, timeout, fn):
150 self.completed = False
151 self._fn = fn
152 self._timer = threading.Timer(timeout, self._onTimer) if timeout else None
153
154 def __enter__(self):
155 if self._timer:
156 self._timer.start()
157 return self
158
159 def __exit__(self, _type, _value, _traceback):
160 if self._timer:
161 self._timer.cancel()
162
163 def _onTimer(self):
164 self._fn()
165 self.completed = True
166
167
Edward Lesmes8e282792018-04-03 18:50:29 -0400168class ThreadPool(object):
Edward Lemurecc27072020-01-06 16:42:34 +0000169 def __init__(self, pool_size=None, timeout=None):
170 self.timeout = timeout
Edward Lesmes8e282792018-04-03 18:50:29 -0400171 self._pool_size = pool_size or multiprocessing.cpu_count()
Bruce Dawson8254f062022-06-17 17:33:08 +0000172 if sys.platform == 'win32':
173 # TODO(crbug.com/1190269) - we can't use more than 56 child processes on
174 # Windows or Python3 may hang.
175 self._pool_size = min(self._pool_size, 56)
Edward Lesmes8e282792018-04-03 18:50:29 -0400176 self._messages = []
177 self._messages_lock = threading.Lock()
178 self._tests = []
179 self._tests_lock = threading.Lock()
180 self._nonparallel_tests = []
181
Edward Lemurecc27072020-01-06 16:42:34 +0000182 def _GetCommand(self, test):
Edward Lemur940c2822019-08-23 00:34:25 +0000183 vpython = 'vpython'
184 if test.python3:
185 vpython += '3'
186 if sys.platform == 'win32':
187 vpython += '.bat'
Edward Lesmes8e282792018-04-03 18:50:29 -0400188
189 cmd = test.cmd
190 if cmd[0] == 'python':
191 cmd = list(cmd)
192 cmd[0] = vpython
193 elif cmd[0].endswith('.py'):
194 cmd = [vpython] + cmd
195
Edward Lemur336e51f2019-11-14 21:42:04 +0000196 # On Windows, scripts on the current directory take precedence over PATH, so
197 # that when testing depot_tools on Windows, calling `vpython.bat` will
198 # execute the copy of vpython of the depot_tools under test instead of the
199 # one in the bot.
200 # As a workaround, we run the tests from the parent directory instead.
201 if (cmd[0] == vpython and
202 'cwd' in test.kwargs and
203 os.path.basename(test.kwargs['cwd']) == 'depot_tools'):
204 test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd'])
205 cmd[1] = os.path.join('depot_tools', cmd[1])
206
Edward Lemurecc27072020-01-06 16:42:34 +0000207 return cmd
208
209 def _RunWithTimeout(self, cmd, stdin, kwargs):
210 p = subprocess.Popen(cmd, **kwargs)
211 with Timer(self.timeout, p.terminate) as timer:
212 stdout, _ = sigint_handler.wait(p, stdin)
Josip Sokcevic6635baf2021-11-19 02:04:39 +0000213 stdout = stdout.decode('utf-8', 'ignore')
Edward Lemurecc27072020-01-06 16:42:34 +0000214 if timer.completed:
215 stdout = 'Process timed out after %ss\n%s' % (self.timeout, stdout)
Josip Sokcevic6635baf2021-11-19 02:04:39 +0000216 return p.returncode, stdout
Edward Lemurecc27072020-01-06 16:42:34 +0000217
218 def CallCommand(self, test):
219 """Runs an external program.
220
Edward Lemura5799e32020-01-17 19:26:51 +0000221 This function converts invocation of .py files and invocations of 'python'
Edward Lemurecc27072020-01-06 16:42:34 +0000222 to vpython invocations.
223 """
224 cmd = self._GetCommand(test)
Edward Lesmes8e282792018-04-03 18:50:29 -0400225 try:
Edward Lemurecc27072020-01-06 16:42:34 +0000226 start = time_time()
227 returncode, stdout = self._RunWithTimeout(cmd, test.stdin, test.kwargs)
228 duration = time_time() - start
Edward Lemur7cf94382019-11-15 22:36:41 +0000229 except Exception:
Edward Lemurecc27072020-01-06 16:42:34 +0000230 duration = time_time() - start
Edward Lesmes8e282792018-04-03 18:50:29 -0400231 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000232 '%s\n%s exec failure (%4.2fs)\n%s' % (
233 test.name, ' '.join(cmd), duration, traceback.format_exc()))
Edward Lemurde9e3ca2019-10-24 21:13:31 +0000234
Edward Lemurecc27072020-01-06 16:42:34 +0000235 if returncode != 0:
Edward Lesmes8e282792018-04-03 18:50:29 -0400236 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000237 '%s\n%s (%4.2fs) failed\n%s' % (
238 test.name, ' '.join(cmd), duration, stdout))
Edward Lemurecc27072020-01-06 16:42:34 +0000239
Edward Lesmes8e282792018-04-03 18:50:29 -0400240 if test.info:
Edward Lemur7cf94382019-11-15 22:36:41 +0000241 return test.info('%s\n%s (%4.2fs)' % (test.name, ' '.join(cmd), duration))
Edward Lesmes8e282792018-04-03 18:50:29 -0400242
243 def AddTests(self, tests, parallel=True):
244 if parallel:
245 self._tests.extend(tests)
246 else:
247 self._nonparallel_tests.extend(tests)
248
249 def RunAsync(self):
250 self._messages = []
251
252 def _WorkerFn():
253 while True:
254 test = None
255 with self._tests_lock:
256 if not self._tests:
257 break
258 test = self._tests.pop()
259 result = self.CallCommand(test)
260 if result:
261 with self._messages_lock:
262 self._messages.append(result)
263
264 def _StartDaemon():
265 t = threading.Thread(target=_WorkerFn)
266 t.daemon = True
267 t.start()
268 return t
269
270 while self._nonparallel_tests:
271 test = self._nonparallel_tests.pop()
272 result = self.CallCommand(test)
273 if result:
274 self._messages.append(result)
275
276 if self._tests:
277 threads = [_StartDaemon() for _ in range(self._pool_size)]
278 for worker in threads:
279 worker.join()
280
281 return self._messages
282
283
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000284def normpath(path):
285 '''Version of os.path.normpath that also changes backward slashes to
286 forward slashes when not running on Windows.
287 '''
288 # This is safe to always do because the Windows version of os.path.normpath
289 # will replace forward slashes with backward slashes.
290 path = path.replace(os.sep, '/')
291 return os.path.normpath(path)
292
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000293
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000294def _RightHandSideLinesImpl(affected_files):
295 """Implements RightHandSideLines for InputApi and GclChange."""
296 for af in affected_files:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000297 lines = af.ChangedContents()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000298 for line in lines:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000299 yield (af, line[0], line[1])
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000300
301
Edward Lemur6eb1d322020-02-27 22:20:15 +0000302def prompt_should_continue(prompt_string):
303 sys.stdout.write(prompt_string)
Dirk Pranke7288f882021-06-03 18:01:30 +0000304 sys.stdout.flush()
Edward Lemur6eb1d322020-02-27 22:20:15 +0000305 response = sys.stdin.readline().strip().lower()
306 return response in ('y', 'yes')
dpranke@chromium.org5ac21012011-03-16 02:58:25 +0000307
308
Josip Sokcevice293d3d2022-02-16 22:52:15 +0000309def _ShouldRunPresubmit(script_text, use_python3):
310 """Try to figure out whether these presubmit checks should be run under
311 python2 or python3. We need to do this without actually trying to
312 compile the text, since the text might compile in one but not the
313 other.
314
315 Args:
316 script_text: The text of the presubmit script.
317 use_python3: if true, will use python3 instead of python2 by default
318 if USE_PYTHON3 is not specified.
319
320 Return:
321 A boolean if presubmit should be executed
322 """
323 m = re.search('^USE_PYTHON3 = (True|False)$', script_text, flags=re.MULTILINE)
324 if m:
325 use_python3 = m.group(1) == 'True'
326
327 return ((sys.version_info.major == 2) and not use_python3) or \
328 ((sys.version_info.major == 3) and use_python3)
329
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000330# Top level object so multiprocessing can pickle
331# Public access through OutputApi object.
332class _PresubmitResult(object):
333 """Base class for result objects."""
334 fatal = False
335 should_prompt = False
336
337 def __init__(self, message, items=None, long_text=''):
338 """
339 message: A short one-line message to indicate errors.
340 items: A list of short strings to indicate where errors occurred.
341 long_text: multi-line text output, e.g. from another tool
342 """
Bruce Dawsondb8622b2022-04-03 15:38:12 +0000343 self._message = _PresubmitResult._ensure_str(message)
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000344 self._items = items or []
Tom McKee61c72652021-07-20 11:56:32 +0000345 self._long_text = _PresubmitResult._ensure_str(long_text.rstrip())
346
347 @staticmethod
348 def _ensure_str(val):
349 """
350 val: A "stringish" value. Can be any of str, unicode or bytes.
351 returns: A str after applying encoding/decoding as needed.
352 Assumes/uses UTF-8 for relevant inputs/outputs.
353
354 We'd prefer to use six.ensure_str but our copy of six is old :(
355 """
356 if isinstance(val, str):
357 return val
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000358
Tom McKee61c72652021-07-20 11:56:32 +0000359 if six.PY2 and isinstance(val, unicode):
360 return val.encode()
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +0000361
362 if six.PY3 and isinstance(val, bytes):
Tom McKee61c72652021-07-20 11:56:32 +0000363 return val.decode()
364 raise ValueError("Unknown string type %s" % type(val))
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000365
Edward Lemur6eb1d322020-02-27 22:20:15 +0000366 def handle(self):
367 sys.stdout.write(self._message)
368 sys.stdout.write('\n')
Takuto Ikutabaa7be02022-08-23 00:19:34 +0000369 for item in self._items:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000370 sys.stdout.write(' ')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000371 # Write separately in case it's unicode.
Edward Lemur6eb1d322020-02-27 22:20:15 +0000372 sys.stdout.write(str(item))
Edward Lemur6eb1d322020-02-27 22:20:15 +0000373 sys.stdout.write('\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000374 if self._long_text:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000375 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000376 # Write separately in case it's unicode.
Tom McKee61c72652021-07-20 11:56:32 +0000377 sys.stdout.write(self._long_text)
Edward Lemur6eb1d322020-02-27 22:20:15 +0000378 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000379
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000380 def json_format(self):
381 return {
382 'message': self._message,
Debrian Figueroa6095d402019-06-28 18:47:18 +0000383 'items': [str(item) for item in self._items],
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000384 'long_text': self._long_text,
385 'fatal': self.fatal
386 }
387
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000388
389# Top level object so multiprocessing can pickle
390# Public access through OutputApi object.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000391class _PresubmitError(_PresubmitResult):
392 """A hard presubmit error."""
393 fatal = True
394
395
396# Top level object so multiprocessing can pickle
397# Public access through OutputApi object.
398class _PresubmitPromptWarning(_PresubmitResult):
399 """An warning that prompts the user if they want to continue."""
400 should_prompt = True
401
402
403# Top level object so multiprocessing can pickle
404# Public access through OutputApi object.
405class _PresubmitNotifyResult(_PresubmitResult):
406 """Just print something to the screen -- but it's not even a warning."""
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000407
408
409# Top level object so multiprocessing can pickle
410# Public access through OutputApi object.
411class _MailTextResult(_PresubmitResult):
412 """A warning that should be included in the review request email."""
413 def __init__(self, *args, **kwargs):
414 super(_MailTextResult, self).__init__()
415 raise NotImplementedError()
416
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000417class GerritAccessor(object):
418 """Limited Gerrit functionality for canned presubmit checks to work.
419
420 To avoid excessive Gerrit calls, caches the results.
421 """
422
Edward Lesmeseb1bd622021-03-01 19:54:07 +0000423 def __init__(self, url=None, project=None, branch=None):
424 self.host = urlparse.urlparse(url).netloc if url else None
425 self.project = project
426 self.branch = branch
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000427 self.cache = {}
Edward Lesmes8170c292021-03-19 20:04:43 +0000428 self.code_owners_enabled = None
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000429
430 def _FetchChangeDetail(self, issue):
431 # Separate function to be easily mocked in tests.
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100432 try:
433 return gerrit_util.GetChangeDetail(
434 self.host, str(issue),
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700435 ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS'])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100436 except gerrit_util.GerritError as e:
437 if e.http_status == 404:
438 raise Exception('Either Gerrit issue %s doesn\'t exist, or '
439 'no credentials to fetch issue details' % issue)
440 raise
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000441
442 def GetChangeInfo(self, issue):
443 """Returns labels and all revisions (patchsets) for this issue.
444
445 The result is a dictionary according to Gerrit REST Api.
446 https://gerrit-review.googlesource.com/Documentation/rest-api.html
447
448 However, API isn't very clear what's inside, so see tests for example.
449 """
450 assert issue
451 cache_key = int(issue)
452 if cache_key not in self.cache:
453 self.cache[cache_key] = self._FetchChangeDetail(issue)
454 return self.cache[cache_key]
455
456 def GetChangeDescription(self, issue, patchset=None):
457 """If patchset is none, fetches current patchset."""
458 info = self.GetChangeInfo(issue)
459 # info is a reference to cache. We'll modify it here adding description to
460 # it to the right patchset, if it is not yet there.
461
462 # Find revision info for the patchset we want.
463 if patchset is not None:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +0000464 for rev, rev_info in info['revisions'].items():
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000465 if str(rev_info['_number']) == str(patchset):
466 break
467 else:
468 raise Exception('patchset %s doesn\'t exist in issue %s' % (
469 patchset, issue))
470 else:
471 rev = info['current_revision']
472 rev_info = info['revisions'][rev]
473
Andrii Shyshkalov9c3a4642017-01-24 17:41:22 +0100474 return rev_info['commit']['message']
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000475
Mun Yong Jang603d01e2017-12-19 16:38:30 -0800476 def GetDestRef(self, issue):
477 ref = self.GetChangeInfo(issue)['branch']
478 if not ref.startswith('refs/'):
479 # NOTE: it is possible to create 'refs/x' branch,
480 # aka 'refs/heads/refs/x'. However, this is ill-advised.
481 ref = 'refs/heads/%s' % ref
482 return ref
483
Edward Lesmes02d4b822020-11-11 00:37:35 +0000484 def _GetApproversForLabel(self, issue, label):
485 change_info = self.GetChangeInfo(issue)
486 label_info = change_info.get('labels', {}).get(label, {})
487 values = label_info.get('values', {}).keys()
488 if not values:
489 return []
490 max_value = max(int(v) for v in values)
491 return [v for v in label_info.get('all', [])
492 if v.get('value', 0) == max_value]
493
Edward Lesmesc4566172021-03-19 16:55:13 +0000494 def IsBotCommitApproved(self, issue):
495 return bool(self._GetApproversForLabel(issue, 'Bot-Commit'))
496
Edward Lesmescf49cb82020-11-11 01:08:36 +0000497 def IsOwnersOverrideApproved(self, issue):
498 return bool(self._GetApproversForLabel(issue, 'Owners-Override'))
499
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000500 def GetChangeOwner(self, issue):
501 return self.GetChangeInfo(issue)['owner']['email']
502
503 def GetChangeReviewers(self, issue, approving_only=True):
Aaron Gable8b478f02017-07-31 15:33:19 -0700504 changeinfo = self.GetChangeInfo(issue)
505 if approving_only:
Edward Lesmes02d4b822020-11-11 00:37:35 +0000506 reviewers = self._GetApproversForLabel(issue, 'Code-Review')
Aaron Gable8b478f02017-07-31 15:33:19 -0700507 else:
508 reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', [])
509 return [r.get('email') for r in reviewers]
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000510
Edward Lemure4d329c2020-02-03 20:41:18 +0000511 def UpdateDescription(self, description, issue):
512 gerrit_util.SetCommitMessage(self.host, issue, description, notify='NONE')
513
Edward Lesmes8170c292021-03-19 20:04:43 +0000514 def IsCodeOwnersEnabledOnRepo(self):
515 if self.code_owners_enabled is None:
516 self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo(
Edward Lesmes392c4072021-03-19 21:58:45 +0000517 self.host, self.project)
Edward Lesmes8170c292021-03-19 20:04:43 +0000518 return self.code_owners_enabled
519
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000520
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000521class OutputApi(object):
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000522 """An instance of OutputApi gets passed to presubmit scripts so that they
523 can output various types of results.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000524 """
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000525 PresubmitResult = _PresubmitResult
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000526 PresubmitError = _PresubmitError
527 PresubmitPromptWarning = _PresubmitPromptWarning
528 PresubmitNotifyResult = _PresubmitNotifyResult
529 MailTextResult = _MailTextResult
530
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000531 def __init__(self, is_committing):
532 self.is_committing = is_committing
Daniel Cheng7227d212017-11-17 08:12:37 -0800533 self.more_cc = []
534
535 def AppendCC(self, cc):
536 """Appends a user to cc for this change."""
537 self.more_cc.append(cc)
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000538
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000539 def PresubmitPromptOrNotify(self, *args, **kwargs):
540 """Warn the user when uploading, but only notify if committing."""
541 if self.is_committing:
542 return self.PresubmitNotifyResult(*args, **kwargs)
543 return self.PresubmitPromptWarning(*args, **kwargs)
544
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000545
546class InputApi(object):
547 """An instance of this object is passed to presubmit scripts so they can
548 know stuff about the change they're looking at.
549 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000550 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800551 # pylint: disable=no-self-use
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000552
maruel@chromium.org3410d912009-06-09 20:56:16 +0000553 # File extensions that are considered source files from a style guide
554 # perspective. Don't modify this list from a presubmit script!
maruel@chromium.orgc33455a2011-06-24 19:14:18 +0000555 #
556 # Files without an extension aren't included in the list. If you want to
local_bot30f774e2020-06-25 18:23:34 +0000557 # filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list.
local_bot64021412020-07-08 21:05:39 +0000558 # Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below.
559 DEFAULT_FILES_TO_CHECK = (
maruel@chromium.org3410d912009-06-09 20:56:16 +0000560 # C++ and friends
Edward Lemura5799e32020-01-17 19:26:51 +0000561 r'.+\.c$', r'.+\.cc$', r'.+\.cpp$', r'.+\.h$', r'.+\.m$', r'.+\.mm$',
562 r'.+\.inl$', r'.+\.asm$', r'.+\.hxx$', r'.+\.hpp$', r'.+\.s$', r'.+\.S$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000563 # Scripts
dpapad443d9132022-05-05 00:17:30 +0000564 r'.+\.js$', r'.+\.ts$', r'.+\.py$', r'.+\.sh$', r'.+\.rb$', r'.+\.pl$',
565 r'.+\.pm$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000566 # Other
Edward Lemura5799e32020-01-17 19:26:51 +0000567 r'.+\.java$', r'.+\.mk$', r'.+\.am$', r'.+\.css$', r'.+\.mojom$',
568 r'.+\.fidl$'
maruel@chromium.org3410d912009-06-09 20:56:16 +0000569 )
570
571 # Path regexp that should be excluded from being considered containing source
572 # files. Don't modify this list from a presubmit script!
local_bot64021412020-07-08 21:05:39 +0000573 DEFAULT_FILES_TO_SKIP = (
Edward Lemura5799e32020-01-17 19:26:51 +0000574 r'testing_support[\\\/]google_appengine[\\\/].*',
575 r'.*\bexperimental[\\\/].*',
Kent Tamura179dd1e2018-04-26 15:07:41 +0900576 # Exclude third_party/.* but NOT third_party/{WebKit,blink}
577 # (crbug.com/539768 and crbug.com/836555).
Edward Lemura5799e32020-01-17 19:26:51 +0000578 r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000579 # Output directories (just in case)
Edward Lemura5799e32020-01-17 19:26:51 +0000580 r'.*\bDebug[\\\/].*',
581 r'.*\bRelease[\\\/].*',
582 r'.*\bxcodebuild[\\\/].*',
583 r'.*\bout[\\\/].*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000584 # All caps files like README and LICENCE.
Edward Lemura5799e32020-01-17 19:26:51 +0000585 r'.*\b[A-Z0-9_]{2,}$',
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000586 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
Edward Lemura5799e32020-01-17 19:26:51 +0000587 r'(|.*[\\\/])\.git[\\\/].*',
588 r'(|.*[\\\/])\.svn[\\\/].*',
maruel@chromium.org7ccb4bb2011-11-07 20:26:20 +0000589 # There is no point in processing a patch file.
Edward Lemura5799e32020-01-17 19:26:51 +0000590 r'.+\.diff$',
591 r'.+\.patch$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000592 )
593
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000594 def __init__(self, change, presubmit_path, is_committing,
Bruce Dawson09c0c072022-05-26 20:28:58 +0000595 verbose, gerrit_obj, dry_run=None, thread_pool=None, parallel=False,
596 no_diffs=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000597 """Builds an InputApi object.
598
599 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000600 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000601 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000602 is_committing: True if the change is about to be committed.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000603 gerrit_obj: provides basic Gerrit codereview functionality.
604 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -0400605 parallel: if true, all tests reported via input_api.RunTests for all
606 PRESUBMIT files will be run in parallel.
Bruce Dawson09c0c072022-05-26 20:28:58 +0000607 no_diffs: if true, implies that --files or --all was specified so some
608 checks can be skipped, and some errors will be messages.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000609 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000610 # Version number of the presubmit_support script.
611 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000612 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000613 self.is_committing = is_committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000614 self.gerrit = gerrit_obj
tandrii@chromium.org57bafac2016-04-28 05:09:03 +0000615 self.dry_run = dry_run
Bruce Dawson09c0c072022-05-26 20:28:58 +0000616 self.no_diffs = no_diffs
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000617
Edward Lesmes8e282792018-04-03 18:50:29 -0400618 self.parallel = parallel
619 self.thread_pool = thread_pool or ThreadPool()
620
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000621 # We expose various modules and functions as attributes of the input_api
622 # so that presubmit scripts don't have to import them.
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +0900623 self.ast = ast
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000624 self.basename = os.path.basename
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000625 self.cpplint = cpplint
dcheng091b7db2016-06-16 01:27:51 -0700626 self.fnmatch = fnmatch
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +0000627 self.gclient_paths = gclient_paths
Yoshisato Yanagisawa57dd17b2019-03-22 09:10:29 +0000628 # TODO(yyanagisawa): stop exposing this when python3 become default.
629 # Since python3's tempfile has TemporaryDirectory, we do not need this.
630 self.temporary_directory = gclient_utils.temporary_directory
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000631 self.glob = glob.glob
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000632 self.json = json
maruel@chromium.org6fba34d2011-06-02 13:45:12 +0000633 self.logging = logging.getLogger('PRESUBMIT')
maruel@chromium.org2b5ce562011-03-31 16:15:44 +0000634 self.os_listdir = os.listdir
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000635 self.os_path = os.path
pgervais@chromium.orgbd0cace2014-10-02 23:23:46 +0000636 self.os_stat = os.stat
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +0000637 self.os_walk = os.walk
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000638 self.re = re
639 self.subprocess = subprocess
Edward Lemurb9830242019-10-30 22:19:20 +0000640 self.sys = sys
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000641 self.tempfile = tempfile
dpranke@chromium.org0d1bdea2011-03-24 22:54:38 +0000642 self.time = time
maruel@chromium.org1487d532009-06-06 00:22:57 +0000643 self.unittest = unittest
Edward Lemurb9830242019-10-30 22:19:20 +0000644 if sys.version_info.major == 2:
645 self.urllib2 = urllib2
Edward Lemur16af3562019-10-17 22:11:33 +0000646 self.urllib_request = urllib_request
647 self.urllib_error = urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000648
Robert Iannucci50258932018-03-19 10:30:59 -0700649 self.is_windows = sys.platform == 'win32'
650
Edward Lemurb9646622019-10-25 20:57:35 +0000651 # Set python_executable to 'vpython' in order to allow scripts in other
652 # repos (e.g. src.git) to automatically pick up that repo's .vpython file,
653 # instead of inheriting the one in depot_tools.
654 self.python_executable = 'vpython'
Erik Staab69135d12021-05-14 22:31:57 +0000655 # Offer a python 3 executable for use during the migration off of python 2.
656 self.python3_executable = 'vpython3'
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000657 self.environ = os.environ
658
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000659 # InputApi.platform is the platform you're currently running on.
660 self.platform = sys.platform
661
iannucci@chromium.org0af3bb32015-06-12 20:44:35 +0000662 self.cpu_count = multiprocessing.cpu_count()
Bruce Dawson8254f062022-06-17 17:33:08 +0000663 if self.is_windows:
664 # TODO(crbug.com/1190269) - we can't use more than 56 child processes on
665 # Windows or Python3 may hang.
666 self.cpu_count = min(self.cpu_count, 56)
iannucci@chromium.org0af3bb32015-06-12 20:44:35 +0000667
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000668 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000669 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000670
671 # We carry the canned checks so presubmit scripts can easily use them.
672 self.canned_checks = presubmit_canned_checks
673
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100674 # Temporary files we must manually remove at the end of a run.
675 self._named_temporary_files = []
Jochen Eisinger72606f82017-04-04 10:44:18 +0200676
Edward Lesmesdc11c6b2021-03-19 19:22:05 +0000677 self.owners_client = None
Bruce Dawsoneb8426e2022-08-05 23:58:15 +0000678 if self.gerrit and not 'PRESUBMIT_SKIP_NETWORK' in self.environ:
Bruce Dawson9b9f4512022-04-27 00:56:52 +0000679 try:
680 self.owners_client = owners_client.GetCodeOwnersClient(
Gavin Maka9677a52022-08-08 22:30:27 +0000681 root=change.RepositoryRoot(),
682 upstream=change.UpstreamBranch(),
Bruce Dawson9b9f4512022-04-27 00:56:52 +0000683 host=self.gerrit.host,
684 project=self.gerrit.project,
685 branch=self.gerrit.branch)
686 except Exception as e:
687 print('Failed to set owners_client - %s' % str(e))
Gavin Maka9677a52022-08-08 22:30:27 +0000688 self.owners_db = owners_db.Database(
689 change.RepositoryRoot(), fopen=open, os_path=self.os_path)
Jochen Eisinger76f5fc62017-04-07 16:27:46 +0200690 self.owners_finder = owners_finder.OwnersFinder
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000691 self.verbose = verbose
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000692 self.Command = CommandData
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000693
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000694 # Replace <hash_map> and <hash_set> as headers that need to be included
Edward Lemura5799e32020-01-17 19:26:51 +0000695 # with 'base/containers/hash_tables.h' instead.
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000696 # Access to a protected member _XX of a client class
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800697 # pylint: disable=protected-access
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000698 self.cpplint._re_pattern_templates = [
danakj@chromium.org18278522013-06-11 22:42:32 +0000699 (a, b, 'base/containers/hash_tables.h')
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000700 if header in ('<hash_map>', '<hash_set>') else (a, b, header)
701 for (a, b, header) in cpplint._re_pattern_templates
702 ]
703
Edward Lemurecc27072020-01-06 16:42:34 +0000704 def SetTimeout(self, timeout):
705 self.thread_pool.timeout = timeout
706
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000707 def PresubmitLocalPath(self):
708 """Returns the local path of the presubmit script currently being run.
709
710 This is useful if you don't want to hard-code absolute paths in the
711 presubmit script. For example, It can be used to find another file
712 relative to the PRESUBMIT.py script, so the whole tree can be branched and
713 the presubmit script still works, without editing its content.
714 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000715 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000716
agable0b65e732016-11-22 09:25:46 -0800717 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000718 """Same as input_api.change.AffectedFiles() except only lists files
719 (and optionally directories) in the same directory as the current presubmit
Bruce Dawson7a0b07a2020-04-23 17:14:40 +0000720 script, or subdirectories thereof. Note that files are listed using the OS
721 path separator, so backslashes are used as separators on Windows.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000722 """
Bruce Dawson31bfd512022-05-10 23:19:39 +0000723 dir_with_slash = normpath(self.PresubmitLocalPath())
724 # normpath strips trailing path separators, so the trailing separator has to
725 # be added after the normpath call.
726 if len(dir_with_slash) > 0:
727 dir_with_slash += os.path.sep
sail@chromium.org5538e022011-05-12 17:53:16 +0000728
Edward Lemurb9830242019-10-30 22:19:20 +0000729 return list(filter(
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000730 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
Edward Lemurb9830242019-10-30 22:19:20 +0000731 self.change.AffectedFiles(include_deletes, file_filter)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000732
agable0b65e732016-11-22 09:25:46 -0800733 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000734 """Returns local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800735 paths = [af.LocalPath() for af in self.AffectedFiles()]
Edward Lemura5799e32020-01-17 19:26:51 +0000736 logging.debug('LocalPaths: %s', paths)
pgervais@chromium.org2f64f782014-04-25 00:12:33 +0000737 return paths
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000738
agable0b65e732016-11-22 09:25:46 -0800739 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000740 """Returns absolute local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800741 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000742
John Budorick16162372018-04-18 10:39:53 -0700743 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
agable0b65e732016-11-22 09:25:46 -0800744 """Same as input_api.change.AffectedTestableFiles() except only lists files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000745 in the same directory as the current presubmit script, or subdirectories
746 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000747 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000748 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +0000749 warn('AffectedTestableFiles(include_deletes=%s)'
750 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000751 category=DeprecationWarning,
752 stacklevel=2)
Josip Sokcevic4de5dea2022-03-23 21:15:14 +0000753 # pylint: disable=consider-using-generator
754 return [
755 x for x in self.AffectedFiles(include_deletes=False, **kwargs)
756 if x.IsTestableFile()
757 ]
agable0b65e732016-11-22 09:25:46 -0800758
759 def AffectedTextFiles(self, include_deletes=None):
760 """An alias to AffectedTestableFiles for backwards compatibility."""
761 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000762
Josip Sokcevic8c955952021-02-01 21:32:57 +0000763 def FilterSourceFile(self,
764 affected_file,
765 files_to_check=None,
766 files_to_skip=None,
767 allow_list=None,
768 block_list=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000769 """Filters out files that aren't considered 'source file'.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000770
local_bot64021412020-07-08 21:05:39 +0000771 If files_to_check or files_to_skip is None, InputApi.DEFAULT_FILES_TO_CHECK
772 and InputApi.DEFAULT_FILES_TO_SKIP is used respectively.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000773
774 The lists will be compiled as regular expression and
775 AffectedFile.LocalPath() needs to pass both list.
776
777 Note: Copy-paste this function to suit your needs or use a lambda function.
778 """
local_bot64021412020-07-08 21:05:39 +0000779 if files_to_check is None:
780 files_to_check = self.DEFAULT_FILES_TO_CHECK
781 if files_to_skip is None:
782 files_to_skip = self.DEFAULT_FILES_TO_SKIP
local_bot30f774e2020-06-25 18:23:34 +0000783
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000784 def Find(affected_file, items):
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000785 local_path = affected_file.LocalPath()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000786 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000787 if self.re.match(item, local_path):
maruel@chromium.org3410d912009-06-09 20:56:16 +0000788 return True
Bruce Dawsona3a014e2022-04-27 23:28:17 +0000789 # Handle the cases where the files regex only handles /, but the local
790 # path uses \.
791 if self.is_windows and self.re.match(item, local_path.replace(
792 '\\', '/')):
793 return True
maruel@chromium.org3410d912009-06-09 20:56:16 +0000794 return False
local_bot64021412020-07-08 21:05:39 +0000795 return (Find(affected_file, files_to_check) and
796 not Find(affected_file, files_to_skip))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000797
798 def AffectedSourceFiles(self, source_file):
agable0b65e732016-11-22 09:25:46 -0800799 """Filter the list of AffectedTestableFiles by the function source_file.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000800
801 If source_file is None, InputApi.FilterSourceFile() is used.
802 """
803 if not source_file:
804 source_file = self.FilterSourceFile
Edward Lemurb9830242019-10-30 22:19:20 +0000805 return list(filter(source_file, self.AffectedTestableFiles()))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000806
807 def RightHandSideLines(self, source_file_filter=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000808 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000809
810 Only lists lines from new or modified text files in the change that are
811 contained by the directory of the currently executing presubmit script.
812
813 This is useful for doing line-by-line regex checks, like checking for
814 trailing whitespace.
815
816 Yields:
817 a 3 tuple:
818 the AffectedFile instance of the current file;
819 integer line number (1-based); and
820 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000821
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000822 Note: The carriage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000823 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000824 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000825 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000826
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000827 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000828 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000829
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000830 Deny reading anything outside the repository.
831 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000832 if isinstance(file_item, AffectedFile):
833 file_item = file_item.AbsoluteLocalPath()
834 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000835 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000836 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000837
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100838 def CreateTemporaryFile(self, **kwargs):
839 """Returns a named temporary file that must be removed with a call to
840 RemoveTemporaryFiles().
841
842 All keyword arguments are forwarded to tempfile.NamedTemporaryFile(),
843 except for |delete|, which is always set to False.
844
845 Presubmit checks that need to create a temporary file and pass it for
846 reading should use this function instead of NamedTemporaryFile(), as
847 Windows fails to open a file that is already open for writing.
848
849 with input_api.CreateTemporaryFile() as f:
850 f.write('xyz')
851 f.close()
852 input_api.subprocess.check_output(['script-that', '--reads-from',
853 f.name])
854
855
856 Note that callers of CreateTemporaryFile() should not worry about removing
857 any temporary file; this is done transparently by the presubmit handling
858 code.
859 """
860 if 'delete' in kwargs:
861 # Prevent users from passing |delete|; we take care of file deletion
862 # ourselves and this prevents unintuitive error messages when we pass
863 # delete=False and 'delete' is also in kwargs.
864 raise TypeError('CreateTemporaryFile() does not take a "delete" '
865 'argument, file deletion is handled automatically by '
866 'the same presubmit_support code that creates InputApi '
867 'objects.')
868 temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs)
869 self._named_temporary_files.append(temp_file.name)
870 return temp_file
871
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000872 @property
873 def tbr(self):
874 """Returns if a change is TBR'ed."""
Jeremy Romandce22502017-06-20 15:37:29 -0400875 return 'TBR' in self.change.tags or self.change.TBRsFromDescription()
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000876
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000877 def RunTests(self, tests_mix, parallel=True):
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000878 tests = []
879 msgs = []
880 for t in tests_mix:
Edward Lesmes8e282792018-04-03 18:50:29 -0400881 if isinstance(t, OutputApi.PresubmitResult) and t:
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000882 msgs.append(t)
883 else:
884 assert issubclass(t.message, _PresubmitResult)
885 tests.append(t)
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000886 if self.verbose:
887 t.info = _PresubmitNotifyResult
Edward Lemur1037c742018-05-01 18:56:04 -0400888 if not t.kwargs.get('cwd'):
889 t.kwargs['cwd'] = self.PresubmitLocalPath()
Edward Lesmes8e282792018-04-03 18:50:29 -0400890 self.thread_pool.AddTests(tests, parallel)
Edward Lemur21000eb2019-05-24 23:25:58 +0000891 # When self.parallel is True (i.e. --parallel is passed as an option)
892 # RunTests doesn't actually run tests. It adds them to a ThreadPool that
893 # will run all tests once all PRESUBMIT files are processed.
894 # Otherwise, it will run them and return the results.
895 if not self.parallel:
Edward Lesmes8e282792018-04-03 18:50:29 -0400896 msgs.extend(self.thread_pool.RunAsync())
897 return msgs
scottmg86099d72016-09-01 09:16:51 -0700898
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000899
nick@chromium.orgff526192013-06-10 19:30:26 +0000900class _DiffCache(object):
901 """Caches diffs retrieved from a particular SCM."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000902 def __init__(self, upstream=None):
903 """Stores the upstream revision against which all diffs will be computed."""
904 self._upstream = upstream
nick@chromium.orgff526192013-06-10 19:30:26 +0000905
906 def GetDiff(self, path, local_root):
907 """Get the diff for a particular path."""
908 raise NotImplementedError()
909
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700910 def GetOldContents(self, path, local_root):
911 """Get the old version for a particular path."""
912 raise NotImplementedError()
913
nick@chromium.orgff526192013-06-10 19:30:26 +0000914
nick@chromium.orgff526192013-06-10 19:30:26 +0000915class _GitDiffCache(_DiffCache):
916 """DiffCache implementation for git; gets all file diffs at once."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000917 def __init__(self, upstream):
918 super(_GitDiffCache, self).__init__(upstream=upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000919 self._diffs_by_file = None
920
921 def GetDiff(self, path, local_root):
Bruce Dawson5f63d3c2022-04-19 17:02:36 +0000922 # Compare against None to distinguish between None and an initialized but
923 # empty dictionary.
924 if self._diffs_by_file == None:
nick@chromium.orgff526192013-06-10 19:30:26 +0000925 # Compute a single diff for all files and parse the output; should
926 # with git this is much faster than computing one diff for each file.
927 diffs = {}
928
929 # Don't specify any filenames below, because there are command line length
930 # limits on some platforms and GenerateDiff would fail.
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000931 unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True,
932 branch=self._upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000933
934 # This regex matches the path twice, separated by a space. Note that
935 # filename itself may contain spaces.
936 file_marker = re.compile('^diff --git (?P<filename>.*) (?P=filename)$')
937 current_diff = []
938 keep_line_endings = True
939 for x in unified_diff.splitlines(keep_line_endings):
940 match = file_marker.match(x)
941 if match:
942 # Marks the start of a new per-file section.
943 diffs[match.group('filename')] = current_diff = [x]
944 elif x.startswith('diff --git'):
945 raise PresubmitFailure('Unexpected diff line: %s' % x)
946 else:
947 current_diff.append(x)
948
949 self._diffs_by_file = dict(
950 (normpath(path), ''.join(diff)) for path, diff in diffs.items())
951
952 if path not in self._diffs_by_file:
Josip Sokcevicc1ab7342022-02-24 19:02:44 +0000953 # SCM didn't have any diff on this file. It could be that the file was not
954 # modified at all (e.g. user used --all flag in git cl presubmit).
955 # Intead of failing, return empty string.
956 # See: https://crbug.com/808346.
Josip Sokcevicc1ab7342022-02-24 19:02:44 +0000957 return ''
nick@chromium.orgff526192013-06-10 19:30:26 +0000958
959 return self._diffs_by_file[path]
960
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700961 def GetOldContents(self, path, local_root):
962 return scm.GIT.GetOldContents(local_root, path, branch=self._upstream)
963
nick@chromium.orgff526192013-06-10 19:30:26 +0000964
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000965class AffectedFile(object):
966 """Representation of a file in a change."""
nick@chromium.orgff526192013-06-10 19:30:26 +0000967
968 DIFF_CACHE = _DiffCache
969
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000970 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800971 # pylint: disable=no-self-use
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000972 def __init__(self, path, action, repository_root, diff_cache):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000973 self._path = path
974 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000975 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000976 self._is_directory = None
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000977 self._cached_changed_contents = None
978 self._cached_new_contents = None
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000979 self._diff_cache = diff_cache
tobiasjs2836bcf2016-08-16 04:08:16 -0700980 logging.debug('%s(%s)', self.__class__.__name__, self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000981
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000982 def LocalPath(self):
983 """Returns the path of this file on the local disk relative to client root.
Andrew Grieve92b8b992017-11-02 09:42:24 -0400984
985 This should be used for error messages but not for accessing files,
986 because presubmit checks are run with CWD=PresubmitLocalPath() (which is
987 often != client root).
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000988 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000989 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000990
991 def AbsoluteLocalPath(self):
992 """Returns the absolute path of this file on the local disk.
993 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000994 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000995
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000996 def Action(self):
997 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000998 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000999
agable0b65e732016-11-22 09:25:46 -08001000 def IsTestableFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001001 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001002
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001003 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +00001004 raise NotImplementedError() # Implement when needed
1005
agable0b65e732016-11-22 09:25:46 -08001006 def IsTextFile(self):
1007 """An alias to IsTestableFile for backwards compatibility."""
1008 return self.IsTestableFile()
1009
Daniel Cheng7a1f04d2017-03-21 19:12:31 -07001010 def OldContents(self):
1011 """Returns an iterator over the lines in the old version of file.
1012
Daniel Cheng2da34fe2017-03-21 20:42:12 -07001013 The old version is the file before any modifications in the user's
Edward Lemura5799e32020-01-17 19:26:51 +00001014 workspace, i.e. the 'left hand side'.
Daniel Cheng7a1f04d2017-03-21 19:12:31 -07001015
1016 Contents will be empty if the file is a directory or does not exist.
1017 Note: The carriage returns (LF or CR) are stripped off.
1018 """
1019 return self._diff_cache.GetOldContents(self.LocalPath(),
1020 self._local_root).splitlines()
1021
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001022 def NewContents(self):
1023 """Returns an iterator over the lines in the new version of file.
1024
Edward Lemura5799e32020-01-17 19:26:51 +00001025 The new version is the file in the user's workspace, i.e. the 'right hand
1026 side'.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001027
1028 Contents will be empty if the file is a directory or does not exist.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001029 Note: The carriage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001030 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001031 if self._cached_new_contents is None:
1032 self._cached_new_contents = []
agable0b65e732016-11-22 09:25:46 -08001033 try:
1034 self._cached_new_contents = gclient_utils.FileRead(
1035 self.AbsoluteLocalPath(), 'rU').splitlines()
1036 except IOError:
1037 pass # File not found? That's fine; maybe it was deleted.
Greg Thompson30cde452021-06-01 16:38:47 +00001038 except UnicodeDecodeError as e:
Dirk Pranke6fc394f2021-05-26 23:25:14 +00001039 # log the filename since we're probably trying to read a binary
1040 # file, and shouldn't be.
1041 print('Error reading %s: %s' % (self.AbsoluteLocalPath(), e))
1042 raise
1043
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001044 return self._cached_new_contents[:]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001045
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001046 def ChangedContents(self, keeplinebreaks=False):
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001047 """Returns a list of tuples (line number, line text) of all new lines.
1048
1049 This relies on the scm diff output describing each changed code section
1050 with a line of the form
1051
1052 ^@@ <old line num>,<old size> <new line num>,<new size> @@$
1053 """
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001054 # Don't return cached results when line breaks are requested.
1055 if not keeplinebreaks and self._cached_changed_contents is not None:
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001056 return self._cached_changed_contents[:]
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001057 result = []
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001058 line_num = 0
1059
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001060 # The keeplinebreaks parameter to splitlines must be True or else the
1061 # CheckForWindowsLineEndings presubmit will be a NOP.
1062 for line in self.GenerateScmDiff().splitlines(keeplinebreaks):
Edward Lemurac5c55f2020-02-29 00:17:16 +00001063 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
1064 if m:
1065 line_num = int(m.groups(1)[0])
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001066 continue
1067 if line.startswith('+') and not line.startswith('++'):
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001068 result.append((line_num, line[1:]))
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001069 if not line.startswith('-'):
1070 line_num += 1
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001071 # Don't cache results with line breaks.
1072 if keeplinebreaks:
1073 return result;
1074 self._cached_changed_contents = result
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001075 return self._cached_changed_contents[:]
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001076
maruel@chromium.org5de13972009-06-10 18:16:06 +00001077 def __str__(self):
1078 return self.LocalPath()
1079
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001080 def GenerateScmDiff(self):
nick@chromium.orgff526192013-06-10 19:30:26 +00001081 return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001082
maruel@chromium.org58407af2011-04-12 23:15:57 +00001083
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001084class GitAffectedFile(AffectedFile):
1085 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +00001086 # Method 'NNN' is abstract in class 'NNN' but is not overridden
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001087 # pylint: disable=abstract-method
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001088
nick@chromium.orgff526192013-06-10 19:30:26 +00001089 DIFF_CACHE = _GitDiffCache
1090
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001091 def __init__(self, *args, **kwargs):
1092 AffectedFile.__init__(self, *args, **kwargs)
1093 self._server_path = None
agable0b65e732016-11-22 09:25:46 -08001094 self._is_testable_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001095
agable0b65e732016-11-22 09:25:46 -08001096 def IsTestableFile(self):
1097 if self._is_testable_file is None:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001098 if self.Action() == 'D':
agable0b65e732016-11-22 09:25:46 -08001099 # A deleted file is not testable.
1100 self._is_testable_file = False
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001101 else:
agable0b65e732016-11-22 09:25:46 -08001102 self._is_testable_file = os.path.isfile(self.AbsoluteLocalPath())
1103 return self._is_testable_file
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001104
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001105
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001106class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001107 """Describe a change.
1108
1109 Used directly by the presubmit scripts to query the current change being
1110 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001111
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001112 Instance members:
nick@chromium.orgff526192013-06-10 19:30:26 +00001113 tags: Dictionary of KEY=VALUE pairs found in the change description.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001114 self.KEY: equivalent to tags['KEY']
1115 """
1116
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001117 _AFFECTED_FILES = AffectedFile
1118
Edward Lemura5799e32020-01-17 19:26:51 +00001119 # Matches key/value (or 'tag') lines in changelist descriptions.
maruel@chromium.org428342a2011-11-10 15:46:33 +00001120 TAG_LINE_RE = re.compile(
maruel@chromium.orgc6f60e82013-04-19 17:01:57 +00001121 '^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001122 scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001123
maruel@chromium.org58407af2011-04-12 23:15:57 +00001124 def __init__(
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001125 self, name, description, local_root, files, issue, patchset, author,
1126 upstream=None):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001127 if files is None:
1128 files = []
1129 self._name = name
chase@chromium.org8e416c82009-10-06 04:30:44 +00001130 # Convert root into an absolute path.
1131 self._local_root = os.path.abspath(local_root)
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001132 self._upstream = upstream
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001133 self.issue = issue
1134 self.patchset = patchset
maruel@chromium.org58407af2011-04-12 23:15:57 +00001135 self.author_email = author
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001136
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001137 self._full_description = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001138 self.tags = {}
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001139 self._description_without_tags = ''
1140 self.SetDescriptionText(description)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001141
maruel@chromium.orge085d812011-10-10 19:49:15 +00001142 assert all(
1143 (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files
1144
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001145 diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream)
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001146 self._affected_files = [
nick@chromium.orgff526192013-06-10 19:30:26 +00001147 self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache)
1148 for action, path in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +00001149 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001150
Edward Lesmes9ce03f82021-01-12 20:13:31 +00001151 def UpstreamBranch(self):
1152 """Returns the upstream branch for the change."""
1153 return self._upstream
1154
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001155 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001156 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001157 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001158
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001159 def DescriptionText(self):
1160 """Returns the user-entered changelist description, minus tags.
1161
Edward Lemura5799e32020-01-17 19:26:51 +00001162 Any line in the user-provided description starting with e.g. 'FOO='
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001163 (whitespace permitted before and around) is considered a tag line. Such
1164 lines are stripped out of the description this function returns.
1165 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001166 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001167
1168 def FullDescriptionText(self):
1169 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001170 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001171
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001172 def SetDescriptionText(self, description):
1173 """Sets the full description text (including tags) to |description|.
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001174
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001175 Also updates the list of tags."""
1176 self._full_description = description
1177
1178 # From the description text, build up a dictionary of key/value pairs
Edward Lemura5799e32020-01-17 19:26:51 +00001179 # plus the description minus all key/value or 'tag' lines.
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001180 description_without_tags = []
1181 self.tags = {}
1182 for line in self._full_description.splitlines():
1183 m = self.TAG_LINE_RE.match(line)
1184 if m:
1185 self.tags[m.group('key')] = m.group('value')
1186 else:
1187 description_without_tags.append(line)
1188
1189 # Change back to text and remove whitespace at end.
1190 self._description_without_tags = (
1191 '\n'.join(description_without_tags).rstrip())
1192
Edward Lemur69bb8be2020-02-03 20:37:38 +00001193 def AddDescriptionFooter(self, key, value):
1194 """Adds the given footer to the change description.
1195
1196 Args:
1197 key: A string with the key for the git footer. It must conform to
1198 the git footers format (i.e. 'List-Of-Tokens') and will be case
1199 normalized so that each token is title-cased.
1200 value: A string with the value for the git footer.
1201 """
1202 description = git_footers.add_footer(
1203 self.FullDescriptionText(), git_footers.normalize_name(key), value)
1204 self.SetDescriptionText(description)
1205
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001206 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001207 """Returns the repository (checkout) root directory for this change,
1208 as an absolute path.
1209 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001210 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001211
1212 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001213 """Return tags directly as attributes on the object."""
Edward Lemura5799e32020-01-17 19:26:51 +00001214 if not re.match(r'^[A-Z_]*$', attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001215 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +00001216 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001217
Edward Lemur69bb8be2020-02-03 20:37:38 +00001218 def GitFootersFromDescription(self):
1219 """Return the git footers present in the description.
1220
1221 Returns:
1222 footers: A dict of {footer: [values]} containing a multimap of the footers
1223 in the change description.
1224 """
1225 return git_footers.parse_footers(self.FullDescriptionText())
1226
Aaron Gablefc03e672017-05-15 14:09:42 -07001227 def BugsFromDescription(self):
1228 """Returns all bugs referenced in the commit description."""
Sean McAllister1e509c52021-10-25 17:54:25 +00001229 bug_tags = ['BUG', 'FIXED']
1230
1231 tags = []
1232 for tag in bug_tags:
1233 values = self.tags.get(tag)
1234 if values:
1235 tags += [value.strip() for value in values.split(',')]
1236
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001237 footers = []
Edward Lemur69bb8be2020-02-03 20:37:38 +00001238 parsed = self.GitFootersFromDescription()
Dan Beam62954042019-10-03 21:20:33 +00001239 unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', [])
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001240 for unsplit_footer in unsplit_footers:
1241 footers += [b.strip() for b in unsplit_footer.split(',')]
Aaron Gable12ef5012017-05-15 14:29:00 -07001242 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001243
1244 def ReviewersFromDescription(self):
1245 """Returns all reviewers listed in the commit description."""
Edward Lemura5799e32020-01-17 19:26:51 +00001246 # We don't support a 'R:' git-footer for reviewers; that is in metadata.
Aaron Gable12ef5012017-05-15 14:29:00 -07001247 tags = [r.strip() for r in self.tags.get('R', '').split(',') if r.strip()]
1248 return sorted(set(tags))
Aaron Gablefc03e672017-05-15 14:09:42 -07001249
1250 def TBRsFromDescription(self):
1251 """Returns all TBR reviewers listed in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001252 tags = [r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()]
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001253 # TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are
1254 # programmatically determined by self-CR+1s.
Edward Lemur69bb8be2020-02-03 20:37:38 +00001255 footers = self.GitFootersFromDescription().get('Tbr', [])
Aaron Gable12ef5012017-05-15 14:29:00 -07001256 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001257
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001258 # TODO(crbug.com/753425): Delete these once we're sure they're unused.
Aaron Gablefc03e672017-05-15 14:09:42 -07001259 @property
1260 def BUG(self):
1261 return ','.join(self.BugsFromDescription())
1262 @property
1263 def R(self):
1264 return ','.join(self.ReviewersFromDescription())
1265 @property
1266 def TBR(self):
1267 return ','.join(self.TBRsFromDescription())
1268
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001269 def AllFiles(self, root=None):
1270 """List all files under source control in the repo."""
1271 raise NotImplementedError()
1272
agable0b65e732016-11-22 09:25:46 -08001273 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001274 """Returns a list of AffectedFile instances for all files in the change.
1275
1276 Args:
1277 include_deletes: If false, deleted files will be filtered out.
sail@chromium.org5538e022011-05-12 17:53:16 +00001278 file_filter: An additional filter to apply.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001279
1280 Returns:
1281 [AffectedFile(path, action), AffectedFile(path, action)]
1282 """
Edward Lemurb9830242019-10-30 22:19:20 +00001283 affected = list(filter(file_filter, self._affected_files))
sail@chromium.org5538e022011-05-12 17:53:16 +00001284
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001285 if include_deletes:
1286 return affected
Edward Lemurb9830242019-10-30 22:19:20 +00001287 return list(filter(lambda x: x.Action() != 'D', affected))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001288
John Budorick16162372018-04-18 10:39:53 -07001289 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001290 """Return a list of the existing text files in a change."""
1291 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +00001292 warn('AffectedTeestableFiles(include_deletes=%s)'
1293 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001294 category=DeprecationWarning,
1295 stacklevel=2)
Edward Lemurb9830242019-10-30 22:19:20 +00001296 return list(filter(
1297 lambda x: x.IsTestableFile(),
1298 self.AffectedFiles(include_deletes=False, **kwargs)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001299
agable0b65e732016-11-22 09:25:46 -08001300 def AffectedTextFiles(self, include_deletes=None):
1301 """An alias to AffectedTestableFiles for backwards compatibility."""
1302 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001303
agable0b65e732016-11-22 09:25:46 -08001304 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001305 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001306 return [af.LocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001307
agable0b65e732016-11-22 09:25:46 -08001308 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001309 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001310 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001311
1312 def RightHandSideLines(self):
Edward Lemura5799e32020-01-17 19:26:51 +00001313 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001314
1315 Lists lines from new or modified text files in the change.
1316
1317 This is useful for doing line-by-line regex checks, like checking for
1318 trailing whitespace.
1319
1320 Yields:
1321 a 3 tuple:
1322 the AffectedFile instance of the current file;
1323 integer line number (1-based); and
1324 the contents of the line as a string.
1325 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001326 return _RightHandSideLinesImpl(
1327 x for x in self.AffectedFiles(include_deletes=False)
agable0b65e732016-11-22 09:25:46 -08001328 if x.IsTestableFile())
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001329
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001330 def OriginalOwnersFiles(self):
1331 """A map from path names of affected OWNERS files to their old content."""
1332 def owners_file_filter(f):
1333 return 'OWNERS' in os.path.split(f.LocalPath())[1]
1334 files = self.AffectedFiles(file_filter=owners_file_filter)
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001335 return {f.LocalPath(): f.OldContents() for f in files}
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001336
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001337
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001338class GitChange(Change):
1339 _AFFECTED_FILES = GitAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001340 scm = 'git'
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001341
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001342 def AllFiles(self, root=None):
1343 """List all files under source control in the repo."""
1344 root = root or self.RepositoryRoot()
1345 return subprocess.check_output(
Aaron Gable7817f022017-12-12 09:43:17 -08001346 ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'],
Josip Sokcevic0b123462021-06-08 20:41:32 +00001347 cwd=root).decode('utf-8', 'ignore').splitlines()
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001348
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001349
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001350def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001351 """Finds all presubmit files that apply to a given set of source files.
1352
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001353 If inherit-review-settings-ok is present right under root, looks for
1354 PRESUBMIT.py in directories enclosing root.
1355
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001356 Args:
1357 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001358 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001359
1360 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001361 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001362 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001363 files = [normpath(os.path.join(root, f)) for f in files]
1364
1365 # List all the individual directories containing files.
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001366 directories = {os.path.dirname(f) for f in files}
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001367
1368 # Ignore root if inherit-review-settings-ok is present.
1369 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
1370 root = None
1371
1372 # Collect all unique directories that may contain PRESUBMIT.py.
1373 candidates = set()
1374 for directory in directories:
1375 while True:
1376 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001377 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001378 candidates.add(directory)
1379 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001380 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001381 parent_dir = os.path.dirname(directory)
1382 if parent_dir == directory:
1383 # We hit the system root directory.
1384 break
1385 directory = parent_dir
1386
1387 # Look for PRESUBMIT.py in all candidate directories.
1388 results = []
1389 for directory in sorted(list(candidates)):
tobiasjsff061c02016-08-17 03:23:57 -07001390 try:
1391 for f in os.listdir(directory):
1392 p = os.path.join(directory, f)
1393 if os.path.isfile(p) and re.match(
1394 r'PRESUBMIT.*\.py$', f) and not f.startswith('PRESUBMIT_test'):
1395 results.append(p)
1396 except OSError:
1397 pass
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001398
tobiasjs2836bcf2016-08-16 04:08:16 -07001399 logging.debug('Presubmit files: %s', ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001400 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001401
1402
rmistry@google.com5626a922015-02-26 14:03:30 +00001403class GetPostUploadExecuter(object):
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001404 def __init__(self, use_python3):
1405 """
1406 Args:
1407 use_python3: if true, will use python3 instead of python2 by default
1408 if USE_PYTHON3 is not specified.
1409 """
1410 self.use_python3 = use_python3
1411
1412 def ExecPresubmitScript(self, script_text, presubmit_path, gerrit_obj,
1413 change):
rmistry@google.com5626a922015-02-26 14:03:30 +00001414 """Executes PostUploadHook() from a single presubmit script.
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001415 Caller is responsible for validating whether the hook should be executed
1416 and should only call this function if it should be.
rmistry@google.com5626a922015-02-26 14:03:30 +00001417
1418 Args:
1419 script_text: The text of the presubmit script.
1420 presubmit_path: Project script to run.
Edward Lemur016a0872020-02-04 22:13:28 +00001421 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001422 change: The Change object.
1423
1424 Return:
1425 A list of results objects.
1426 """
1427 context = {}
1428 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001429 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1430 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001431 except Exception as e:
rmistry@google.com5626a922015-02-26 14:03:30 +00001432 raise PresubmitFailure('"%s" had an exception.\n%s'
1433 % (presubmit_path, e))
1434
1435 function_name = 'PostUploadHook'
1436 if function_name not in context:
1437 return {}
1438 post_upload_hook = context[function_name]
1439 if not len(inspect.getargspec(post_upload_hook)[0]) == 3:
1440 raise PresubmitFailure(
1441 'Expected function "PostUploadHook" to take three arguments.')
Edward Lemur016a0872020-02-04 22:13:28 +00001442 return post_upload_hook(gerrit_obj, change, OutputApi(False))
rmistry@google.com5626a922015-02-26 14:03:30 +00001443
1444
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001445def _MergeMasters(masters1, masters2):
1446 """Merges two master maps. Merges also the tests of each builder."""
1447 result = {}
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001448 for (master, builders) in itertools.chain(masters1.items(),
1449 masters2.items()):
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001450 new_builders = result.setdefault(master, {})
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001451 for (builder, tests) in builders.items():
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001452 new_builders.setdefault(builder, set([])).update(tests)
1453 return result
1454
1455
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001456def DoPostUploadExecuter(change, gerrit_obj, verbose, use_python3=False):
rmistry@google.com5626a922015-02-26 14:03:30 +00001457 """Execute the post upload hook.
1458
1459 Args:
1460 change: The Change object.
Edward Lemur016a0872020-02-04 22:13:28 +00001461 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001462 verbose: Prints debug info.
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001463 use_python3: if true, default to using Python3 for presubmit checks
1464 rather than Python2.
rmistry@google.com5626a922015-02-26 14:03:30 +00001465 """
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001466 python_version = 'Python %s' % sys.version_info.major
1467 sys.stdout.write('Running %s post upload checks ...\n' % python_version)
rmistry@google.com5626a922015-02-26 14:03:30 +00001468 presubmit_files = ListRelevantPresubmitFiles(
Edward Lemur6eb1d322020-02-27 22:20:15 +00001469 change.LocalPaths(), change.RepositoryRoot())
rmistry@google.com5626a922015-02-26 14:03:30 +00001470 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001471 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
rmistry@google.com5626a922015-02-26 14:03:30 +00001472 results = []
Josip Sokcevice293d3d2022-02-16 22:52:15 +00001473 executer = GetPostUploadExecuter(use_python3)
rmistry@google.com5626a922015-02-26 14:03:30 +00001474 # The root presubmit file should be executed after the ones in subdirectories.
1475 # i.e. the specific post upload hooks should run before the general ones.
1476 # Thus, reverse the order provided by ListRelevantPresubmitFiles.
1477 presubmit_files.reverse()
1478
1479 for filename in presubmit_files:
1480 filename = os.path.abspath(filename)
rmistry@google.com5626a922015-02-26 14:03:30 +00001481 # Accept CRLF presubmit script.
Bruce Dawson6e70e862022-07-01 19:37:01 +00001482 presubmit_script = gclient_utils.FileRead(filename).replace('\r\n', '\n')
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001483 if _ShouldRunPresubmit(presubmit_script, use_python3):
Bruce Dawson8d78bd12022-08-12 19:17:03 +00001484 if sys.version_info[0] == 2:
1485 sys.stdout.write('Running %s under Python 2.\n' % filename)
1486 elif verbose:
1487 sys.stdout.write('Running %s\n' % filename)
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001488 results.extend(executer.ExecPresubmitScript(
1489 presubmit_script, filename, gerrit_obj, change))
rmistry@google.com5626a922015-02-26 14:03:30 +00001490
Edward Lemur6eb1d322020-02-27 22:20:15 +00001491 if not results:
1492 return 0
1493
1494 sys.stdout.write('\n')
1495 sys.stdout.write('** Post Upload Hook Messages **\n')
1496
1497 exit_code = 0
1498 for result in results:
1499 if result.fatal:
1500 exit_code = 1
1501 result.handle()
1502 sys.stdout.write('\n')
1503
1504 return exit_code
rmistry@google.com5626a922015-02-26 14:03:30 +00001505
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001506class PresubmitExecuter(object):
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001507 def __init__(self, change, committing, verbose, gerrit_obj, dry_run=None,
Bruce Dawson09c0c072022-05-26 20:28:58 +00001508 thread_pool=None, parallel=False, use_python3=False,
1509 no_diffs=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001510 """
1511 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001512 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001513 committing: True if 'git cl land' is running, False if 'git cl upload' is.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001514 gerrit_obj: provides basic Gerrit codereview functionality.
1515 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001516 parallel: if true, all tests reported via input_api.RunTests for all
1517 PRESUBMIT files will be run in parallel.
Dirk Pranke6f0df682021-06-25 00:42:33 +00001518 use_python3: if true, will use python3 instead of python2 by default
1519 if USE_PYTHON3 is not specified.
Bruce Dawson09c0c072022-05-26 20:28:58 +00001520 no_diffs: if true, implies that --files or --all was specified so some
1521 checks can be skipped, and some errors will be messages.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001522 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001523 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001524 self.committing = committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001525 self.gerrit = gerrit_obj
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001526 self.verbose = verbose
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001527 self.dry_run = dry_run
Daniel Cheng7227d212017-11-17 08:12:37 -08001528 self.more_cc = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001529 self.thread_pool = thread_pool
1530 self.parallel = parallel
Dirk Pranke6f0df682021-06-25 00:42:33 +00001531 self.use_python3 = use_python3
Bruce Dawson09c0c072022-05-26 20:28:58 +00001532 self.no_diffs = no_diffs
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001533
1534 def ExecPresubmitScript(self, script_text, presubmit_path):
1535 """Executes a single presubmit script.
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001536 Caller is responsible for validating whether the hook should be executed
1537 and should only call this function if it should be.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001538
1539 Args:
1540 script_text: The text of the presubmit script.
1541 presubmit_path: The path to the presubmit file (this will be reported via
1542 input_api.PresubmitLocalPath()).
1543
1544 Return:
1545 A list of result objects, empty if no problems.
1546 """
chase@chromium.org8e416c82009-10-06 04:30:44 +00001547 # Change to the presubmit file's directory to support local imports.
1548 main_path = os.getcwd()
Saagar Sanghavi98b332f2020-07-31 17:19:15 +00001549 presubmit_dir = os.path.dirname(presubmit_path)
1550 os.chdir(presubmit_dir)
chase@chromium.org8e416c82009-10-06 04:30:44 +00001551
1552 # Load the presubmit script into context.
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001553 input_api = InputApi(self.change, presubmit_path, self.committing,
Aaron Gable668c1d82018-04-03 10:19:16 -07001554 self.verbose, gerrit_obj=self.gerrit,
Edward Lesmes8e282792018-04-03 18:50:29 -04001555 dry_run=self.dry_run, thread_pool=self.thread_pool,
Bruce Dawson09c0c072022-05-26 20:28:58 +00001556 parallel=self.parallel, no_diffs=self.no_diffs)
Daniel Cheng7227d212017-11-17 08:12:37 -08001557 output_api = OutputApi(self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001558 context = {}
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001559
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001560 try:
Bruce Dawson0ba2fd42022-07-21 13:47:21 +00001561 exec(compile(script_text, presubmit_path, 'exec', dont_inherit=True),
Raul Tambre09e64b42019-05-14 01:57:22 +00001562 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001563 except Exception as e:
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001564 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001565
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001566 context['__args'] = (input_api, output_api)
Ben Pastene8351dc12020-08-06 05:01:35 +00001567
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001568 # Get path of presubmit directory relative to repository root.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001569 # Always use forward slashes, so that path is same in *nix and Windows
1570 root = input_api.change.RepositoryRoot()
1571 rel_path = os.path.relpath(presubmit_dir, root)
1572 rel_path = rel_path.replace(os.path.sep, '/')
Ben Pastene8351dc12020-08-06 05:01:35 +00001573
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001574 # Get the URL of git remote origin and use it to identify host and project
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001575 host = project = ''
1576 if self.gerrit:
1577 host = self.gerrit.host or ''
1578 project = self.gerrit.project or ''
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001579
1580 # Prefix for test names
1581 prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path)
1582
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001583 # Perform all the desired presubmit checks.
1584 results = []
Ben Pastene8351dc12020-08-06 05:01:35 +00001585
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001586 try:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001587 version = [
1588 int(x) for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.')
1589 ]
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001590
Scott Leecc2fe9b2020-11-19 19:38:06 +00001591 with rdb_wrapper.client(prefix) as sink:
1592 if version >= [2, 0, 0]:
Andrew Grievebdfb8f22022-02-02 21:56:47 +00001593 # Copy the keys to prevent "dictionary changed size during iteration"
1594 # exception if checks add globals to context. E.g. sometimes the
1595 # Python runtime will add __warningregistry__.
1596 for function_name in list(context.keys()):
Scott Leecc2fe9b2020-11-19 19:38:06 +00001597 if not function_name.startswith('Check'):
1598 continue
1599 if function_name.endswith('Commit') and not self.committing:
1600 continue
1601 if function_name.endswith('Upload') and self.committing:
1602 continue
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001603 logging.debug('Running %s in %s', function_name, presubmit_path)
1604 results.extend(
Bruce Dawson8fa42e22022-03-29 17:05:38 +00001605 self._run_check_function(function_name, context, sink,
1606 presubmit_path))
Scott Leecc2fe9b2020-11-19 19:38:06 +00001607 logging.debug('Running %s done.', function_name)
1608 self.more_cc.extend(output_api.more_cc)
1609
1610 else: # Old format
1611 if self.committing:
1612 function_name = 'CheckChangeOnCommit'
1613 else:
1614 function_name = 'CheckChangeOnUpload'
Andrew Grievebdfb8f22022-02-02 21:56:47 +00001615 if function_name in list(context.keys()):
Scott Leecc2fe9b2020-11-19 19:38:06 +00001616 logging.debug('Running %s in %s', function_name, presubmit_path)
1617 results.extend(
Bruce Dawson8fa42e22022-03-29 17:05:38 +00001618 self._run_check_function(function_name, context, sink,
1619 presubmit_path))
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001620 logging.debug('Running %s done.', function_name)
1621 self.more_cc.extend(output_api.more_cc)
1622
1623 finally:
1624 for f in input_api._named_temporary_files:
1625 os.remove(f)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001626
chase@chromium.org8e416c82009-10-06 04:30:44 +00001627 # Return the process to the original working directory.
1628 os.chdir(main_path)
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001629 return results
1630
Bruce Dawson8fa42e22022-03-29 17:05:38 +00001631 def _run_check_function(self, function_name, context, sink, presubmit_path):
Scott Leecc2fe9b2020-11-19 19:38:06 +00001632 """Evaluates and returns the result of a given presubmit function.
1633
1634 If sink is given, the result of the presubmit function will be reported
1635 to the ResultSink.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001636
1637 Args:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001638 function_name: the name of the presubmit function to evaluate
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001639 context: a context dictionary in which the function will be evaluated
Scott Leecc2fe9b2020-11-19 19:38:06 +00001640 sink: an instance of ResultSink. None, by default.
1641 Returns:
1642 the result of the presubmit function call.
1643 """
1644 start_time = time_time()
1645 try:
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001646 result = eval(function_name + '(*__args)', context)
1647 self._check_result_type(result)
Allen Webbfe7d7092021-05-18 02:05:49 +00001648 except Exception:
Bruce Dawson10a82862022-05-27 19:25:56 +00001649 _, e_value, _ = sys.exc_info()
1650 result = [
1651 OutputApi.PresubmitError(
1652 'Evaluation of %s failed: %s, %s' %
1653 (function_name, e_value, traceback.format_exc()))
1654 ]
Scott Leecc2fe9b2020-11-19 19:38:06 +00001655
Bruce Dawson2bdc49f2021-05-21 19:13:03 +00001656 elapsed_time = time_time() - start_time
1657 if elapsed_time > 10.0:
Bruce Dawson6757d462022-07-13 04:04:40 +00001658 sys.stdout.write('%6.1fs to run %s from %s.\n' %
1659 (elapsed_time, function_name, presubmit_path))
Scott Leecc2fe9b2020-11-19 19:38:06 +00001660 if sink:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001661 status = rdb_wrapper.STATUS_PASS
1662 if any(r.fatal for r in result):
1663 status = rdb_wrapper.STATUS_FAIL
1664 sink.report(function_name, status, elapsed_time)
1665
1666 return result
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001667
Saagar Sanghavi9949ab72020-07-20 20:56:40 +00001668 def _check_result_type(self, result):
1669 """Helper function which ensures result is a list, and all elements are
1670 instances of OutputApi.PresubmitResult"""
1671 if not isinstance(result, (tuple, list)):
1672 raise PresubmitFailure('Presubmit functions must return a tuple or list')
1673 if not all(isinstance(res, OutputApi.PresubmitResult) for res in result):
1674 raise PresubmitFailure(
1675 'All presubmit results must be of types derived from '
1676 'output_api.PresubmitResult')
1677
1678
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001679def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001680 committing,
1681 verbose,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001682 default_presubmit,
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001683 may_prompt,
Aaron Gable668c1d82018-04-03 10:19:16 -07001684 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001685 dry_run=None,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00001686 parallel=False,
Dirk Pranke6f0df682021-06-25 00:42:33 +00001687 json_output=None,
Bruce Dawson09c0c072022-05-26 20:28:58 +00001688 use_python3=False,
1689 no_diffs=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001690 """Runs all presubmit checks that apply to the files in the change.
1691
1692 This finds all PRESUBMIT.py files in directories enclosing the files in the
1693 change (up to the repository root) and calls the relevant entrypoint function
1694 depending on whether the change is being committed or uploaded.
1695
1696 Prints errors, warnings and notifications. Prompts the user for warnings
1697 when needed.
1698
1699 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001700 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001701 committing: True if 'git cl land' is running, False if 'git cl upload' is.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001702 verbose: Prints debug info.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001703 default_presubmit: A default presubmit script to execute in any case.
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001704 may_prompt: Enable (y/n) questions on warning or error. If False,
1705 any questions are answered with yes by default.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001706 gerrit_obj: provides basic Gerrit codereview functionality.
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001707 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001708 parallel: if true, all tests specified by input_api.RunTests in all
1709 PRESUBMIT files will be run in parallel.
Dirk Pranke6f0df682021-06-25 00:42:33 +00001710 use_python3: if true, default to using Python3 for presubmit checks
1711 rather than Python2.
Bruce Dawson09c0c072022-05-26 20:28:58 +00001712 no_diffs: if true, implies that --files or --all was specified so some
1713 checks can be skipped, and some errors will be messages.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001714 Return:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001715 1 if presubmit checks failed or 0 otherwise.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001716 """
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001717 old_environ = os.environ
1718 try:
1719 # Make sure python subprocesses won't generate .pyc files.
1720 os.environ = os.environ.copy()
Edward Lemurb9830242019-10-30 22:19:20 +00001721 os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001722
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001723 python_version = 'Python %s' % sys.version_info.major
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001724 if committing:
Dirk Pranke6fc394f2021-05-26 23:25:14 +00001725 sys.stdout.write('Running %s presubmit commit checks ...\n' %
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001726 python_version)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001727 else:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001728 sys.stdout.write('Running %s presubmit upload checks ...\n' %
1729 python_version)
Edward Lemurecc27072020-01-06 16:42:34 +00001730 start_time = time_time()
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001731 presubmit_files = ListRelevantPresubmitFiles(
agable0b65e732016-11-22 09:25:46 -08001732 change.AbsoluteLocalPaths(), change.RepositoryRoot())
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001733 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001734 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001735 results = []
Bruce Dawsonf2b04212022-06-10 16:42:54 +00001736 depot_tools = os.path.dirname(os.path.abspath(__file__))
1737 python2_usage_log_file = os.path.join(depot_tools, 'python2_usage.txt')
1738 if os.path.exists(python2_usage_log_file):
1739 os.remove(python2_usage_log_file)
Edward Lesmes8e282792018-04-03 18:50:29 -04001740 thread_pool = ThreadPool()
Edward Lemur7e3c67f2018-07-20 20:52:49 +00001741 executer = PresubmitExecuter(change, committing, verbose, gerrit_obj,
Bruce Dawson09c0c072022-05-26 20:28:58 +00001742 dry_run, thread_pool, parallel, use_python3,
1743 no_diffs)
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001744 skipped_count = 0;
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001745 if default_presubmit:
1746 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001747 sys.stdout.write('Running default presubmit script.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001748 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001749 if _ShouldRunPresubmit(default_presubmit, use_python3):
1750 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
1751 else:
1752 skipped_count += 1
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001753 for filename in presubmit_files:
1754 filename = os.path.abspath(filename)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001755 # Accept CRLF presubmit script.
Bruce Dawson6e70e862022-07-01 19:37:01 +00001756 presubmit_script = gclient_utils.FileRead(filename).replace('\r\n', '\n')
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001757 if _ShouldRunPresubmit(presubmit_script, use_python3):
Bruce Dawson8d78bd12022-08-12 19:17:03 +00001758 if sys.version_info[0] == 2:
1759 sys.stdout.write('Running %s under Python 2.\n' % filename)
1760 elif verbose:
1761 sys.stdout.write('Running %s\n' % filename)
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001762 results += executer.ExecPresubmitScript(presubmit_script, filename)
1763 else:
1764 skipped_count += 1
Bruce Dawson76fe1a72022-05-25 20:52:21 +00001765
Edward Lesmes8e282792018-04-03 18:50:29 -04001766 results += thread_pool.RunAsync()
1767
Bruce Dawsonf2b04212022-06-10 16:42:54 +00001768 if os.path.exists(python2_usage_log_file):
1769 with open(python2_usage_log_file) as f:
1770 python2_usage = [x.strip() for x in f.readlines()]
1771 results.append(
1772 OutputApi(committing).PresubmitPromptWarning(
1773 'Python 2 scripts were run during %s presubmits. Please see '
1774 'https://bugs.chromium.org/p/chromium/issues/detail?id=1313804'
1775 '#c61 for tips on resolving this.'
1776 % python_version,
1777 items=python2_usage))
1778
Edward Lemur6eb1d322020-02-27 22:20:15 +00001779 messages = {}
1780 should_prompt = False
1781 presubmits_failed = False
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001782 for result in results:
1783 if result.fatal:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001784 presubmits_failed = True
1785 messages.setdefault('ERRORS', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001786 elif result.should_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001787 should_prompt = True
1788 messages.setdefault('Warnings', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001789 else:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001790 messages.setdefault('Messages', []).append(result)
pam@chromium.orged9a0832009-09-09 22:48:55 +00001791
Bruce Dawson76fe1a72022-05-25 20:52:21 +00001792 # Print the different message types in a consistent order. ERRORS go last
1793 # so that they will be most visible in the local-presubmit output.
1794 for name in ['Messages', 'Warnings', 'ERRORS']:
1795 if name in messages:
1796 items = messages[name]
Gavin Makd22bf602022-07-11 21:10:41 +00001797 sys.stdout.write('** Presubmit %s: %d **\n' % (name, len(items)))
Bruce Dawson76fe1a72022-05-25 20:52:21 +00001798 for item in items:
1799 item.handle()
1800 sys.stdout.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001801
Edward Lemurecc27072020-01-06 16:42:34 +00001802 total_time = time_time() - start_time
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001803 if total_time > 1.0:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001804 sys.stdout.write(
Louis Romero4ca9e1c2022-03-10 10:29:11 +00001805 'Presubmit checks took %.1fs to calculate.\n' % total_time)
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001806
Edward Lemur6eb1d322020-02-27 22:20:15 +00001807 if not should_prompt and not presubmits_failed:
Louis Romero4ca9e1c2022-03-10 10:29:11 +00001808 sys.stdout.write('%s presubmit checks passed.\n\n' % python_version)
Josip Sokcevic7592e0a2022-01-12 00:57:54 +00001809 elif should_prompt and not presubmits_failed:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001810 sys.stdout.write('There were %s presubmit warnings. ' % python_version)
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001811 if may_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001812 presubmits_failed = not prompt_should_continue(
1813 'Are you sure you wish to continue? (y/N): ')
Garrett Beatyacb807e2020-08-28 18:17:24 +00001814 else:
1815 sys.stdout.write('\n')
Bruce Dawson76fe1a72022-05-25 20:52:21 +00001816 else:
1817 sys.stdout.write('There were %s presubmit errors.\n' % python_version)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001818
Edward Lemur1dc66e12020-02-21 21:36:34 +00001819 if json_output:
1820 # Write the presubmit results to json output
1821 presubmit_results = {
1822 'errors': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001823 error.json_format()
1824 for error in messages.get('ERRORS', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001825 ],
1826 'notifications': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001827 notification.json_format()
1828 for notification in messages.get('Messages', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001829 ],
1830 'warnings': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001831 warning.json_format()
1832 for warning in messages.get('Warnings', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001833 ],
Edward Lemur6eb1d322020-02-27 22:20:15 +00001834 'more_cc': executer.more_cc,
Josip Sokcevic632bbc02022-05-19 05:32:50 +00001835 'skipped_presubmits': skipped_count,
Edward Lemur1dc66e12020-02-21 21:36:34 +00001836 }
1837
1838 gclient_utils.FileWrite(
1839 json_output, json.dumps(presubmit_results, sort_keys=True))
1840
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001841 global _ASKED_FOR_FEEDBACK
1842 # Ask for feedback one time out of 5.
Aravind Vasudevanc5f0cbb2022-01-24 23:56:57 +00001843 if (results and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
Edward Lemur6eb1d322020-02-27 22:20:15 +00001844 sys.stdout.write(
maruel@chromium.org1ce8e662014-01-14 15:23:00 +00001845 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n'
1846 'to figure out which PRESUBMIT.py was run, then run git blame\n'
1847 'on the file to figure out who to ask for help.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001848 _ASKED_FOR_FEEDBACK = True
Edward Lemur6eb1d322020-02-27 22:20:15 +00001849
1850 return 1 if presubmits_failed else 0
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001851 finally:
1852 os.environ = old_environ
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001853
1854
Edward Lemur50984a62020-02-06 18:10:18 +00001855def _scan_sub_dirs(mask, recursive):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001856 if not recursive:
pgervais@chromium.orge57b09d2014-05-07 00:58:13 +00001857 return [x for x in glob.glob(mask) if x not in ('.svn', '.git')]
Lei Zhang9611c4c2017-04-04 01:41:56 -07001858
1859 results = []
1860 for root, dirs, files in os.walk('.'):
1861 if '.svn' in dirs:
1862 dirs.remove('.svn')
1863 if '.git' in dirs:
1864 dirs.remove('.git')
1865 for name in files:
1866 if fnmatch.fnmatch(name, mask):
1867 results.append(os.path.join(root, name))
1868 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001869
1870
Edward Lemur50984a62020-02-06 18:10:18 +00001871def _parse_files(args, recursive):
tobiasjs2836bcf2016-08-16 04:08:16 -07001872 logging.debug('Searching for %s', args)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001873 files = []
1874 for arg in args:
Edward Lemur50984a62020-02-06 18:10:18 +00001875 files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001876 return files
1877
1878
Edward Lemur50984a62020-02-06 18:10:18 +00001879def _parse_change(parser, options):
1880 """Process change options.
1881
1882 Args:
1883 parser: The parser used to parse the arguments from command line.
1884 options: The arguments parsed from command line.
1885 Returns:
1886 A GitChange if the change root is a git repository, or a Change otherwise.
1887 """
1888 if options.files and options.all_files:
1889 parser.error('<files> cannot be specified when --all-files is set.')
1890
Edward Lesmes44ea3ff2020-02-05 00:14:30 +00001891 change_scm = scm.determine_scm(options.root)
Edward Lemur50984a62020-02-06 18:10:18 +00001892 if change_scm != 'git' and not options.files:
1893 parser.error('<files> is not optional for unversioned directories.')
1894
1895 if options.files:
Josip Sokcevic017544d2022-03-31 23:47:53 +00001896 if options.source_controlled_only:
1897 # Get the filtered set of files from SCM.
1898 change_files = []
1899 for name in scm.GIT.GetAllFiles(options.root):
1900 for mask in options.files:
1901 if fnmatch.fnmatch(name, mask):
1902 change_files.append(('M', name))
1903 break
1904 else:
1905 # Get the filtered set of files from a directory scan.
1906 change_files = _parse_files(options.files, options.recursive)
Edward Lemur50984a62020-02-06 18:10:18 +00001907 elif options.all_files:
1908 change_files = [('M', f) for f in scm.GIT.GetAllFiles(options.root)]
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001909 else:
Edward Lemur50984a62020-02-06 18:10:18 +00001910 change_files = scm.GIT.CaptureStatus(
Edward Lemur7f6dec02020-02-06 20:23:58 +00001911 options.root, options.upstream or None)
Edward Lemur50984a62020-02-06 18:10:18 +00001912
1913 logging.info('Found %d file(s).', len(change_files))
1914
1915 change_class = GitChange if change_scm == 'git' else Change
1916 return change_class(
1917 options.name,
1918 options.description,
1919 options.root,
1920 change_files,
1921 options.issue,
1922 options.patchset,
1923 options.author,
1924 upstream=options.upstream)
1925
1926
1927def _parse_gerrit_options(parser, options):
1928 """Process gerrit options.
1929
1930 SIDE EFFECTS: Modifies options.author and options.description from Gerrit if
1931 options.gerrit_fetch is set.
1932
1933 Args:
1934 parser: The parser used to parse the arguments from command line.
1935 options: The arguments parsed from command line.
1936 Returns:
Stephen Martinisfb09de22021-02-25 03:41:13 +00001937 A GerritAccessor object if options.gerrit_url is set, or None otherwise.
Edward Lemur50984a62020-02-06 18:10:18 +00001938 """
1939 gerrit_obj = None
Stephen Martinisfb09de22021-02-25 03:41:13 +00001940 if options.gerrit_url:
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001941 gerrit_obj = GerritAccessor(
1942 url=options.gerrit_url,
1943 project=options.gerrit_project,
1944 branch=options.gerrit_branch)
Edward Lemur50984a62020-02-06 18:10:18 +00001945
1946 if not options.gerrit_fetch:
1947 return gerrit_obj
1948
1949 if not options.gerrit_url or not options.issue or not options.patchset:
1950 parser.error(
1951 '--gerrit_fetch requires --gerrit_url, --issue and --patchset.')
1952
1953 options.author = gerrit_obj.GetChangeOwner(options.issue)
1954 options.description = gerrit_obj.GetChangeDescription(
1955 options.issue, options.patchset)
1956
1957 logging.info('Got author: "%s"', options.author)
1958 logging.info('Got description: """\n%s\n"""', options.description)
1959
1960 return gerrit_obj
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001961
1962
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001963@contextlib.contextmanager
1964def canned_check_filter(method_names):
1965 filtered = {}
1966 try:
1967 for method_name in method_names:
1968 if not hasattr(presubmit_canned_checks, method_name):
Gavin Make6a62332020-12-04 21:57:10 +00001969 logging.warning('Skipping unknown "canned" check %s' % method_name)
Aaron Gableecee74c2018-04-02 15:13:08 -07001970 continue
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001971 filtered[method_name] = getattr(presubmit_canned_checks, method_name)
1972 setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
1973 yield
1974 finally:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001975 for name, method in filtered.items():
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001976 setattr(presubmit_canned_checks, name, method)
1977
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +00001978
sbc@chromium.org013731e2015-02-26 18:28:43 +00001979def main(argv=None):
Edward Lemura5799e32020-01-17 19:26:51 +00001980 parser = argparse.ArgumentParser(usage='%(prog)s [options] <files...>')
1981 hooks = parser.add_mutually_exclusive_group()
1982 hooks.add_argument('-c', '--commit', action='store_true',
1983 help='Use commit instead of upload checks.')
1984 hooks.add_argument('-u', '--upload', action='store_false', dest='commit',
1985 help='Use upload instead of commit checks.')
Edward Lemur75526302020-02-27 22:31:05 +00001986 hooks.add_argument('--post_upload', action='store_true',
1987 help='Run post-upload commit hooks.')
Edward Lemura5799e32020-01-17 19:26:51 +00001988 parser.add_argument('-r', '--recursive', action='store_true',
1989 help='Act recursively.')
1990 parser.add_argument('-v', '--verbose', action='count', default=0,
1991 help='Use 2 times for more debug info.')
1992 parser.add_argument('--name', default='no name')
1993 parser.add_argument('--author')
Edward Lemur227d5102020-02-25 23:45:35 +00001994 desc = parser.add_mutually_exclusive_group()
1995 desc.add_argument('--description', default='', help='The change description.')
1996 desc.add_argument('--description_file',
1997 help='File to read change description from.')
Edward Lemura5799e32020-01-17 19:26:51 +00001998 parser.add_argument('--issue', type=int, default=0)
1999 parser.add_argument('--patchset', type=int, default=0)
2000 parser.add_argument('--root', default=os.getcwd(),
2001 help='Search for PRESUBMIT.py up to this directory. '
2002 'If inherit-review-settings-ok is present in this '
2003 'directory, parent directories up to the root file '
2004 'system directories will also be searched.')
2005 parser.add_argument('--upstream',
2006 help='Git only: the base ref or upstream branch against '
2007 'which the diff should be computed.')
2008 parser.add_argument('--default_presubmit')
2009 parser.add_argument('--may_prompt', action='store_true', default=False)
2010 parser.add_argument('--skip_canned', action='append', default=[],
2011 help='A list of checks to skip which appear in '
2012 'presubmit_canned_checks. Can be provided multiple times '
2013 'to skip multiple canned checks.')
2014 parser.add_argument('--dry_run', action='store_true', help=argparse.SUPPRESS)
2015 parser.add_argument('--gerrit_url', help=argparse.SUPPRESS)
Edward Lesmeseb1bd622021-03-01 19:54:07 +00002016 parser.add_argument('--gerrit_project', help=argparse.SUPPRESS)
2017 parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS)
Edward Lemura5799e32020-01-17 19:26:51 +00002018 parser.add_argument('--gerrit_fetch', action='store_true',
2019 help=argparse.SUPPRESS)
2020 parser.add_argument('--parallel', action='store_true',
2021 help='Run all tests specified by input_api.RunTests in '
2022 'all PRESUBMIT files in parallel.')
2023 parser.add_argument('--json_output',
2024 help='Write presubmit errors to json output.')
Edward Lemur1dc66e12020-02-21 21:36:34 +00002025 parser.add_argument('--all_files', action='store_true',
Edward Lemur50984a62020-02-06 18:10:18 +00002026 help='Mark all files under source control as modified.')
Bruce Dawson09c0c072022-05-26 20:28:58 +00002027
Edward Lemura5799e32020-01-17 19:26:51 +00002028 parser.add_argument('files', nargs='*',
2029 help='List of files to be marked as modified when '
2030 'executing presubmit or post-upload hooks. fnmatch '
2031 'wildcards can also be used.')
Josip Sokcevic017544d2022-03-31 23:47:53 +00002032 parser.add_argument('--source_controlled_only', action='store_true',
2033 help='Constrain \'files\' to those in source control.')
Dirk Pranke6f0df682021-06-25 00:42:33 +00002034 parser.add_argument('--use-python3', action='store_true',
2035 help='Use python3 for presubmit checks by default')
Bruce Dawson09c0c072022-05-26 20:28:58 +00002036 parser.add_argument('--no_diffs', action='store_true',
2037 help='Assume that all "modified" files have no diffs.')
Edward Lemura5799e32020-01-17 19:26:51 +00002038 options = parser.parse_args(argv)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002039
Erik Staabcca5c492020-04-16 17:40:07 +00002040 log_level = logging.ERROR
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002041 if options.verbose >= 2:
Erik Staabcca5c492020-04-16 17:40:07 +00002042 log_level = logging.DEBUG
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002043 elif options.verbose:
Erik Staabcca5c492020-04-16 17:40:07 +00002044 log_level = logging.INFO
2045 log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d '
2046 '%(filename)s] %(message)s')
2047 logging.basicConfig(format=log_format, level=log_level)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002048
Edward Lemur227d5102020-02-25 23:45:35 +00002049 if options.description_file:
2050 options.description = gclient_utils.FileRead(options.description_file)
Edward Lemur50984a62020-02-06 18:10:18 +00002051 gerrit_obj = _parse_gerrit_options(parser, options)
2052 change = _parse_change(parser, options)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002053
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002054 try:
Edward Lemur75526302020-02-27 22:31:05 +00002055 if options.post_upload:
Josip Sokcevice293d3d2022-02-16 22:52:15 +00002056 return DoPostUploadExecuter(change, gerrit_obj, options.verbose,
2057 options.use_python3)
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002058 with canned_check_filter(options.skip_canned):
Edward Lemur6eb1d322020-02-27 22:20:15 +00002059 return DoPresubmitChecks(
Edward Lemur50984a62020-02-06 18:10:18 +00002060 change,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002061 options.commit,
2062 options.verbose,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002063 options.default_presubmit,
2064 options.may_prompt,
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00002065 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04002066 options.dry_run,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00002067 options.parallel,
Dirk Pranke6f0df682021-06-25 00:42:33 +00002068 options.json_output,
Bruce Dawson09c0c072022-05-26 20:28:58 +00002069 options.use_python3,
2070 options.no_diffs)
Raul Tambre7c938462019-05-24 16:35:35 +00002071 except PresubmitFailure as e:
Josip Sokcevic0399e172022-03-21 23:11:51 +00002072 import utils
Raul Tambre80ee78e2019-05-06 22:41:05 +00002073 print(e, file=sys.stderr)
2074 print('Maybe your depot_tools is out of date?', file=sys.stderr)
Josip Sokcevic0399e172022-03-21 23:11:51 +00002075 print('depot_tools version: %s' % utils.depot_tools_version(),
2076 file=sys.stderr)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002077 return 2
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002078
2079
2080if __name__ == '__main__':
maruel@chromium.org35625c72011-03-23 17:34:02 +00002081 fix_encoding.fix_encoding()
sbc@chromium.org013731e2015-02-26 18:28:43 +00002082 try:
2083 sys.exit(main())
2084 except KeyboardInterrupt:
2085 sys.stderr.write('interrupted\n')
sergiybf8a3b382016-07-05 11:21:30 -07002086 sys.exit(2)