blob: 90cac72e80aaa6b5fa658349971abb0369982bd4 [file] [log] [blame]
maruel@chromium.org725f1c32011-04-01 20:24:54 +00001#!/usr/bin/env python
maruel@chromium.org3bbf2942012-01-10 16:52:06 +00002# Copyright (c) 2012 The Chromium Authors. All rights reserved.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Enables directory-specific presubmit checks to run at upload and/or commit.
7"""
8
Raul Tambre80ee78e2019-05-06 22:41:05 +00009from __future__ import print_function
10
Saagar Sanghavi99816902020-08-11 22:41:25 +000011__version__ = '2.0.0'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000012
13# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
14# caching (between all different invocations of presubmit scripts for a given
15# change). We should add it as our presubmit scripts start feeling slow.
16
Edward Lemura5799e32020-01-17 19:26:51 +000017import argparse
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +090018import ast # Exposed through the API.
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +000019import contextlib
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +000020import cpplint
dcheng091b7db2016-06-16 01:27:51 -070021import fnmatch # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000022import glob
asvitkine@chromium.org15169952011-09-27 14:30:53 +000023import inspect
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +000024import itertools
maruel@chromium.org4f6852c2012-04-20 20:39:20 +000025import json # Exposed through the API.
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +000026import logging
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000027import multiprocessing
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000028import os # Somewhat exposed through the API.
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000029import random
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000030import re # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040031import signal
Allen Webbfe7d7092021-05-18 02:05:49 +000032import six
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000033import sys # Parts exposed through API.
34import tempfile # Exposed through the API.
Edward Lesmes8e282792018-04-03 18:50:29 -040035import threading
jam@chromium.org2a891dc2009-08-20 20:33:37 +000036import time
Edward Lemurde9e3ca2019-10-24 21:13:31 +000037import traceback
maruel@chromium.org1487d532009-06-06 00:22:57 +000038import unittest # Exposed through the API.
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +000039from warnings import warn
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000040
41# Local imports.
maruel@chromium.org35625c72011-03-23 17:34:02 +000042import fix_encoding
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +000043import gclient_paths # Exposed through the API
44import gclient_utils
Aaron Gableb584c4f2017-04-26 16:28:08 -070045import git_footers
tandrii@chromium.org015ebae2016-04-25 19:37:22 +000046import gerrit_util
Edward Lesmes9ce03f82021-01-12 20:13:31 +000047import owners as owners_db
48import owners_client
Jochen Eisinger76f5fc62017-04-07 16:27:46 +020049import owners_finder
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000050import presubmit_canned_checks
Saagar Sanghavi9949ab72020-07-20 20:56:40 +000051import rdb_wrapper
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +000052import scm
maruel@chromium.org84f4fe32011-04-06 13:26:45 +000053import subprocess2 as subprocess # Exposed through the API.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000054
Edward Lemur16af3562019-10-17 22:11:33 +000055if sys.version_info.major == 2:
56 # TODO(1009814): Expose urllib2 only through urllib_request and urllib_error
57 import urllib2 # Exposed through the API.
58 import urlparse
59 import urllib2 as urllib_request
60 import urllib2 as urllib_error
61else:
62 import urllib.parse as urlparse
63 import urllib.request as urllib_request
64 import urllib.error as urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000065
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +000066# Ask for feedback only once in program lifetime.
67_ASKED_FOR_FEEDBACK = False
68
Edward Lemurecc27072020-01-06 16:42:34 +000069def time_time():
70 # Use this so that it can be mocked in tests without interfering with python
71 # system machinery.
72 return time.time()
73
74
maruel@chromium.org899e1c12011-04-07 17:03:18 +000075class PresubmitFailure(Exception):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +000076 pass
77
78
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000079class CommandData(object):
Edward Lemur940c2822019-08-23 00:34:25 +000080 def __init__(self, name, cmd, kwargs, message, python3=False):
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000081 self.name = name
82 self.cmd = cmd
Edward Lesmes8e282792018-04-03 18:50:29 -040083 self.stdin = kwargs.get('stdin', None)
Edward Lemur2d6b67c2019-08-23 22:25:41 +000084 self.kwargs = kwargs.copy()
Edward Lesmes8e282792018-04-03 18:50:29 -040085 self.kwargs['stdout'] = subprocess.PIPE
86 self.kwargs['stderr'] = subprocess.STDOUT
87 self.kwargs['stdin'] = subprocess.PIPE
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000088 self.message = message
89 self.info = None
Edward Lemur940c2822019-08-23 00:34:25 +000090 self.python3 = python3
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +000091
ilevy@chromium.orgbc117312013-04-20 03:57:56 +000092
Edward Lesmes8e282792018-04-03 18:50:29 -040093# Adapted from
94# https://github.com/google/gtest-parallel/blob/master/gtest_parallel.py#L37
95#
96# An object that catches SIGINT sent to the Python process and notices
97# if processes passed to wait() die by SIGINT (we need to look for
98# both of those cases, because pressing Ctrl+C can result in either
99# the main process or one of the subprocesses getting the signal).
100#
101# Before a SIGINT is seen, wait(p) will simply call p.wait() and
102# return the result. Once a SIGINT has been seen (in the main process
103# or a subprocess, including the one the current call is waiting for),
Edward Lemur9a5bb612019-09-26 02:01:52 +0000104# wait(p) will call p.terminate().
Edward Lesmes8e282792018-04-03 18:50:29 -0400105class SigintHandler(object):
Edward Lesmes8e282792018-04-03 18:50:29 -0400106 sigint_returncodes = {-signal.SIGINT, # Unix
107 -1073741510, # Windows
108 }
109 def __init__(self):
110 self.__lock = threading.Lock()
111 self.__processes = set()
112 self.__got_sigint = False
Edward Lemur9a5bb612019-09-26 02:01:52 +0000113 self.__previous_signal = signal.signal(signal.SIGINT, self.interrupt)
Edward Lesmes8e282792018-04-03 18:50:29 -0400114
115 def __on_sigint(self):
116 self.__got_sigint = True
117 while self.__processes:
118 try:
119 self.__processes.pop().terminate()
120 except OSError:
121 pass
122
Edward Lemur9a5bb612019-09-26 02:01:52 +0000123 def interrupt(self, signal_num, frame):
Edward Lesmes8e282792018-04-03 18:50:29 -0400124 with self.__lock:
125 self.__on_sigint()
Edward Lemur9a5bb612019-09-26 02:01:52 +0000126 self.__previous_signal(signal_num, frame)
Edward Lesmes8e282792018-04-03 18:50:29 -0400127
128 def got_sigint(self):
129 with self.__lock:
130 return self.__got_sigint
131
132 def wait(self, p, stdin):
133 with self.__lock:
134 if self.__got_sigint:
135 p.terminate()
136 self.__processes.add(p)
137 stdout, stderr = p.communicate(stdin)
138 code = p.returncode
139 with self.__lock:
140 self.__processes.discard(p)
141 if code in self.sigint_returncodes:
142 self.__on_sigint()
Edward Lesmes8e282792018-04-03 18:50:29 -0400143 return stdout, stderr
144
145sigint_handler = SigintHandler()
146
147
Edward Lemurecc27072020-01-06 16:42:34 +0000148class Timer(object):
149 def __init__(self, timeout, fn):
150 self.completed = False
151 self._fn = fn
152 self._timer = threading.Timer(timeout, self._onTimer) if timeout else None
153
154 def __enter__(self):
155 if self._timer:
156 self._timer.start()
157 return self
158
159 def __exit__(self, _type, _value, _traceback):
160 if self._timer:
161 self._timer.cancel()
162
163 def _onTimer(self):
164 self._fn()
165 self.completed = True
166
167
Edward Lesmes8e282792018-04-03 18:50:29 -0400168class ThreadPool(object):
Edward Lemurecc27072020-01-06 16:42:34 +0000169 def __init__(self, pool_size=None, timeout=None):
170 self.timeout = timeout
Edward Lesmes8e282792018-04-03 18:50:29 -0400171 self._pool_size = pool_size or multiprocessing.cpu_count()
172 self._messages = []
173 self._messages_lock = threading.Lock()
174 self._tests = []
175 self._tests_lock = threading.Lock()
176 self._nonparallel_tests = []
177
Edward Lemurecc27072020-01-06 16:42:34 +0000178 def _GetCommand(self, test):
Edward Lemur940c2822019-08-23 00:34:25 +0000179 vpython = 'vpython'
180 if test.python3:
181 vpython += '3'
182 if sys.platform == 'win32':
183 vpython += '.bat'
Edward Lesmes8e282792018-04-03 18:50:29 -0400184
185 cmd = test.cmd
186 if cmd[0] == 'python':
187 cmd = list(cmd)
188 cmd[0] = vpython
189 elif cmd[0].endswith('.py'):
190 cmd = [vpython] + cmd
191
Edward Lemur336e51f2019-11-14 21:42:04 +0000192 # On Windows, scripts on the current directory take precedence over PATH, so
193 # that when testing depot_tools on Windows, calling `vpython.bat` will
194 # execute the copy of vpython of the depot_tools under test instead of the
195 # one in the bot.
196 # As a workaround, we run the tests from the parent directory instead.
197 if (cmd[0] == vpython and
198 'cwd' in test.kwargs and
199 os.path.basename(test.kwargs['cwd']) == 'depot_tools'):
200 test.kwargs['cwd'] = os.path.dirname(test.kwargs['cwd'])
201 cmd[1] = os.path.join('depot_tools', cmd[1])
202
Edward Lemurecc27072020-01-06 16:42:34 +0000203 return cmd
204
205 def _RunWithTimeout(self, cmd, stdin, kwargs):
206 p = subprocess.Popen(cmd, **kwargs)
207 with Timer(self.timeout, p.terminate) as timer:
208 stdout, _ = sigint_handler.wait(p, stdin)
209 if timer.completed:
210 stdout = 'Process timed out after %ss\n%s' % (self.timeout, stdout)
211 return p.returncode, stdout
212
213 def CallCommand(self, test):
214 """Runs an external program.
215
Edward Lemura5799e32020-01-17 19:26:51 +0000216 This function converts invocation of .py files and invocations of 'python'
Edward Lemurecc27072020-01-06 16:42:34 +0000217 to vpython invocations.
218 """
219 cmd = self._GetCommand(test)
Edward Lesmes8e282792018-04-03 18:50:29 -0400220 try:
Edward Lemurecc27072020-01-06 16:42:34 +0000221 start = time_time()
222 returncode, stdout = self._RunWithTimeout(cmd, test.stdin, test.kwargs)
223 duration = time_time() - start
Edward Lemur7cf94382019-11-15 22:36:41 +0000224 except Exception:
Edward Lemurecc27072020-01-06 16:42:34 +0000225 duration = time_time() - start
Edward Lesmes8e282792018-04-03 18:50:29 -0400226 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000227 '%s\n%s exec failure (%4.2fs)\n%s' % (
228 test.name, ' '.join(cmd), duration, traceback.format_exc()))
Edward Lemurde9e3ca2019-10-24 21:13:31 +0000229
Edward Lemurecc27072020-01-06 16:42:34 +0000230 if returncode != 0:
Edward Lesmes8e282792018-04-03 18:50:29 -0400231 return test.message(
Edward Lemur7cf94382019-11-15 22:36:41 +0000232 '%s\n%s (%4.2fs) failed\n%s' % (
233 test.name, ' '.join(cmd), duration, stdout))
Edward Lemurecc27072020-01-06 16:42:34 +0000234
Edward Lesmes8e282792018-04-03 18:50:29 -0400235 if test.info:
Edward Lemur7cf94382019-11-15 22:36:41 +0000236 return test.info('%s\n%s (%4.2fs)' % (test.name, ' '.join(cmd), duration))
Edward Lesmes8e282792018-04-03 18:50:29 -0400237
238 def AddTests(self, tests, parallel=True):
239 if parallel:
240 self._tests.extend(tests)
241 else:
242 self._nonparallel_tests.extend(tests)
243
244 def RunAsync(self):
245 self._messages = []
246
247 def _WorkerFn():
248 while True:
249 test = None
250 with self._tests_lock:
251 if not self._tests:
252 break
253 test = self._tests.pop()
254 result = self.CallCommand(test)
255 if result:
256 with self._messages_lock:
257 self._messages.append(result)
258
259 def _StartDaemon():
260 t = threading.Thread(target=_WorkerFn)
261 t.daemon = True
262 t.start()
263 return t
264
265 while self._nonparallel_tests:
266 test = self._nonparallel_tests.pop()
267 result = self.CallCommand(test)
268 if result:
269 self._messages.append(result)
270
271 if self._tests:
272 threads = [_StartDaemon() for _ in range(self._pool_size)]
273 for worker in threads:
274 worker.join()
275
276 return self._messages
277
278
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000279def normpath(path):
280 '''Version of os.path.normpath that also changes backward slashes to
281 forward slashes when not running on Windows.
282 '''
283 # This is safe to always do because the Windows version of os.path.normpath
284 # will replace forward slashes with backward slashes.
285 path = path.replace(os.sep, '/')
286 return os.path.normpath(path)
287
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000288
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000289def _RightHandSideLinesImpl(affected_files):
290 """Implements RightHandSideLines for InputApi and GclChange."""
291 for af in affected_files:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000292 lines = af.ChangedContents()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000293 for line in lines:
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000294 yield (af, line[0], line[1])
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000295
296
Edward Lemur6eb1d322020-02-27 22:20:15 +0000297def prompt_should_continue(prompt_string):
298 sys.stdout.write(prompt_string)
299 response = sys.stdin.readline().strip().lower()
300 return response in ('y', 'yes')
dpranke@chromium.org5ac21012011-03-16 02:58:25 +0000301
302
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000303# Top level object so multiprocessing can pickle
304# Public access through OutputApi object.
305class _PresubmitResult(object):
306 """Base class for result objects."""
307 fatal = False
308 should_prompt = False
309
310 def __init__(self, message, items=None, long_text=''):
311 """
312 message: A short one-line message to indicate errors.
313 items: A list of short strings to indicate where errors occurred.
314 long_text: multi-line text output, e.g. from another tool
315 """
316 self._message = message
317 self._items = items or []
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000318 self._long_text = long_text.rstrip()
319
Edward Lemur6eb1d322020-02-27 22:20:15 +0000320 def handle(self):
321 sys.stdout.write(self._message)
322 sys.stdout.write('\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000323 for index, item in enumerate(self._items):
Edward Lemur6eb1d322020-02-27 22:20:15 +0000324 sys.stdout.write(' ')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000325 # Write separately in case it's unicode.
Edward Lemur6eb1d322020-02-27 22:20:15 +0000326 sys.stdout.write(str(item))
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000327 if index < len(self._items) - 1:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000328 sys.stdout.write(' \\')
329 sys.stdout.write('\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000330 if self._long_text:
Edward Lemur6eb1d322020-02-27 22:20:15 +0000331 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000332 # Write separately in case it's unicode.
Edward Lemur6eb1d322020-02-27 22:20:15 +0000333 sys.stdout.write(self._long_text)
334 sys.stdout.write('\n***************\n')
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000335
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000336 def json_format(self):
337 return {
338 'message': self._message,
Debrian Figueroa6095d402019-06-28 18:47:18 +0000339 'items': [str(item) for item in self._items],
Debrian Figueroadd2737e2019-06-21 23:50:13 +0000340 'long_text': self._long_text,
341 'fatal': self.fatal
342 }
343
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000344
345# Top level object so multiprocessing can pickle
346# Public access through OutputApi object.
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000347class _PresubmitError(_PresubmitResult):
348 """A hard presubmit error."""
349 fatal = True
350
351
352# Top level object so multiprocessing can pickle
353# Public access through OutputApi object.
354class _PresubmitPromptWarning(_PresubmitResult):
355 """An warning that prompts the user if they want to continue."""
356 should_prompt = True
357
358
359# Top level object so multiprocessing can pickle
360# Public access through OutputApi object.
361class _PresubmitNotifyResult(_PresubmitResult):
362 """Just print something to the screen -- but it's not even a warning."""
363 pass
364
365
366# Top level object so multiprocessing can pickle
367# Public access through OutputApi object.
368class _MailTextResult(_PresubmitResult):
369 """A warning that should be included in the review request email."""
370 def __init__(self, *args, **kwargs):
371 super(_MailTextResult, self).__init__()
372 raise NotImplementedError()
373
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000374class GerritAccessor(object):
375 """Limited Gerrit functionality for canned presubmit checks to work.
376
377 To avoid excessive Gerrit calls, caches the results.
378 """
379
Edward Lesmeseb1bd622021-03-01 19:54:07 +0000380 def __init__(self, url=None, project=None, branch=None):
381 self.host = urlparse.urlparse(url).netloc if url else None
382 self.project = project
383 self.branch = branch
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000384 self.cache = {}
Edward Lesmes8170c292021-03-19 20:04:43 +0000385 self.code_owners_enabled = None
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000386
387 def _FetchChangeDetail(self, issue):
388 # Separate function to be easily mocked in tests.
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100389 try:
390 return gerrit_util.GetChangeDetail(
391 self.host, str(issue),
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700392 ['ALL_REVISIONS', 'DETAILED_LABELS', 'ALL_COMMITS'])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100393 except gerrit_util.GerritError as e:
394 if e.http_status == 404:
395 raise Exception('Either Gerrit issue %s doesn\'t exist, or '
396 'no credentials to fetch issue details' % issue)
397 raise
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000398
399 def GetChangeInfo(self, issue):
400 """Returns labels and all revisions (patchsets) for this issue.
401
402 The result is a dictionary according to Gerrit REST Api.
403 https://gerrit-review.googlesource.com/Documentation/rest-api.html
404
405 However, API isn't very clear what's inside, so see tests for example.
406 """
407 assert issue
408 cache_key = int(issue)
409 if cache_key not in self.cache:
410 self.cache[cache_key] = self._FetchChangeDetail(issue)
411 return self.cache[cache_key]
412
413 def GetChangeDescription(self, issue, patchset=None):
414 """If patchset is none, fetches current patchset."""
415 info = self.GetChangeInfo(issue)
416 # info is a reference to cache. We'll modify it here adding description to
417 # it to the right patchset, if it is not yet there.
418
419 # Find revision info for the patchset we want.
420 if patchset is not None:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +0000421 for rev, rev_info in info['revisions'].items():
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000422 if str(rev_info['_number']) == str(patchset):
423 break
424 else:
425 raise Exception('patchset %s doesn\'t exist in issue %s' % (
426 patchset, issue))
427 else:
428 rev = info['current_revision']
429 rev_info = info['revisions'][rev]
430
Andrii Shyshkalov9c3a4642017-01-24 17:41:22 +0100431 return rev_info['commit']['message']
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000432
Mun Yong Jang603d01e2017-12-19 16:38:30 -0800433 def GetDestRef(self, issue):
434 ref = self.GetChangeInfo(issue)['branch']
435 if not ref.startswith('refs/'):
436 # NOTE: it is possible to create 'refs/x' branch,
437 # aka 'refs/heads/refs/x'. However, this is ill-advised.
438 ref = 'refs/heads/%s' % ref
439 return ref
440
Edward Lesmes02d4b822020-11-11 00:37:35 +0000441 def _GetApproversForLabel(self, issue, label):
442 change_info = self.GetChangeInfo(issue)
443 label_info = change_info.get('labels', {}).get(label, {})
444 values = label_info.get('values', {}).keys()
445 if not values:
446 return []
447 max_value = max(int(v) for v in values)
448 return [v for v in label_info.get('all', [])
449 if v.get('value', 0) == max_value]
450
Edward Lesmesc4566172021-03-19 16:55:13 +0000451 def IsBotCommitApproved(self, issue):
452 return bool(self._GetApproversForLabel(issue, 'Bot-Commit'))
453
Edward Lesmescf49cb82020-11-11 01:08:36 +0000454 def IsOwnersOverrideApproved(self, issue):
455 return bool(self._GetApproversForLabel(issue, 'Owners-Override'))
456
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000457 def GetChangeOwner(self, issue):
458 return self.GetChangeInfo(issue)['owner']['email']
459
460 def GetChangeReviewers(self, issue, approving_only=True):
Aaron Gable8b478f02017-07-31 15:33:19 -0700461 changeinfo = self.GetChangeInfo(issue)
462 if approving_only:
Edward Lesmes02d4b822020-11-11 00:37:35 +0000463 reviewers = self._GetApproversForLabel(issue, 'Code-Review')
Aaron Gable8b478f02017-07-31 15:33:19 -0700464 else:
465 reviewers = changeinfo.get('reviewers', {}).get('REVIEWER', [])
466 return [r.get('email') for r in reviewers]
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000467
Edward Lemure4d329c2020-02-03 20:41:18 +0000468 def UpdateDescription(self, description, issue):
469 gerrit_util.SetCommitMessage(self.host, issue, description, notify='NONE')
470
Edward Lesmes8170c292021-03-19 20:04:43 +0000471 def IsCodeOwnersEnabledOnRepo(self):
472 if self.code_owners_enabled is None:
473 self.code_owners_enabled = gerrit_util.IsCodeOwnersEnabledOnRepo(
Edward Lesmes392c4072021-03-19 21:58:45 +0000474 self.host, self.project)
Edward Lesmes8170c292021-03-19 20:04:43 +0000475 return self.code_owners_enabled
476
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000477
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000478class OutputApi(object):
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000479 """An instance of OutputApi gets passed to presubmit scripts so that they
480 can output various types of results.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000481 """
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000482 PresubmitResult = _PresubmitResult
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000483 PresubmitError = _PresubmitError
484 PresubmitPromptWarning = _PresubmitPromptWarning
485 PresubmitNotifyResult = _PresubmitNotifyResult
486 MailTextResult = _MailTextResult
487
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000488 def __init__(self, is_committing):
489 self.is_committing = is_committing
Daniel Cheng7227d212017-11-17 08:12:37 -0800490 self.more_cc = []
491
492 def AppendCC(self, cc):
493 """Appends a user to cc for this change."""
494 self.more_cc.append(cc)
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000495
wez@chromium.orga6d011e2013-03-26 17:31:49 +0000496 def PresubmitPromptOrNotify(self, *args, **kwargs):
497 """Warn the user when uploading, but only notify if committing."""
498 if self.is_committing:
499 return self.PresubmitNotifyResult(*args, **kwargs)
500 return self.PresubmitPromptWarning(*args, **kwargs)
501
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000502
503class InputApi(object):
504 """An instance of this object is passed to presubmit scripts so they can
505 know stuff about the change they're looking at.
506 """
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000507 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800508 # pylint: disable=no-self-use
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000509
maruel@chromium.org3410d912009-06-09 20:56:16 +0000510 # File extensions that are considered source files from a style guide
511 # perspective. Don't modify this list from a presubmit script!
maruel@chromium.orgc33455a2011-06-24 19:14:18 +0000512 #
513 # Files without an extension aren't included in the list. If you want to
local_bot30f774e2020-06-25 18:23:34 +0000514 # filter them as source files, add r'(^|.*?[\\\/])[^.]+$' to the allow list.
local_bot64021412020-07-08 21:05:39 +0000515 # Note that ALL CAPS files are skipped in DEFAULT_FILES_TO_SKIP below.
516 DEFAULT_FILES_TO_CHECK = (
maruel@chromium.org3410d912009-06-09 20:56:16 +0000517 # C++ and friends
Edward Lemura5799e32020-01-17 19:26:51 +0000518 r'.+\.c$', r'.+\.cc$', r'.+\.cpp$', r'.+\.h$', r'.+\.m$', r'.+\.mm$',
519 r'.+\.inl$', r'.+\.asm$', r'.+\.hxx$', r'.+\.hpp$', r'.+\.s$', r'.+\.S$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000520 # Scripts
Edward Lemura5799e32020-01-17 19:26:51 +0000521 r'.+\.js$', r'.+\.py$', r'.+\.sh$', r'.+\.rb$', r'.+\.pl$', r'.+\.pm$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000522 # Other
Edward Lemura5799e32020-01-17 19:26:51 +0000523 r'.+\.java$', r'.+\.mk$', r'.+\.am$', r'.+\.css$', r'.+\.mojom$',
524 r'.+\.fidl$'
maruel@chromium.org3410d912009-06-09 20:56:16 +0000525 )
526
527 # Path regexp that should be excluded from being considered containing source
528 # files. Don't modify this list from a presubmit script!
local_bot64021412020-07-08 21:05:39 +0000529 DEFAULT_FILES_TO_SKIP = (
Edward Lemura5799e32020-01-17 19:26:51 +0000530 r'testing_support[\\\/]google_appengine[\\\/].*',
531 r'.*\bexperimental[\\\/].*',
Kent Tamura179dd1e2018-04-26 15:07:41 +0900532 # Exclude third_party/.* but NOT third_party/{WebKit,blink}
533 # (crbug.com/539768 and crbug.com/836555).
Edward Lemura5799e32020-01-17 19:26:51 +0000534 r'.*\bthird_party[\\\/](?!(WebKit|blink)[\\\/]).*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000535 # Output directories (just in case)
Edward Lemura5799e32020-01-17 19:26:51 +0000536 r'.*\bDebug[\\\/].*',
537 r'.*\bRelease[\\\/].*',
538 r'.*\bxcodebuild[\\\/].*',
539 r'.*\bout[\\\/].*',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000540 # All caps files like README and LICENCE.
Edward Lemura5799e32020-01-17 19:26:51 +0000541 r'.*\b[A-Z0-9_]{2,}$',
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000542 # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
Edward Lemura5799e32020-01-17 19:26:51 +0000543 r'(|.*[\\\/])\.git[\\\/].*',
544 r'(|.*[\\\/])\.svn[\\\/].*',
maruel@chromium.org7ccb4bb2011-11-07 20:26:20 +0000545 # There is no point in processing a patch file.
Edward Lemura5799e32020-01-17 19:26:51 +0000546 r'.+\.diff$',
547 r'.+\.patch$',
maruel@chromium.org3410d912009-06-09 20:56:16 +0000548 )
549
local_bot30f774e2020-06-25 18:23:34 +0000550 # TODO(https://crbug.com/1098562): Remove once no longer used
551 @property
552 def DEFAULT_WHITE_LIST(self):
local_bot64021412020-07-08 21:05:39 +0000553 return self.DEFAULT_FILES_TO_CHECK
local_bot30f774e2020-06-25 18:23:34 +0000554
555 # TODO(https://crbug.com/1098562): Remove once no longer used
local_bot37ce2012020-06-26 17:39:24 +0000556 @DEFAULT_WHITE_LIST.setter
557 def DEFAULT_WHITE_LIST(self, value):
local_bot64021412020-07-08 21:05:39 +0000558 self.DEFAULT_FILES_TO_CHECK = value
559
560 # TODO(https://crbug.com/1098562): Remove once no longer used
561 @property
562 def DEFAULT_ALLOW_LIST(self):
563 return self.DEFAULT_FILES_TO_CHECK
564
565 # TODO(https://crbug.com/1098562): Remove once no longer used
566 @DEFAULT_ALLOW_LIST.setter
567 def DEFAULT_ALLOW_LIST(self, value):
568 self.DEFAULT_FILES_TO_CHECK = value
local_bot37ce2012020-06-26 17:39:24 +0000569
570 # TODO(https://crbug.com/1098562): Remove once no longer used
local_bot30f774e2020-06-25 18:23:34 +0000571 @property
572 def DEFAULT_BLACK_LIST(self):
local_bot64021412020-07-08 21:05:39 +0000573 return self.DEFAULT_FILES_TO_SKIP
local_bot30f774e2020-06-25 18:23:34 +0000574
local_bot37ce2012020-06-26 17:39:24 +0000575 # TODO(https://crbug.com/1098562): Remove once no longer used
576 @DEFAULT_BLACK_LIST.setter
577 def DEFAULT_BLACK_LIST(self, value):
local_bot64021412020-07-08 21:05:39 +0000578 self.DEFAULT_FILES_TO_SKIP = value
579
580 # TODO(https://crbug.com/1098562): Remove once no longer used
581 @property
582 def DEFAULT_BLOCK_LIST(self):
583 return self.DEFAULT_FILES_TO_SKIP
584
585 # TODO(https://crbug.com/1098562): Remove once no longer used
586 @DEFAULT_BLOCK_LIST.setter
587 def DEFAULT_BLOCK_LIST(self, value):
588 self.DEFAULT_FILES_TO_SKIP = value
local_bot37ce2012020-06-26 17:39:24 +0000589
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000590 def __init__(self, change, presubmit_path, is_committing,
Edward Lesmes8e282792018-04-03 18:50:29 -0400591 verbose, gerrit_obj, dry_run=None, thread_pool=None, parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000592 """Builds an InputApi object.
593
594 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000595 change: A presubmit.Change object.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000596 presubmit_path: The path to the presubmit script being processed.
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000597 is_committing: True if the change is about to be committed.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000598 gerrit_obj: provides basic Gerrit codereview functionality.
599 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -0400600 parallel: if true, all tests reported via input_api.RunTests for all
601 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000602 """
maruel@chromium.org9711bba2009-05-22 23:51:39 +0000603 # Version number of the presubmit_support script.
604 self.version = [int(x) for x in __version__.split('.')]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000605 self.change = change
maruel@chromium.orgd7dccf52009-06-06 18:51:58 +0000606 self.is_committing = is_committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +0000607 self.gerrit = gerrit_obj
tandrii@chromium.org57bafac2016-04-28 05:09:03 +0000608 self.dry_run = dry_run
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000609
Edward Lesmes8e282792018-04-03 18:50:29 -0400610 self.parallel = parallel
611 self.thread_pool = thread_pool or ThreadPool()
612
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000613 # We expose various modules and functions as attributes of the input_api
614 # so that presubmit scripts don't have to import them.
Takeshi Yoshino07a6bea2017-08-02 02:44:06 +0900615 self.ast = ast
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000616 self.basename = os.path.basename
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000617 self.cpplint = cpplint
dcheng091b7db2016-06-16 01:27:51 -0700618 self.fnmatch = fnmatch
Yoshisato Yanagisawa04600b42019-03-15 03:03:41 +0000619 self.gclient_paths = gclient_paths
Yoshisato Yanagisawa57dd17b2019-03-22 09:10:29 +0000620 # TODO(yyanagisawa): stop exposing this when python3 become default.
621 # Since python3's tempfile has TemporaryDirectory, we do not need this.
622 self.temporary_directory = gclient_utils.temporary_directory
dpranke@chromium.org17cc2442012-10-17 21:12:09 +0000623 self.glob = glob.glob
maruel@chromium.orgfb11c7b2010-03-18 18:22:14 +0000624 self.json = json
maruel@chromium.org6fba34d2011-06-02 13:45:12 +0000625 self.logging = logging.getLogger('PRESUBMIT')
maruel@chromium.org2b5ce562011-03-31 16:15:44 +0000626 self.os_listdir = os.listdir
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000627 self.os_path = os.path
pgervais@chromium.orgbd0cace2014-10-02 23:23:46 +0000628 self.os_stat = os.stat
Yoshisato Yanagisawa406de132018-06-29 05:43:25 +0000629 self.os_walk = os.walk
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000630 self.re = re
631 self.subprocess = subprocess
Edward Lemurb9830242019-10-30 22:19:20 +0000632 self.sys = sys
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000633 self.tempfile = tempfile
dpranke@chromium.org0d1bdea2011-03-24 22:54:38 +0000634 self.time = time
maruel@chromium.org1487d532009-06-06 00:22:57 +0000635 self.unittest = unittest
Edward Lemurb9830242019-10-30 22:19:20 +0000636 if sys.version_info.major == 2:
637 self.urllib2 = urllib2
Edward Lemur16af3562019-10-17 22:11:33 +0000638 self.urllib_request = urllib_request
639 self.urllib_error = urllib_error
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000640
Robert Iannucci50258932018-03-19 10:30:59 -0700641 self.is_windows = sys.platform == 'win32'
642
Edward Lemurb9646622019-10-25 20:57:35 +0000643 # Set python_executable to 'vpython' in order to allow scripts in other
644 # repos (e.g. src.git) to automatically pick up that repo's .vpython file,
645 # instead of inheriting the one in depot_tools.
646 self.python_executable = 'vpython'
Erik Staab69135d12021-05-14 22:31:57 +0000647 # Offer a python 3 executable for use during the migration off of python 2.
648 self.python3_executable = 'vpython3'
maruel@chromium.orgc0b22972009-06-25 16:19:14 +0000649 self.environ = os.environ
650
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000651 # InputApi.platform is the platform you're currently running on.
652 self.platform = sys.platform
653
iannucci@chromium.org0af3bb32015-06-12 20:44:35 +0000654 self.cpu_count = multiprocessing.cpu_count()
655
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000656 # The local path of the currently-being-processed presubmit script.
maruel@chromium.org3d235242009-05-15 12:40:48 +0000657 self._current_presubmit_path = os.path.dirname(presubmit_path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000658
659 # We carry the canned checks so presubmit scripts can easily use them.
660 self.canned_checks = presubmit_canned_checks
661
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100662 # Temporary files we must manually remove at the end of a run.
663 self._named_temporary_files = []
Jochen Eisinger72606f82017-04-04 10:44:18 +0200664
Edward Lesmesdc11c6b2021-03-19 19:22:05 +0000665 self.owners_client = None
666 if self.gerrit:
667 self.owners_client = owners_client.GetCodeOwnersClient(
668 root=change.RepositoryRoot(),
669 upstream=change.UpstreamBranch(),
670 host=self.gerrit.host,
671 project=self.gerrit.project,
672 branch=self.gerrit.branch)
Edward Lesmes9ce03f82021-01-12 20:13:31 +0000673 self.owners_db = owners_db.Database(
674 change.RepositoryRoot(), fopen=open, os_path=self.os_path)
Jochen Eisinger76f5fc62017-04-07 16:27:46 +0200675 self.owners_finder = owners_finder.OwnersFinder
maruel@chromium.org899e1c12011-04-07 17:03:18 +0000676 self.verbose = verbose
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000677 self.Command = CommandData
dpranke@chromium.org2a009622011-03-01 02:43:31 +0000678
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000679 # Replace <hash_map> and <hash_set> as headers that need to be included
Edward Lemura5799e32020-01-17 19:26:51 +0000680 # with 'base/containers/hash_tables.h' instead.
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000681 # Access to a protected member _XX of a client class
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800682 # pylint: disable=protected-access
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000683 self.cpplint._re_pattern_templates = [
danakj@chromium.org18278522013-06-11 22:42:32 +0000684 (a, b, 'base/containers/hash_tables.h')
enne@chromium.orge72c5f52013-04-16 00:36:40 +0000685 if header in ('<hash_map>', '<hash_set>') else (a, b, header)
686 for (a, b, header) in cpplint._re_pattern_templates
687 ]
688
Edward Lemurecc27072020-01-06 16:42:34 +0000689 def SetTimeout(self, timeout):
690 self.thread_pool.timeout = timeout
691
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000692 def PresubmitLocalPath(self):
693 """Returns the local path of the presubmit script currently being run.
694
695 This is useful if you don't want to hard-code absolute paths in the
696 presubmit script. For example, It can be used to find another file
697 relative to the PRESUBMIT.py script, so the whole tree can be branched and
698 the presubmit script still works, without editing its content.
699 """
maruel@chromium.org3d235242009-05-15 12:40:48 +0000700 return self._current_presubmit_path
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000701
agable0b65e732016-11-22 09:25:46 -0800702 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000703 """Same as input_api.change.AffectedFiles() except only lists files
704 (and optionally directories) in the same directory as the current presubmit
Bruce Dawson7a0b07a2020-04-23 17:14:40 +0000705 script, or subdirectories thereof. Note that files are listed using the OS
706 path separator, so backslashes are used as separators on Windows.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000707 """
Edward Lemura5799e32020-01-17 19:26:51 +0000708 dir_with_slash = normpath('%s/' % self.PresubmitLocalPath())
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000709 if len(dir_with_slash) == 1:
710 dir_with_slash = ''
sail@chromium.org5538e022011-05-12 17:53:16 +0000711
Edward Lemurb9830242019-10-30 22:19:20 +0000712 return list(filter(
maruel@chromium.org4661e0c2009-06-04 00:45:26 +0000713 lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
Edward Lemurb9830242019-10-30 22:19:20 +0000714 self.change.AffectedFiles(include_deletes, file_filter)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000715
agable0b65e732016-11-22 09:25:46 -0800716 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000717 """Returns local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800718 paths = [af.LocalPath() for af in self.AffectedFiles()]
Edward Lemura5799e32020-01-17 19:26:51 +0000719 logging.debug('LocalPaths: %s', paths)
pgervais@chromium.org2f64f782014-04-25 00:12:33 +0000720 return paths
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000721
agable0b65e732016-11-22 09:25:46 -0800722 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000723 """Returns absolute local paths of input_api.AffectedFiles()."""
agable0b65e732016-11-22 09:25:46 -0800724 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000725
John Budorick16162372018-04-18 10:39:53 -0700726 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
agable0b65e732016-11-22 09:25:46 -0800727 """Same as input_api.change.AffectedTestableFiles() except only lists files
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000728 in the same directory as the current presubmit script, or subdirectories
729 thereof.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000730 """
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000731 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +0000732 warn('AffectedTestableFiles(include_deletes=%s)'
733 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000734 category=DeprecationWarning,
735 stacklevel=2)
Edward Lemurb9830242019-10-30 22:19:20 +0000736 return list(filter(
737 lambda x: x.IsTestableFile(),
738 self.AffectedFiles(include_deletes=False, **kwargs)))
agable0b65e732016-11-22 09:25:46 -0800739
740 def AffectedTextFiles(self, include_deletes=None):
741 """An alias to AffectedTestableFiles for backwards compatibility."""
742 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000743
Josip Sokcevic8c955952021-02-01 21:32:57 +0000744 def FilterSourceFile(self,
745 affected_file,
746 files_to_check=None,
747 files_to_skip=None,
748 allow_list=None,
749 block_list=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000750 """Filters out files that aren't considered 'source file'.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000751
local_bot64021412020-07-08 21:05:39 +0000752 If files_to_check or files_to_skip is None, InputApi.DEFAULT_FILES_TO_CHECK
753 and InputApi.DEFAULT_FILES_TO_SKIP is used respectively.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000754
755 The lists will be compiled as regular expression and
756 AffectedFile.LocalPath() needs to pass both list.
757
758 Note: Copy-paste this function to suit your needs or use a lambda function.
759 """
local_bot64021412020-07-08 21:05:39 +0000760 if files_to_check is None:
761 files_to_check = self.DEFAULT_FILES_TO_CHECK
762 if files_to_skip is None:
763 files_to_skip = self.DEFAULT_FILES_TO_SKIP
local_bot30f774e2020-06-25 18:23:34 +0000764
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000765 def Find(affected_file, items):
maruel@chromium.orgab05d582011-02-09 23:41:22 +0000766 local_path = affected_file.LocalPath()
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000767 for item in items:
maruel@chromium.orgdf1595a2009-06-11 02:00:13 +0000768 if self.re.match(item, local_path):
maruel@chromium.org3410d912009-06-09 20:56:16 +0000769 return True
770 return False
local_bot64021412020-07-08 21:05:39 +0000771 return (Find(affected_file, files_to_check) and
772 not Find(affected_file, files_to_skip))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000773
774 def AffectedSourceFiles(self, source_file):
agable0b65e732016-11-22 09:25:46 -0800775 """Filter the list of AffectedTestableFiles by the function source_file.
maruel@chromium.org3410d912009-06-09 20:56:16 +0000776
777 If source_file is None, InputApi.FilterSourceFile() is used.
778 """
779 if not source_file:
780 source_file = self.FilterSourceFile
Edward Lemurb9830242019-10-30 22:19:20 +0000781 return list(filter(source_file, self.AffectedTestableFiles()))
maruel@chromium.org3410d912009-06-09 20:56:16 +0000782
783 def RightHandSideLines(self, source_file_filter=None):
Edward Lemura5799e32020-01-17 19:26:51 +0000784 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000785
786 Only lists lines from new or modified text files in the change that are
787 contained by the directory of the currently executing presubmit script.
788
789 This is useful for doing line-by-line regex checks, like checking for
790 trailing whitespace.
791
792 Yields:
793 a 3 tuple:
794 the AffectedFile instance of the current file;
795 integer line number (1-based); and
796 the contents of the line as a string.
maruel@chromium.org1487d532009-06-06 00:22:57 +0000797
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000798 Note: The carriage return (LF or CR) is stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000799 """
maruel@chromium.org3410d912009-06-09 20:56:16 +0000800 files = self.AffectedSourceFiles(source_file_filter)
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +0000801 return _RightHandSideLinesImpl(files)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000802
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000803 def ReadFile(self, file_item, mode='r'):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000804 """Reads an arbitrary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000805
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000806 Deny reading anything outside the repository.
807 """
maruel@chromium.orge3608df2009-11-10 20:22:57 +0000808 if isinstance(file_item, AffectedFile):
809 file_item = file_item.AbsoluteLocalPath()
810 if not file_item.startswith(self.change.RepositoryRoot()):
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000811 raise IOError('Access outside the repository root is denied.')
maruel@chromium.org5aeb7dd2009-11-17 18:09:01 +0000812 return gclient_utils.FileRead(file_item, mode)
maruel@chromium.org44a17ad2009-06-08 14:14:35 +0000813
Raphael Kubo da Costaf2d16152017-11-10 18:07:58 +0100814 def CreateTemporaryFile(self, **kwargs):
815 """Returns a named temporary file that must be removed with a call to
816 RemoveTemporaryFiles().
817
818 All keyword arguments are forwarded to tempfile.NamedTemporaryFile(),
819 except for |delete|, which is always set to False.
820
821 Presubmit checks that need to create a temporary file and pass it for
822 reading should use this function instead of NamedTemporaryFile(), as
823 Windows fails to open a file that is already open for writing.
824
825 with input_api.CreateTemporaryFile() as f:
826 f.write('xyz')
827 f.close()
828 input_api.subprocess.check_output(['script-that', '--reads-from',
829 f.name])
830
831
832 Note that callers of CreateTemporaryFile() should not worry about removing
833 any temporary file; this is done transparently by the presubmit handling
834 code.
835 """
836 if 'delete' in kwargs:
837 # Prevent users from passing |delete|; we take care of file deletion
838 # ourselves and this prevents unintuitive error messages when we pass
839 # delete=False and 'delete' is also in kwargs.
840 raise TypeError('CreateTemporaryFile() does not take a "delete" '
841 'argument, file deletion is handled automatically by '
842 'the same presubmit_support code that creates InputApi '
843 'objects.')
844 temp_file = self.tempfile.NamedTemporaryFile(delete=False, **kwargs)
845 self._named_temporary_files.append(temp_file.name)
846 return temp_file
847
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000848 @property
849 def tbr(self):
850 """Returns if a change is TBR'ed."""
Jeremy Romandce22502017-06-20 15:37:29 -0400851 return 'TBR' in self.change.tags or self.change.TBRsFromDescription()
maruel@chromium.orgcc73ad62011-07-06 17:39:26 +0000852
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000853 def RunTests(self, tests_mix, parallel=True):
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000854 tests = []
855 msgs = []
856 for t in tests_mix:
Edward Lesmes8e282792018-04-03 18:50:29 -0400857 if isinstance(t, OutputApi.PresubmitResult) and t:
ilevy@chromium.orgbc117312013-04-20 03:57:56 +0000858 msgs.append(t)
859 else:
860 assert issubclass(t.message, _PresubmitResult)
861 tests.append(t)
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +0000862 if self.verbose:
863 t.info = _PresubmitNotifyResult
Edward Lemur1037c742018-05-01 18:56:04 -0400864 if not t.kwargs.get('cwd'):
865 t.kwargs['cwd'] = self.PresubmitLocalPath()
Edward Lesmes8e282792018-04-03 18:50:29 -0400866 self.thread_pool.AddTests(tests, parallel)
Edward Lemur21000eb2019-05-24 23:25:58 +0000867 # When self.parallel is True (i.e. --parallel is passed as an option)
868 # RunTests doesn't actually run tests. It adds them to a ThreadPool that
869 # will run all tests once all PRESUBMIT files are processed.
870 # Otherwise, it will run them and return the results.
871 if not self.parallel:
Edward Lesmes8e282792018-04-03 18:50:29 -0400872 msgs.extend(self.thread_pool.RunAsync())
873 return msgs
scottmg86099d72016-09-01 09:16:51 -0700874
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000875
nick@chromium.orgff526192013-06-10 19:30:26 +0000876class _DiffCache(object):
877 """Caches diffs retrieved from a particular SCM."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000878 def __init__(self, upstream=None):
879 """Stores the upstream revision against which all diffs will be computed."""
880 self._upstream = upstream
nick@chromium.orgff526192013-06-10 19:30:26 +0000881
882 def GetDiff(self, path, local_root):
883 """Get the diff for a particular path."""
884 raise NotImplementedError()
885
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700886 def GetOldContents(self, path, local_root):
887 """Get the old version for a particular path."""
888 raise NotImplementedError()
889
nick@chromium.orgff526192013-06-10 19:30:26 +0000890
nick@chromium.orgff526192013-06-10 19:30:26 +0000891class _GitDiffCache(_DiffCache):
892 """DiffCache implementation for git; gets all file diffs at once."""
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000893 def __init__(self, upstream):
894 super(_GitDiffCache, self).__init__(upstream=upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000895 self._diffs_by_file = None
896
897 def GetDiff(self, path, local_root):
898 if not self._diffs_by_file:
899 # Compute a single diff for all files and parse the output; should
900 # with git this is much faster than computing one diff for each file.
901 diffs = {}
902
903 # Don't specify any filenames below, because there are command line length
904 # limits on some platforms and GenerateDiff would fail.
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000905 unified_diff = scm.GIT.GenerateDiff(local_root, files=[], full_move=True,
906 branch=self._upstream)
nick@chromium.orgff526192013-06-10 19:30:26 +0000907
908 # This regex matches the path twice, separated by a space. Note that
909 # filename itself may contain spaces.
910 file_marker = re.compile('^diff --git (?P<filename>.*) (?P=filename)$')
911 current_diff = []
912 keep_line_endings = True
913 for x in unified_diff.splitlines(keep_line_endings):
914 match = file_marker.match(x)
915 if match:
916 # Marks the start of a new per-file section.
917 diffs[match.group('filename')] = current_diff = [x]
918 elif x.startswith('diff --git'):
919 raise PresubmitFailure('Unexpected diff line: %s' % x)
920 else:
921 current_diff.append(x)
922
923 self._diffs_by_file = dict(
924 (normpath(path), ''.join(diff)) for path, diff in diffs.items())
925
926 if path not in self._diffs_by_file:
927 raise PresubmitFailure(
928 'Unified diff did not contain entry for file %s' % path)
929
930 return self._diffs_by_file[path]
931
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700932 def GetOldContents(self, path, local_root):
933 return scm.GIT.GetOldContents(local_root, path, branch=self._upstream)
934
nick@chromium.orgff526192013-06-10 19:30:26 +0000935
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000936class AffectedFile(object):
937 """Representation of a file in a change."""
nick@chromium.orgff526192013-06-10 19:30:26 +0000938
939 DIFF_CACHE = _DiffCache
940
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +0000941 # Method could be a function
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800942 # pylint: disable=no-self-use
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000943 def __init__(self, path, action, repository_root, diff_cache):
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000944 self._path = path
945 self._action = action
maruel@chromium.org4ff922a2009-06-12 20:20:19 +0000946 self._local_root = repository_root
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000947 self._is_directory = None
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +0000948 self._cached_changed_contents = None
949 self._cached_new_contents = None
agable@chromium.orgea84ef12014-04-30 19:55:12 +0000950 self._diff_cache = diff_cache
tobiasjs2836bcf2016-08-16 04:08:16 -0700951 logging.debug('%s(%s)', self.__class__.__name__, self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000952
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000953 def LocalPath(self):
954 """Returns the path of this file on the local disk relative to client root.
Andrew Grieve92b8b992017-11-02 09:42:24 -0400955
956 This should be used for error messages but not for accessing files,
957 because presubmit checks are run with CWD=PresubmitLocalPath() (which is
958 often != client root).
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000959 """
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000960 return normpath(self._path)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000961
962 def AbsoluteLocalPath(self):
963 """Returns the absolute path of this file on the local disk.
964 """
chase@chromium.org8e416c82009-10-06 04:30:44 +0000965 return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000966
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000967 def Action(self):
968 """Returns the action on this opened file, e.g. A, M, D, etc."""
maruel@chromium.org15bdffa2009-05-29 11:16:29 +0000969 return self._action
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000970
agable0b65e732016-11-22 09:25:46 -0800971 def IsTestableFile(self):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000972 """Returns True if the file is a text file and not a binary file.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +0000973
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +0000974 Deleted files are not text file."""
maruel@chromium.org1e08c002009-05-28 19:09:33 +0000975 raise NotImplementedError() # Implement when needed
976
agable0b65e732016-11-22 09:25:46 -0800977 def IsTextFile(self):
978 """An alias to IsTestableFile for backwards compatibility."""
979 return self.IsTestableFile()
980
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700981 def OldContents(self):
982 """Returns an iterator over the lines in the old version of file.
983
Daniel Cheng2da34fe2017-03-21 20:42:12 -0700984 The old version is the file before any modifications in the user's
Edward Lemura5799e32020-01-17 19:26:51 +0000985 workspace, i.e. the 'left hand side'.
Daniel Cheng7a1f04d2017-03-21 19:12:31 -0700986
987 Contents will be empty if the file is a directory or does not exist.
988 Note: The carriage returns (LF or CR) are stripped off.
989 """
990 return self._diff_cache.GetOldContents(self.LocalPath(),
991 self._local_root).splitlines()
992
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000993 def NewContents(self):
994 """Returns an iterator over the lines in the new version of file.
995
Edward Lemura5799e32020-01-17 19:26:51 +0000996 The new version is the file in the user's workspace, i.e. the 'right hand
997 side'.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +0000998
999 Contents will be empty if the file is a directory or does not exist.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001000 Note: The carriage returns (LF or CR) are stripped off.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001001 """
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001002 if self._cached_new_contents is None:
1003 self._cached_new_contents = []
agable0b65e732016-11-22 09:25:46 -08001004 try:
1005 self._cached_new_contents = gclient_utils.FileRead(
1006 self.AbsoluteLocalPath(), 'rU').splitlines()
1007 except IOError:
1008 pass # File not found? That's fine; maybe it was deleted.
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001009 return self._cached_new_contents[:]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001010
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001011 def ChangedContents(self, keeplinebreaks=False):
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001012 """Returns a list of tuples (line number, line text) of all new lines.
1013
1014 This relies on the scm diff output describing each changed code section
1015 with a line of the form
1016
1017 ^@@ <old line num>,<old size> <new line num>,<new size> @@$
1018 """
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001019 # Don't return cached results when line breaks are requested.
1020 if not keeplinebreaks and self._cached_changed_contents is not None:
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001021 return self._cached_changed_contents[:]
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001022 result = []
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001023 line_num = 0
1024
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001025 # The keeplinebreaks parameter to splitlines must be True or else the
1026 # CheckForWindowsLineEndings presubmit will be a NOP.
1027 for line in self.GenerateScmDiff().splitlines(keeplinebreaks):
Edward Lemurac5c55f2020-02-29 00:17:16 +00001028 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
1029 if m:
1030 line_num = int(m.groups(1)[0])
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001031 continue
1032 if line.startswith('+') and not line.startswith('++'):
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001033 result.append((line_num, line[1:]))
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001034 if not line.startswith('-'):
1035 line_num += 1
Bruce Dawsonf0bcfdd2021-05-21 18:10:23 +00001036 # Don't cache results with line breaks.
1037 if keeplinebreaks:
1038 return result;
1039 self._cached_changed_contents = result
nick@chromium.org2a3ab7e2011-04-27 22:06:27 +00001040 return self._cached_changed_contents[:]
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001041
maruel@chromium.org5de13972009-06-10 18:16:06 +00001042 def __str__(self):
1043 return self.LocalPath()
1044
maruel@chromium.orgab05d582011-02-09 23:41:22 +00001045 def GenerateScmDiff(self):
nick@chromium.orgff526192013-06-10 19:30:26 +00001046 return self._diff_cache.GetDiff(self.LocalPath(), self._local_root)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001047
maruel@chromium.org58407af2011-04-12 23:15:57 +00001048
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001049class GitAffectedFile(AffectedFile):
1050 """Representation of a file in a change out of a git checkout."""
maruel@chromium.orgb17b55b2010-11-03 14:42:37 +00001051 # Method 'NNN' is abstract in class 'NNN' but is not overridden
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -08001052 # pylint: disable=abstract-method
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001053
nick@chromium.orgff526192013-06-10 19:30:26 +00001054 DIFF_CACHE = _GitDiffCache
1055
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001056 def __init__(self, *args, **kwargs):
1057 AffectedFile.__init__(self, *args, **kwargs)
1058 self._server_path = None
agable0b65e732016-11-22 09:25:46 -08001059 self._is_testable_file = None
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001060
agable0b65e732016-11-22 09:25:46 -08001061 def IsTestableFile(self):
1062 if self._is_testable_file is None:
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001063 if self.Action() == 'D':
agable0b65e732016-11-22 09:25:46 -08001064 # A deleted file is not testable.
1065 self._is_testable_file = False
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001066 else:
agable0b65e732016-11-22 09:25:46 -08001067 self._is_testable_file = os.path.isfile(self.AbsoluteLocalPath())
1068 return self._is_testable_file
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001069
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001070
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001071class Change(object):
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001072 """Describe a change.
1073
1074 Used directly by the presubmit scripts to query the current change being
1075 tested.
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001076
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001077 Instance members:
nick@chromium.orgff526192013-06-10 19:30:26 +00001078 tags: Dictionary of KEY=VALUE pairs found in the change description.
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001079 self.KEY: equivalent to tags['KEY']
1080 """
1081
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001082 _AFFECTED_FILES = AffectedFile
1083
Edward Lemura5799e32020-01-17 19:26:51 +00001084 # Matches key/value (or 'tag') lines in changelist descriptions.
maruel@chromium.org428342a2011-11-10 15:46:33 +00001085 TAG_LINE_RE = re.compile(
maruel@chromium.orgc6f60e82013-04-19 17:01:57 +00001086 '^[ \t]*(?P<key>[A-Z][A-Z_0-9]*)[ \t]*=[ \t]*(?P<value>.*?)[ \t]*$')
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001087 scm = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001088
maruel@chromium.org58407af2011-04-12 23:15:57 +00001089 def __init__(
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001090 self, name, description, local_root, files, issue, patchset, author,
1091 upstream=None):
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001092 if files is None:
1093 files = []
1094 self._name = name
chase@chromium.org8e416c82009-10-06 04:30:44 +00001095 # Convert root into an absolute path.
1096 self._local_root = os.path.abspath(local_root)
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001097 self._upstream = upstream
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001098 self.issue = issue
1099 self.patchset = patchset
maruel@chromium.org58407af2011-04-12 23:15:57 +00001100 self.author_email = author
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001101
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001102 self._full_description = ''
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001103 self.tags = {}
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001104 self._description_without_tags = ''
1105 self.SetDescriptionText(description)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001106
maruel@chromium.orge085d812011-10-10 19:49:15 +00001107 assert all(
1108 (isinstance(f, (list, tuple)) and len(f) == 2) for f in files), files
1109
agable@chromium.orgea84ef12014-04-30 19:55:12 +00001110 diff_cache = self._AFFECTED_FILES.DIFF_CACHE(self._upstream)
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001111 self._affected_files = [
nick@chromium.orgff526192013-06-10 19:30:26 +00001112 self._AFFECTED_FILES(path, action.strip(), self._local_root, diff_cache)
1113 for action, path in files
maruel@chromium.orgdbbeedc2009-05-22 20:26:17 +00001114 ]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001115
Edward Lesmes9ce03f82021-01-12 20:13:31 +00001116 def UpstreamBranch(self):
1117 """Returns the upstream branch for the change."""
1118 return self._upstream
1119
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001120 def Name(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001121 """Returns the change name."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001122 return self._name
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001123
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001124 def DescriptionText(self):
1125 """Returns the user-entered changelist description, minus tags.
1126
Edward Lemura5799e32020-01-17 19:26:51 +00001127 Any line in the user-provided description starting with e.g. 'FOO='
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001128 (whitespace permitted before and around) is considered a tag line. Such
1129 lines are stripped out of the description this function returns.
1130 """
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001131 return self._description_without_tags
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001132
1133 def FullDescriptionText(self):
1134 """Returns the complete changelist description including tags."""
maruel@chromium.org6ebe68a2009-05-27 23:43:40 +00001135 return self._full_description
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001136
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001137 def SetDescriptionText(self, description):
1138 """Sets the full description text (including tags) to |description|.
pgervais@chromium.org92c30092014-04-15 00:30:37 +00001139
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001140 Also updates the list of tags."""
1141 self._full_description = description
1142
1143 # From the description text, build up a dictionary of key/value pairs
Edward Lemura5799e32020-01-17 19:26:51 +00001144 # plus the description minus all key/value or 'tag' lines.
isherman@chromium.orgb5cded62014-03-25 17:47:57 +00001145 description_without_tags = []
1146 self.tags = {}
1147 for line in self._full_description.splitlines():
1148 m = self.TAG_LINE_RE.match(line)
1149 if m:
1150 self.tags[m.group('key')] = m.group('value')
1151 else:
1152 description_without_tags.append(line)
1153
1154 # Change back to text and remove whitespace at end.
1155 self._description_without_tags = (
1156 '\n'.join(description_without_tags).rstrip())
1157
Edward Lemur69bb8be2020-02-03 20:37:38 +00001158 def AddDescriptionFooter(self, key, value):
1159 """Adds the given footer to the change description.
1160
1161 Args:
1162 key: A string with the key for the git footer. It must conform to
1163 the git footers format (i.e. 'List-Of-Tokens') and will be case
1164 normalized so that each token is title-cased.
1165 value: A string with the value for the git footer.
1166 """
1167 description = git_footers.add_footer(
1168 self.FullDescriptionText(), git_footers.normalize_name(key), value)
1169 self.SetDescriptionText(description)
1170
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001171 def RepositoryRoot(self):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001172 """Returns the repository (checkout) root directory for this change,
1173 as an absolute path.
1174 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001175 return self._local_root
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001176
1177 def __getattr__(self, attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001178 """Return tags directly as attributes on the object."""
Edward Lemura5799e32020-01-17 19:26:51 +00001179 if not re.match(r'^[A-Z_]*$', attr):
maruel@chromium.org92022ec2009-06-11 01:59:28 +00001180 raise AttributeError(self, attr)
maruel@chromium.orge1a524f2009-05-27 14:43:46 +00001181 return self.tags.get(attr)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001182
Edward Lemur69bb8be2020-02-03 20:37:38 +00001183 def GitFootersFromDescription(self):
1184 """Return the git footers present in the description.
1185
1186 Returns:
1187 footers: A dict of {footer: [values]} containing a multimap of the footers
1188 in the change description.
1189 """
1190 return git_footers.parse_footers(self.FullDescriptionText())
1191
Aaron Gablefc03e672017-05-15 14:09:42 -07001192 def BugsFromDescription(self):
1193 """Returns all bugs referenced in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001194 tags = [b.strip() for b in self.tags.get('BUG', '').split(',') if b.strip()]
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001195 footers = []
Edward Lemur69bb8be2020-02-03 20:37:38 +00001196 parsed = self.GitFootersFromDescription()
Dan Beam62954042019-10-03 21:20:33 +00001197 unsplit_footers = parsed.get('Bug', []) + parsed.get('Fixed', [])
Caleb Rouleauc0546b92019-02-22 06:12:57 +00001198 for unsplit_footer in unsplit_footers:
1199 footers += [b.strip() for b in unsplit_footer.split(',')]
Aaron Gable12ef5012017-05-15 14:29:00 -07001200 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001201
1202 def ReviewersFromDescription(self):
1203 """Returns all reviewers listed in the commit description."""
Edward Lemura5799e32020-01-17 19:26:51 +00001204 # We don't support a 'R:' git-footer for reviewers; that is in metadata.
Aaron Gable12ef5012017-05-15 14:29:00 -07001205 tags = [r.strip() for r in self.tags.get('R', '').split(',') if r.strip()]
1206 return sorted(set(tags))
Aaron Gablefc03e672017-05-15 14:09:42 -07001207
1208 def TBRsFromDescription(self):
1209 """Returns all TBR reviewers listed in the commit description."""
Aaron Gable12ef5012017-05-15 14:29:00 -07001210 tags = [r.strip() for r in self.tags.get('TBR', '').split(',') if r.strip()]
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001211 # TODO(crbug.com/839208): Remove support for 'Tbr:' when TBRs are
1212 # programmatically determined by self-CR+1s.
Edward Lemur69bb8be2020-02-03 20:37:38 +00001213 footers = self.GitFootersFromDescription().get('Tbr', [])
Aaron Gable12ef5012017-05-15 14:29:00 -07001214 return sorted(set(tags + footers))
Aaron Gablefc03e672017-05-15 14:09:42 -07001215
Aaron Gable6e7ddb62020-05-27 22:23:29 +00001216 # TODO(crbug.com/753425): Delete these once we're sure they're unused.
Aaron Gablefc03e672017-05-15 14:09:42 -07001217 @property
1218 def BUG(self):
1219 return ','.join(self.BugsFromDescription())
1220 @property
1221 def R(self):
1222 return ','.join(self.ReviewersFromDescription())
1223 @property
1224 def TBR(self):
1225 return ','.join(self.TBRsFromDescription())
1226
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001227 def AllFiles(self, root=None):
1228 """List all files under source control in the repo."""
1229 raise NotImplementedError()
1230
agable0b65e732016-11-22 09:25:46 -08001231 def AffectedFiles(self, include_deletes=True, file_filter=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001232 """Returns a list of AffectedFile instances for all files in the change.
1233
1234 Args:
1235 include_deletes: If false, deleted files will be filtered out.
sail@chromium.org5538e022011-05-12 17:53:16 +00001236 file_filter: An additional filter to apply.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001237
1238 Returns:
1239 [AffectedFile(path, action), AffectedFile(path, action)]
1240 """
Edward Lemurb9830242019-10-30 22:19:20 +00001241 affected = list(filter(file_filter, self._affected_files))
sail@chromium.org5538e022011-05-12 17:53:16 +00001242
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001243 if include_deletes:
1244 return affected
Edward Lemurb9830242019-10-30 22:19:20 +00001245 return list(filter(lambda x: x.Action() != 'D', affected))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001246
John Budorick16162372018-04-18 10:39:53 -07001247 def AffectedTestableFiles(self, include_deletes=None, **kwargs):
maruel@chromium.org77c4f0f2009-05-29 18:53:04 +00001248 """Return a list of the existing text files in a change."""
1249 if include_deletes is not None:
Edward Lemura5799e32020-01-17 19:26:51 +00001250 warn('AffectedTeestableFiles(include_deletes=%s)'
1251 ' is deprecated and ignored' % str(include_deletes),
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001252 category=DeprecationWarning,
1253 stacklevel=2)
Edward Lemurb9830242019-10-30 22:19:20 +00001254 return list(filter(
1255 lambda x: x.IsTestableFile(),
1256 self.AffectedFiles(include_deletes=False, **kwargs)))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001257
agable0b65e732016-11-22 09:25:46 -08001258 def AffectedTextFiles(self, include_deletes=None):
1259 """An alias to AffectedTestableFiles for backwards compatibility."""
1260 return self.AffectedTestableFiles(include_deletes=include_deletes)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001261
agable0b65e732016-11-22 09:25:46 -08001262 def LocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001263 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001264 return [af.LocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001265
agable0b65e732016-11-22 09:25:46 -08001266 def AbsoluteLocalPaths(self):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001267 """Convenience function."""
agable0b65e732016-11-22 09:25:46 -08001268 return [af.AbsoluteLocalPath() for af in self.AffectedFiles()]
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001269
1270 def RightHandSideLines(self):
Edward Lemura5799e32020-01-17 19:26:51 +00001271 """An iterator over all text lines in 'new' version of changed files.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001272
1273 Lists lines from new or modified text files in the change.
1274
1275 This is useful for doing line-by-line regex checks, like checking for
1276 trailing whitespace.
1277
1278 Yields:
1279 a 3 tuple:
1280 the AffectedFile instance of the current file;
1281 integer line number (1-based); and
1282 the contents of the line as a string.
1283 """
maruel@chromium.orgcb2985f2010-11-03 14:08:31 +00001284 return _RightHandSideLinesImpl(
1285 x for x in self.AffectedFiles(include_deletes=False)
agable0b65e732016-11-22 09:25:46 -08001286 if x.IsTestableFile())
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001287
Jochen Eisingerd0573ec2017-04-13 10:55:06 +02001288 def OriginalOwnersFiles(self):
1289 """A map from path names of affected OWNERS files to their old content."""
1290 def owners_file_filter(f):
1291 return 'OWNERS' in os.path.split(f.LocalPath())[1]
1292 files = self.AffectedFiles(file_filter=owners_file_filter)
1293 return dict([(f.LocalPath(), f.OldContents()) for f in files])
1294
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001295
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001296class GitChange(Change):
1297 _AFFECTED_FILES = GitAffectedFile
maruel@chromium.orgc1938752011-04-12 23:11:13 +00001298 scm = 'git'
thestig@chromium.orgda8cddd2009-08-13 00:25:55 +00001299
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001300 def AllFiles(self, root=None):
1301 """List all files under source control in the repo."""
1302 root = root or self.RepositoryRoot()
1303 return subprocess.check_output(
Aaron Gable7817f022017-12-12 09:43:17 -08001304 ['git', '-c', 'core.quotePath=false', 'ls-files', '--', '.'],
1305 cwd=root).splitlines()
agable@chromium.org40a3d0b2014-05-15 01:59:16 +00001306
maruel@chromium.orgc70a2202009-06-17 12:55:10 +00001307
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001308def ListRelevantPresubmitFiles(files, root):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001309 """Finds all presubmit files that apply to a given set of source files.
1310
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001311 If inherit-review-settings-ok is present right under root, looks for
1312 PRESUBMIT.py in directories enclosing root.
1313
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001314 Args:
1315 files: An iterable container containing file paths.
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001316 root: Path where to stop searching.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001317
1318 Return:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001319 List of absolute paths of the existing PRESUBMIT.py scripts.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001320 """
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001321 files = [normpath(os.path.join(root, f)) for f in files]
1322
1323 # List all the individual directories containing files.
1324 directories = set([os.path.dirname(f) for f in files])
1325
1326 # Ignore root if inherit-review-settings-ok is present.
1327 if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
1328 root = None
1329
1330 # Collect all unique directories that may contain PRESUBMIT.py.
1331 candidates = set()
1332 for directory in directories:
1333 while True:
1334 if directory in candidates:
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001335 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001336 candidates.add(directory)
1337 if directory == root:
maruel@chromium.org4661e0c2009-06-04 00:45:26 +00001338 break
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001339 parent_dir = os.path.dirname(directory)
1340 if parent_dir == directory:
1341 # We hit the system root directory.
1342 break
1343 directory = parent_dir
1344
1345 # Look for PRESUBMIT.py in all candidate directories.
1346 results = []
1347 for directory in sorted(list(candidates)):
tobiasjsff061c02016-08-17 03:23:57 -07001348 try:
1349 for f in os.listdir(directory):
1350 p = os.path.join(directory, f)
1351 if os.path.isfile(p) and re.match(
1352 r'PRESUBMIT.*\.py$', f) and not f.startswith('PRESUBMIT_test'):
1353 results.append(p)
1354 except OSError:
1355 pass
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001356
tobiasjs2836bcf2016-08-16 04:08:16 -07001357 logging.debug('Presubmit files: %s', ','.join(results))
maruel@chromium.orgb1901a62010-06-16 00:18:47 +00001358 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001359
1360
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001361class GetTryMastersExecuter(object):
1362 @staticmethod
1363 def ExecPresubmitScript(script_text, presubmit_path, project, change):
1364 """Executes GetPreferredTryMasters() from a single presubmit script.
1365
1366 Args:
1367 script_text: The text of the presubmit script.
1368 presubmit_path: Project script to run.
1369 project: Project name to pass to presubmit script for bot selection.
1370
1371 Return:
1372 A map of try masters to map of builders to set of tests.
1373 """
1374 context = {}
1375 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001376 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1377 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001378 except Exception as e:
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001379 raise PresubmitFailure('"%s" had an exception.\n%s'
1380 % (presubmit_path, e))
1381
1382 function_name = 'GetPreferredTryMasters'
1383 if function_name not in context:
1384 return {}
1385 get_preferred_try_masters = context[function_name]
1386 if not len(inspect.getargspec(get_preferred_try_masters)[0]) == 2:
1387 raise PresubmitFailure(
1388 'Expected function "GetPreferredTryMasters" to take two arguments.')
1389 return get_preferred_try_masters(project, change)
1390
1391
rmistry@google.com5626a922015-02-26 14:03:30 +00001392class GetPostUploadExecuter(object):
1393 @staticmethod
Edward Lemur016a0872020-02-04 22:13:28 +00001394 def ExecPresubmitScript(script_text, presubmit_path, gerrit_obj, change):
rmistry@google.com5626a922015-02-26 14:03:30 +00001395 """Executes PostUploadHook() from a single presubmit script.
1396
1397 Args:
1398 script_text: The text of the presubmit script.
1399 presubmit_path: Project script to run.
Edward Lemur016a0872020-02-04 22:13:28 +00001400 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001401 change: The Change object.
1402
1403 Return:
1404 A list of results objects.
1405 """
1406 context = {}
1407 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001408 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1409 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001410 except Exception as e:
rmistry@google.com5626a922015-02-26 14:03:30 +00001411 raise PresubmitFailure('"%s" had an exception.\n%s'
1412 % (presubmit_path, e))
1413
1414 function_name = 'PostUploadHook'
1415 if function_name not in context:
1416 return {}
1417 post_upload_hook = context[function_name]
1418 if not len(inspect.getargspec(post_upload_hook)[0]) == 3:
1419 raise PresubmitFailure(
1420 'Expected function "PostUploadHook" to take three arguments.')
Edward Lemur016a0872020-02-04 22:13:28 +00001421 return post_upload_hook(gerrit_obj, change, OutputApi(False))
rmistry@google.com5626a922015-02-26 14:03:30 +00001422
1423
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001424def _MergeMasters(masters1, masters2):
1425 """Merges two master maps. Merges also the tests of each builder."""
1426 result = {}
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001427 for (master, builders) in itertools.chain(masters1.items(),
1428 masters2.items()):
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001429 new_builders = result.setdefault(master, {})
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001430 for (builder, tests) in builders.items():
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001431 new_builders.setdefault(builder, set([])).update(tests)
1432 return result
1433
1434
1435def DoGetTryMasters(change,
1436 changed_files,
1437 repository_root,
1438 default_presubmit,
1439 project,
1440 verbose,
1441 output_stream):
1442 """Get the list of try masters from the presubmit scripts.
1443
1444 Args:
1445 changed_files: List of modified files.
1446 repository_root: The repository root.
1447 default_presubmit: A default presubmit script to execute in any case.
1448 project: Optional name of a project used in selecting trybots.
1449 verbose: Prints debug info.
1450 output_stream: A stream to write debug output to.
1451
1452 Return:
1453 Map of try masters to map of builders to set of tests.
1454 """
1455 presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
1456 if not presubmit_files and verbose:
Edward Lemura5799e32020-01-17 19:26:51 +00001457 output_stream.write('Warning, no PRESUBMIT.py found.\n')
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001458 results = {}
1459 executer = GetTryMastersExecuter()
1460
1461 if default_presubmit:
1462 if verbose:
Edward Lemura5799e32020-01-17 19:26:51 +00001463 output_stream.write('Running default presubmit script.\n')
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001464 fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
1465 results = _MergeMasters(results, executer.ExecPresubmitScript(
1466 default_presubmit, fake_path, project, change))
1467 for filename in presubmit_files:
1468 filename = os.path.abspath(filename)
1469 if verbose:
Edward Lemura5799e32020-01-17 19:26:51 +00001470 output_stream.write('Running %s\n' % filename)
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001471 # Accept CRLF presubmit script.
1472 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1473 results = _MergeMasters(results, executer.ExecPresubmitScript(
1474 presubmit_script, filename, project, change))
1475
1476 # Make sets to lists again for later JSON serialization.
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001477 for builders in results.values():
machenbach@chromium.org58a69cb2014-03-01 02:08:29 +00001478 for builder in builders:
1479 builders[builder] = list(builders[builder])
1480
1481 if results and verbose:
1482 output_stream.write('%s\n' % str(results))
1483 return results
1484
1485
rmistry@google.com5626a922015-02-26 14:03:30 +00001486def DoPostUploadExecuter(change,
Edward Lemur016a0872020-02-04 22:13:28 +00001487 gerrit_obj,
Edward Lemur6eb1d322020-02-27 22:20:15 +00001488 verbose):
rmistry@google.com5626a922015-02-26 14:03:30 +00001489 """Execute the post upload hook.
1490
1491 Args:
1492 change: The Change object.
Edward Lemur016a0872020-02-04 22:13:28 +00001493 gerrit_obj: The GerritAccessor object.
rmistry@google.com5626a922015-02-26 14:03:30 +00001494 verbose: Prints debug info.
rmistry@google.com5626a922015-02-26 14:03:30 +00001495 """
1496 presubmit_files = ListRelevantPresubmitFiles(
Edward Lemur6eb1d322020-02-27 22:20:15 +00001497 change.LocalPaths(), change.RepositoryRoot())
rmistry@google.com5626a922015-02-26 14:03:30 +00001498 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001499 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
rmistry@google.com5626a922015-02-26 14:03:30 +00001500 results = []
1501 executer = GetPostUploadExecuter()
1502 # The root presubmit file should be executed after the ones in subdirectories.
1503 # i.e. the specific post upload hooks should run before the general ones.
1504 # Thus, reverse the order provided by ListRelevantPresubmitFiles.
1505 presubmit_files.reverse()
1506
1507 for filename in presubmit_files:
1508 filename = os.path.abspath(filename)
1509 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001510 sys.stdout.write('Running %s\n' % filename)
rmistry@google.com5626a922015-02-26 14:03:30 +00001511 # Accept CRLF presubmit script.
1512 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1513 results.extend(executer.ExecPresubmitScript(
Edward Lemur016a0872020-02-04 22:13:28 +00001514 presubmit_script, filename, gerrit_obj, change))
rmistry@google.com5626a922015-02-26 14:03:30 +00001515
Edward Lemur6eb1d322020-02-27 22:20:15 +00001516 if not results:
1517 return 0
1518
1519 sys.stdout.write('\n')
1520 sys.stdout.write('** Post Upload Hook Messages **\n')
1521
1522 exit_code = 0
1523 for result in results:
1524 if result.fatal:
1525 exit_code = 1
1526 result.handle()
1527 sys.stdout.write('\n')
1528
1529 return exit_code
rmistry@google.com5626a922015-02-26 14:03:30 +00001530
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001531class PresubmitExecuter(object):
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001532 def __init__(self, change, committing, verbose, gerrit_obj, dry_run=None,
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001533 thread_pool=None, parallel=False):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001534 """
1535 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001536 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001537 committing: True if 'git cl land' is running, False if 'git cl upload' is.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001538 gerrit_obj: provides basic Gerrit codereview functionality.
1539 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001540 parallel: if true, all tests reported via input_api.RunTests for all
1541 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001542 """
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001543 self.change = change
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001544 self.committing = committing
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001545 self.gerrit = gerrit_obj
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001546 self.verbose = verbose
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001547 self.dry_run = dry_run
Daniel Cheng7227d212017-11-17 08:12:37 -08001548 self.more_cc = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001549 self.thread_pool = thread_pool
1550 self.parallel = parallel
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001551
1552 def ExecPresubmitScript(self, script_text, presubmit_path):
1553 """Executes a single presubmit script.
1554
1555 Args:
1556 script_text: The text of the presubmit script.
1557 presubmit_path: The path to the presubmit file (this will be reported via
1558 input_api.PresubmitLocalPath()).
1559
1560 Return:
1561 A list of result objects, empty if no problems.
1562 """
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001563
chase@chromium.org8e416c82009-10-06 04:30:44 +00001564 # Change to the presubmit file's directory to support local imports.
1565 main_path = os.getcwd()
Saagar Sanghavi98b332f2020-07-31 17:19:15 +00001566 presubmit_dir = os.path.dirname(presubmit_path)
1567 os.chdir(presubmit_dir)
chase@chromium.org8e416c82009-10-06 04:30:44 +00001568
1569 # Load the presubmit script into context.
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001570 input_api = InputApi(self.change, presubmit_path, self.committing,
Aaron Gable668c1d82018-04-03 10:19:16 -07001571 self.verbose, gerrit_obj=self.gerrit,
Edward Lesmes8e282792018-04-03 18:50:29 -04001572 dry_run=self.dry_run, thread_pool=self.thread_pool,
1573 parallel=self.parallel)
Daniel Cheng7227d212017-11-17 08:12:37 -08001574 output_api = OutputApi(self.committing)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001575 context = {}
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001576
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001577 # Try to figure out whether these presubmit checks should be run under
1578 # python2 or python3. We need to do this without actually trying to
1579 # compile the text, since the text might compile in one but not the
1580 # other.
1581 m = re.search('^USE_PYTHON3 = True$', script_text, flags=re.MULTILINE)
1582 use_python3 = m is not None
1583 if (((sys.version_info.major == 2) and use_python3) or
1584 ((sys.version_info.major == 3) and not use_python3)):
1585 return []
1586
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001587 try:
Raul Tambre09e64b42019-05-14 01:57:22 +00001588 exec(compile(script_text, 'PRESUBMIT.py', 'exec', dont_inherit=True),
1589 context)
Raul Tambre7c938462019-05-24 16:35:35 +00001590 except Exception as e:
maruel@chromium.org899e1c12011-04-07 17:03:18 +00001591 raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001592
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001593 context['__args'] = (input_api, output_api)
Ben Pastene8351dc12020-08-06 05:01:35 +00001594
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001595 # Get path of presubmit directory relative to repository root.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001596 # Always use forward slashes, so that path is same in *nix and Windows
1597 root = input_api.change.RepositoryRoot()
1598 rel_path = os.path.relpath(presubmit_dir, root)
1599 rel_path = rel_path.replace(os.path.sep, '/')
Ben Pastene8351dc12020-08-06 05:01:35 +00001600
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001601 # Get the URL of git remote origin and use it to identify host and project
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001602 host = project = ''
1603 if self.gerrit:
1604 host = self.gerrit.host or ''
1605 project = self.gerrit.project or ''
Saagar Sanghavi531d9922020-08-10 20:14:01 +00001606
1607 # Prefix for test names
1608 prefix = 'presubmit:%s/%s:%s/' % (host, project, rel_path)
1609
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001610 # Perform all the desired presubmit checks.
1611 results = []
Ben Pastene8351dc12020-08-06 05:01:35 +00001612
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001613 try:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001614 version = [
1615 int(x) for x in context.get('PRESUBMIT_VERSION', '0.0.0').split('.')
1616 ]
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001617
Scott Leecc2fe9b2020-11-19 19:38:06 +00001618 with rdb_wrapper.client(prefix) as sink:
1619 if version >= [2, 0, 0]:
1620 for function_name in context:
1621 if not function_name.startswith('Check'):
1622 continue
1623 if function_name.endswith('Commit') and not self.committing:
1624 continue
1625 if function_name.endswith('Upload') and self.committing:
1626 continue
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001627 logging.debug('Running %s in %s', function_name, presubmit_path)
1628 results.extend(
Scott Leecc2fe9b2020-11-19 19:38:06 +00001629 self._run_check_function(function_name, context, sink))
1630 logging.debug('Running %s done.', function_name)
1631 self.more_cc.extend(output_api.more_cc)
1632
1633 else: # Old format
1634 if self.committing:
1635 function_name = 'CheckChangeOnCommit'
1636 else:
1637 function_name = 'CheckChangeOnUpload'
1638 if function_name in context:
1639 logging.debug('Running %s in %s', function_name, presubmit_path)
1640 results.extend(
1641 self._run_check_function(function_name, context, sink))
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001642 logging.debug('Running %s done.', function_name)
1643 self.more_cc.extend(output_api.more_cc)
1644
1645 finally:
1646 for f in input_api._named_temporary_files:
1647 os.remove(f)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001648
chase@chromium.org8e416c82009-10-06 04:30:44 +00001649 # Return the process to the original working directory.
1650 os.chdir(main_path)
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001651 return results
1652
Scott Leecc2fe9b2020-11-19 19:38:06 +00001653 def _run_check_function(self, function_name, context, sink=None):
1654 """Evaluates and returns the result of a given presubmit function.
1655
1656 If sink is given, the result of the presubmit function will be reported
1657 to the ResultSink.
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001658
1659 Args:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001660 function_name: the name of the presubmit function to evaluate
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001661 context: a context dictionary in which the function will be evaluated
Scott Leecc2fe9b2020-11-19 19:38:06 +00001662 sink: an instance of ResultSink. None, by default.
1663 Returns:
1664 the result of the presubmit function call.
1665 """
1666 start_time = time_time()
1667 try:
Saagar Sanghavi03b15132020-08-10 16:43:41 +00001668 result = eval(function_name + '(*__args)', context)
1669 self._check_result_type(result)
Allen Webbfe7d7092021-05-18 02:05:49 +00001670 except Exception:
Scott Leecc2fe9b2020-11-19 19:38:06 +00001671 if sink:
1672 elapsed_time = time_time() - start_time
1673 sink.report(function_name, rdb_wrapper.STATUS_FAIL, elapsed_time)
Allen Webbfe7d7092021-05-18 02:05:49 +00001674 # TODO(crbug.com/953884): replace reraise with native py3:
Allen Webbb65bbfe2021-05-11 21:22:01 +00001675 # raise .. from e
Allen Webbfe7d7092021-05-18 02:05:49 +00001676 e_type, e_value, e_tb = sys.exc_info()
1677 six.reraise(e_type, 'Evaluation of %s failed: %s' % (function_name,
1678 e_value),
1679 e_tb)
Scott Leecc2fe9b2020-11-19 19:38:06 +00001680
1681 if sink:
1682 elapsed_time = time_time() - start_time
1683 status = rdb_wrapper.STATUS_PASS
1684 if any(r.fatal for r in result):
1685 status = rdb_wrapper.STATUS_FAIL
1686 sink.report(function_name, status, elapsed_time)
1687
1688 return result
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001689
Saagar Sanghavi9949ab72020-07-20 20:56:40 +00001690 def _check_result_type(self, result):
1691 """Helper function which ensures result is a list, and all elements are
1692 instances of OutputApi.PresubmitResult"""
1693 if not isinstance(result, (tuple, list)):
1694 raise PresubmitFailure('Presubmit functions must return a tuple or list')
1695 if not all(isinstance(res, OutputApi.PresubmitResult) for res in result):
1696 raise PresubmitFailure(
1697 'All presubmit results must be of types derived from '
1698 'output_api.PresubmitResult')
1699
1700
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001701def DoPresubmitChecks(change,
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001702 committing,
1703 verbose,
maruel@chromium.orgb0dfd352009-06-10 14:12:54 +00001704 default_presubmit,
dpranke@chromium.org970c5222011-03-12 00:32:24 +00001705 may_prompt,
Aaron Gable668c1d82018-04-03 10:19:16 -07001706 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04001707 dry_run=None,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00001708 parallel=False,
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001709 json_output=None):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001710 """Runs all presubmit checks that apply to the files in the change.
1711
1712 This finds all PRESUBMIT.py files in directories enclosing the files in the
1713 change (up to the repository root) and calls the relevant entrypoint function
1714 depending on whether the change is being committed or uploaded.
1715
1716 Prints errors, warnings and notifications. Prompts the user for warnings
1717 when needed.
1718
1719 Args:
maruel@chromium.org4ff922a2009-06-12 20:20:19 +00001720 change: The Change object.
agable92bec4f2016-08-24 09:27:27 -07001721 committing: True if 'git cl land' is running, False if 'git cl upload' is.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001722 verbose: Prints debug info.
maruel@chromium.org0ff1fab2009-05-22 13:08:15 +00001723 default_presubmit: A default presubmit script to execute in any case.
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001724 may_prompt: Enable (y/n) questions on warning or error. If False,
1725 any questions are answered with yes by default.
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00001726 gerrit_obj: provides basic Gerrit codereview functionality.
tandrii@chromium.org57bafac2016-04-28 05:09:03 +00001727 dry_run: if true, some Checks will be skipped.
Edward Lesmes8e282792018-04-03 18:50:29 -04001728 parallel: if true, all tests specified by input_api.RunTests in all
1729 PRESUBMIT files will be run in parallel.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001730
1731 Return:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001732 1 if presubmit checks failed or 0 otherwise.
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001733 """
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001734 old_environ = os.environ
1735 try:
1736 # Make sure python subprocesses won't generate .pyc files.
1737 os.environ = os.environ.copy()
Edward Lemurb9830242019-10-30 22:19:20 +00001738 os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001739
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001740 python_version = 'Python %s' % sys.version_info.major
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001741 if committing:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001742 sys.stdout.write('Running %s presubmit commit checks ...\n' %
1743 python_version)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001744 else:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001745 sys.stdout.write('Running %s presubmit upload checks ...\n' %
1746 python_version)
Edward Lemurecc27072020-01-06 16:42:34 +00001747 start_time = time_time()
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001748 presubmit_files = ListRelevantPresubmitFiles(
agable0b65e732016-11-22 09:25:46 -08001749 change.AbsoluteLocalPaths(), change.RepositoryRoot())
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001750 if not presubmit_files and verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001751 sys.stdout.write('Warning, no PRESUBMIT.py found.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001752 results = []
Edward Lesmes8e282792018-04-03 18:50:29 -04001753 thread_pool = ThreadPool()
Edward Lemur7e3c67f2018-07-20 20:52:49 +00001754 executer = PresubmitExecuter(change, committing, verbose, gerrit_obj,
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001755 dry_run, thread_pool, parallel)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001756 if default_presubmit:
1757 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001758 sys.stdout.write('Running default presubmit script.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001759 fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
1760 results += executer.ExecPresubmitScript(default_presubmit, fake_path)
1761 for filename in presubmit_files:
1762 filename = os.path.abspath(filename)
1763 if verbose:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001764 sys.stdout.write('Running %s\n' % filename)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001765 # Accept CRLF presubmit script.
1766 presubmit_script = gclient_utils.FileRead(filename, 'rU')
1767 results += executer.ExecPresubmitScript(presubmit_script, filename)
Edward Lesmes8e282792018-04-03 18:50:29 -04001768 results += thread_pool.RunAsync()
1769
Edward Lemur6eb1d322020-02-27 22:20:15 +00001770 messages = {}
1771 should_prompt = False
1772 presubmits_failed = False
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001773 for result in results:
1774 if result.fatal:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001775 presubmits_failed = True
1776 messages.setdefault('ERRORS', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001777 elif result.should_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001778 should_prompt = True
1779 messages.setdefault('Warnings', []).append(result)
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001780 else:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001781 messages.setdefault('Messages', []).append(result)
pam@chromium.orged9a0832009-09-09 22:48:55 +00001782
Edward Lemur6eb1d322020-02-27 22:20:15 +00001783 sys.stdout.write('\n')
1784 for name, items in messages.items():
1785 sys.stdout.write('** Presubmit %s **\n' % name)
1786 for item in items:
1787 item.handle()
1788 sys.stdout.write('\n')
pam@chromium.orged9a0832009-09-09 22:48:55 +00001789
Edward Lemurecc27072020-01-06 16:42:34 +00001790 total_time = time_time() - start_time
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001791 if total_time > 1.0:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001792 sys.stdout.write(
1793 'Presubmit checks took %.1fs to calculate.\n\n' % total_time)
maruel@chromium.orgce8e46b2009-06-26 22:31:51 +00001794
Edward Lemur6eb1d322020-02-27 22:20:15 +00001795 if not should_prompt and not presubmits_failed:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001796 sys.stdout.write('%s presubmit checks passed.\n' % python_version)
Edward Lemur6eb1d322020-02-27 22:20:15 +00001797 elif should_prompt:
Dirk Pranke61bf6e82021-04-23 00:50:21 +00001798 sys.stdout.write('There were %s presubmit warnings. ' % python_version)
Quinten Yearsley516fe7f2016-12-14 11:50:18 -08001799 if may_prompt:
Edward Lemur6eb1d322020-02-27 22:20:15 +00001800 presubmits_failed = not prompt_should_continue(
1801 'Are you sure you wish to continue? (y/N): ')
Garrett Beatyacb807e2020-08-28 18:17:24 +00001802 else:
1803 sys.stdout.write('\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001804
Edward Lemur1dc66e12020-02-21 21:36:34 +00001805 if json_output:
1806 # Write the presubmit results to json output
1807 presubmit_results = {
1808 'errors': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001809 error.json_format()
1810 for error in messages.get('ERRORS', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001811 ],
1812 'notifications': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001813 notification.json_format()
1814 for notification in messages.get('Messages', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001815 ],
1816 'warnings': [
Edward Lemur6eb1d322020-02-27 22:20:15 +00001817 warning.json_format()
1818 for warning in messages.get('Warnings', [])
Edward Lemur1dc66e12020-02-21 21:36:34 +00001819 ],
Edward Lemur6eb1d322020-02-27 22:20:15 +00001820 'more_cc': executer.more_cc,
Edward Lemur1dc66e12020-02-21 21:36:34 +00001821 }
1822
1823 gclient_utils.FileWrite(
1824 json_output, json.dumps(presubmit_results, sort_keys=True))
1825
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001826 global _ASKED_FOR_FEEDBACK
1827 # Ask for feedback one time out of 5.
1828 if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
Edward Lemur6eb1d322020-02-27 22:20:15 +00001829 sys.stdout.write(
maruel@chromium.org1ce8e662014-01-14 15:23:00 +00001830 'Was the presubmit check useful? If not, run "git cl presubmit -v"\n'
1831 'to figure out which PRESUBMIT.py was run, then run git blame\n'
1832 'on the file to figure out who to ask for help.\n')
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001833 _ASKED_FOR_FEEDBACK = True
Edward Lemur6eb1d322020-02-27 22:20:15 +00001834
1835 return 1 if presubmits_failed else 0
maruel@chromium.orgea7c8552011-04-18 14:12:07 +00001836 finally:
1837 os.environ = old_environ
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001838
1839
Edward Lemur50984a62020-02-06 18:10:18 +00001840def _scan_sub_dirs(mask, recursive):
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001841 if not recursive:
pgervais@chromium.orge57b09d2014-05-07 00:58:13 +00001842 return [x for x in glob.glob(mask) if x not in ('.svn', '.git')]
Lei Zhang9611c4c2017-04-04 01:41:56 -07001843
1844 results = []
1845 for root, dirs, files in os.walk('.'):
1846 if '.svn' in dirs:
1847 dirs.remove('.svn')
1848 if '.git' in dirs:
1849 dirs.remove('.git')
1850 for name in files:
1851 if fnmatch.fnmatch(name, mask):
1852 results.append(os.path.join(root, name))
1853 return results
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001854
1855
Edward Lemur50984a62020-02-06 18:10:18 +00001856def _parse_files(args, recursive):
tobiasjs2836bcf2016-08-16 04:08:16 -07001857 logging.debug('Searching for %s', args)
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001858 files = []
1859 for arg in args:
Edward Lemur50984a62020-02-06 18:10:18 +00001860 files.extend([('M', f) for f in _scan_sub_dirs(arg, recursive)])
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00001861 return files
1862
1863
Edward Lemur50984a62020-02-06 18:10:18 +00001864def _parse_change(parser, options):
1865 """Process change options.
1866
1867 Args:
1868 parser: The parser used to parse the arguments from command line.
1869 options: The arguments parsed from command line.
1870 Returns:
1871 A GitChange if the change root is a git repository, or a Change otherwise.
1872 """
1873 if options.files and options.all_files:
1874 parser.error('<files> cannot be specified when --all-files is set.')
1875
Edward Lesmes44ea3ff2020-02-05 00:14:30 +00001876 change_scm = scm.determine_scm(options.root)
Edward Lemur50984a62020-02-06 18:10:18 +00001877 if change_scm != 'git' and not options.files:
1878 parser.error('<files> is not optional for unversioned directories.')
1879
1880 if options.files:
1881 change_files = _parse_files(options.files, options.recursive)
1882 elif options.all_files:
1883 change_files = [('M', f) for f in scm.GIT.GetAllFiles(options.root)]
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001884 else:
Edward Lemur50984a62020-02-06 18:10:18 +00001885 change_files = scm.GIT.CaptureStatus(
Edward Lemur7f6dec02020-02-06 20:23:58 +00001886 options.root, options.upstream or None)
Edward Lemur50984a62020-02-06 18:10:18 +00001887
1888 logging.info('Found %d file(s).', len(change_files))
1889
1890 change_class = GitChange if change_scm == 'git' else Change
1891 return change_class(
1892 options.name,
1893 options.description,
1894 options.root,
1895 change_files,
1896 options.issue,
1897 options.patchset,
1898 options.author,
1899 upstream=options.upstream)
1900
1901
1902def _parse_gerrit_options(parser, options):
1903 """Process gerrit options.
1904
1905 SIDE EFFECTS: Modifies options.author and options.description from Gerrit if
1906 options.gerrit_fetch is set.
1907
1908 Args:
1909 parser: The parser used to parse the arguments from command line.
1910 options: The arguments parsed from command line.
1911 Returns:
Stephen Martinisfb09de22021-02-25 03:41:13 +00001912 A GerritAccessor object if options.gerrit_url is set, or None otherwise.
Edward Lemur50984a62020-02-06 18:10:18 +00001913 """
1914 gerrit_obj = None
Stephen Martinisfb09de22021-02-25 03:41:13 +00001915 if options.gerrit_url:
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001916 gerrit_obj = GerritAccessor(
1917 url=options.gerrit_url,
1918 project=options.gerrit_project,
1919 branch=options.gerrit_branch)
Edward Lemur50984a62020-02-06 18:10:18 +00001920
1921 if not options.gerrit_fetch:
1922 return gerrit_obj
1923
1924 if not options.gerrit_url or not options.issue or not options.patchset:
1925 parser.error(
1926 '--gerrit_fetch requires --gerrit_url, --issue and --patchset.')
1927
1928 options.author = gerrit_obj.GetChangeOwner(options.issue)
1929 options.description = gerrit_obj.GetChangeDescription(
1930 options.issue, options.patchset)
1931
1932 logging.info('Got author: "%s"', options.author)
1933 logging.info('Got description: """\n%s\n"""', options.description)
1934
1935 return gerrit_obj
maruel@chromium.org5c8c6de2011-03-18 16:20:18 +00001936
1937
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001938@contextlib.contextmanager
1939def canned_check_filter(method_names):
1940 filtered = {}
1941 try:
1942 for method_name in method_names:
1943 if not hasattr(presubmit_canned_checks, method_name):
Gavin Make6a62332020-12-04 21:57:10 +00001944 logging.warning('Skipping unknown "canned" check %s' % method_name)
Aaron Gableecee74c2018-04-02 15:13:08 -07001945 continue
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001946 filtered[method_name] = getattr(presubmit_canned_checks, method_name)
1947 setattr(presubmit_canned_checks, method_name, lambda *_a, **_kw: [])
1948 yield
1949 finally:
Marc-Antoine Ruel8e57b4b2019-10-11 01:01:36 +00001950 for name, method in filtered.items():
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00001951 setattr(presubmit_canned_checks, name, method)
1952
maruel@chromium.orgffeb2f32013-12-03 13:55:22 +00001953
sbc@chromium.org013731e2015-02-26 18:28:43 +00001954def main(argv=None):
Edward Lemura5799e32020-01-17 19:26:51 +00001955 parser = argparse.ArgumentParser(usage='%(prog)s [options] <files...>')
1956 hooks = parser.add_mutually_exclusive_group()
1957 hooks.add_argument('-c', '--commit', action='store_true',
1958 help='Use commit instead of upload checks.')
1959 hooks.add_argument('-u', '--upload', action='store_false', dest='commit',
1960 help='Use upload instead of commit checks.')
Edward Lemur75526302020-02-27 22:31:05 +00001961 hooks.add_argument('--post_upload', action='store_true',
1962 help='Run post-upload commit hooks.')
Edward Lemura5799e32020-01-17 19:26:51 +00001963 parser.add_argument('-r', '--recursive', action='store_true',
1964 help='Act recursively.')
1965 parser.add_argument('-v', '--verbose', action='count', default=0,
1966 help='Use 2 times for more debug info.')
1967 parser.add_argument('--name', default='no name')
1968 parser.add_argument('--author')
Edward Lemur227d5102020-02-25 23:45:35 +00001969 desc = parser.add_mutually_exclusive_group()
1970 desc.add_argument('--description', default='', help='The change description.')
1971 desc.add_argument('--description_file',
1972 help='File to read change description from.')
Edward Lemura5799e32020-01-17 19:26:51 +00001973 parser.add_argument('--issue', type=int, default=0)
1974 parser.add_argument('--patchset', type=int, default=0)
1975 parser.add_argument('--root', default=os.getcwd(),
1976 help='Search for PRESUBMIT.py up to this directory. '
1977 'If inherit-review-settings-ok is present in this '
1978 'directory, parent directories up to the root file '
1979 'system directories will also be searched.')
1980 parser.add_argument('--upstream',
1981 help='Git only: the base ref or upstream branch against '
1982 'which the diff should be computed.')
1983 parser.add_argument('--default_presubmit')
1984 parser.add_argument('--may_prompt', action='store_true', default=False)
1985 parser.add_argument('--skip_canned', action='append', default=[],
1986 help='A list of checks to skip which appear in '
1987 'presubmit_canned_checks. Can be provided multiple times '
1988 'to skip multiple canned checks.')
1989 parser.add_argument('--dry_run', action='store_true', help=argparse.SUPPRESS)
1990 parser.add_argument('--gerrit_url', help=argparse.SUPPRESS)
Edward Lesmeseb1bd622021-03-01 19:54:07 +00001991 parser.add_argument('--gerrit_project', help=argparse.SUPPRESS)
1992 parser.add_argument('--gerrit_branch', help=argparse.SUPPRESS)
Edward Lemura5799e32020-01-17 19:26:51 +00001993 parser.add_argument('--gerrit_fetch', action='store_true',
1994 help=argparse.SUPPRESS)
1995 parser.add_argument('--parallel', action='store_true',
1996 help='Run all tests specified by input_api.RunTests in '
1997 'all PRESUBMIT files in parallel.')
1998 parser.add_argument('--json_output',
1999 help='Write presubmit errors to json output.')
Edward Lemur1dc66e12020-02-21 21:36:34 +00002000 parser.add_argument('--all_files', action='store_true',
Edward Lemur50984a62020-02-06 18:10:18 +00002001 help='Mark all files under source control as modified.')
Edward Lemura5799e32020-01-17 19:26:51 +00002002 parser.add_argument('files', nargs='*',
2003 help='List of files to be marked as modified when '
2004 'executing presubmit or post-upload hooks. fnmatch '
2005 'wildcards can also be used.')
Edward Lemura5799e32020-01-17 19:26:51 +00002006 options = parser.parse_args(argv)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002007
Erik Staabcca5c492020-04-16 17:40:07 +00002008 log_level = logging.ERROR
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002009 if options.verbose >= 2:
Erik Staabcca5c492020-04-16 17:40:07 +00002010 log_level = logging.DEBUG
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002011 elif options.verbose:
Erik Staabcca5c492020-04-16 17:40:07 +00002012 log_level = logging.INFO
2013 log_format = ('[%(levelname).1s%(asctime)s %(process)d %(thread)d '
2014 '%(filename)s] %(message)s')
2015 logging.basicConfig(format=log_format, level=log_level)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002016
Edward Lemur227d5102020-02-25 23:45:35 +00002017 if options.description_file:
2018 options.description = gclient_utils.FileRead(options.description_file)
Edward Lemur50984a62020-02-06 18:10:18 +00002019 gerrit_obj = _parse_gerrit_options(parser, options)
2020 change = _parse_change(parser, options)
pgervais@chromium.org92c30092014-04-15 00:30:37 +00002021
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002022 try:
Edward Lemur75526302020-02-27 22:31:05 +00002023 if options.post_upload:
2024 return DoPostUploadExecuter(
2025 change,
2026 gerrit_obj,
2027 options.verbose)
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002028 with canned_check_filter(options.skip_canned):
Edward Lemur6eb1d322020-02-27 22:20:15 +00002029 return DoPresubmitChecks(
Edward Lemur50984a62020-02-06 18:10:18 +00002030 change,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002031 options.commit,
2032 options.verbose,
iannucci@chromium.org8a4a2bc2013-03-08 08:13:20 +00002033 options.default_presubmit,
2034 options.may_prompt,
tandrii@chromium.org37b07a72016-04-29 16:42:28 +00002035 gerrit_obj,
Edward Lesmes8e282792018-04-03 18:50:29 -04002036 options.dry_run,
Debrian Figueroadd2737e2019-06-21 23:50:13 +00002037 options.parallel,
Edward Lesmeseb1bd622021-03-01 19:54:07 +00002038 options.json_output)
Raul Tambre7c938462019-05-24 16:35:35 +00002039 except PresubmitFailure as e:
Raul Tambre80ee78e2019-05-06 22:41:05 +00002040 print(e, file=sys.stderr)
2041 print('Maybe your depot_tools is out of date?', file=sys.stderr)
maruel@chromium.org899e1c12011-04-07 17:03:18 +00002042 return 2
maruel@google.comfb2b8eb2009-04-23 21:03:42 +00002043
2044
2045if __name__ == '__main__':
maruel@chromium.org35625c72011-03-23 17:34:02 +00002046 fix_encoding.fix_encoding()
sbc@chromium.org013731e2015-02-26 18:28:43 +00002047 try:
2048 sys.exit(main())
2049 except KeyboardInterrupt:
2050 sys.stderr.write('interrupted\n')
sergiybf8a3b382016-07-05 11:21:30 -07002051 sys.exit(2)