blob: de65305f33a9bb203f797016d7d03ac3e566b9ef [file] [log] [blame]
Prathmesh Prabhuc5254652016-12-22 12:58:05 -08001#!/usr/bin/env python2
Mike Frysingerb7d552e2017-11-23 11:50:47 -05002# -*- coding: utf-8 -*-
Jon Salz98255932012-08-18 14:48:02 +08003# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07004# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
Mike Frysingerae409522014-02-01 03:16:11 -05007"""Presubmit checks to run when doing `repo upload`.
8
9You can add new checks by adding a functions to the HOOKS constants.
10"""
11
Mike Frysinger09d6a3d2013-10-08 22:21:03 -040012from __future__ import print_function
13
Filipe Brandenburger4b542b12015-10-09 12:46:31 -070014import argparse
Alex Deymo643ac4c2015-09-03 10:40:50 -070015import collections
Ryan Cui9b651632011-05-11 11:38:58 -070016import ConfigParser
Daniel Erate3ea3fc2015-02-13 15:27:52 -070017import fnmatch
Jon Salz3ee59de2012-08-18 13:54:22 +080018import functools
Dale Curtis2975c432011-05-03 17:25:20 -070019import json
Mandeep Singh Baines116ad102011-04-27 15:16:37 -070020import os
Ryan Cuiec4d6332011-05-02 14:15:25 -070021import re
Mandeep Singh Bainesa7ffa4b2011-05-03 11:37:02 -070022import sys
Peter Ammon811f6702014-06-12 15:45:38 -070023import stat
Aviv Keshet5ac59522017-01-31 14:28:27 -080024import StringIO
Mandeep Singh Baines116ad102011-04-27 15:16:37 -070025
Ryan Cui1562fb82011-05-09 11:01:31 -070026from errors import (VerifyException, HookFailure, PrintErrorForProject,
27 PrintErrorsForCommit)
Ryan Cuiec4d6332011-05-02 14:15:25 -070028
David Jamesc3b68b32013-04-03 09:17:03 -070029# If repo imports us, the __name__ will be __builtin__, and the wrapper will
30# be in $CHROMEOS_CHECKOUT/.repo/repo/main.py, so we need to go two directories
31# up. The same logic also happens to work if we're executed directly.
32if __name__ in ('__builtin__', '__main__'):
33 sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))
34
Mike Frysinger66142932014-12-18 14:55:57 -050035from chromite.lib import commandline
Aviv Keshet5ac59522017-01-31 14:28:27 -080036from chromite.lib import constants
Rahul Chaudhry0e515342015-08-07 12:00:43 -070037from chromite.lib import cros_build_lib
Mike Frysingerd3bd32c2014-11-24 23:34:29 -050038from chromite.lib import git
Daniel Erata350fd32014-09-29 14:02:34 -070039from chromite.lib import osutils
David Jamesc3b68b32013-04-03 09:17:03 -070040from chromite.lib import patch
Mike Frysinger2ec70ed2014-08-17 19:28:34 -040041from chromite.licensing import licenses_lib
David Jamesc3b68b32013-04-03 09:17:03 -070042
Vadim Bendebury2b62d742014-06-22 13:14:51 -070043PRE_SUBMIT = 'pre-submit'
Ryan Cuiec4d6332011-05-02 14:15:25 -070044
45COMMON_INCLUDED_PATHS = [
Mike Frysingerae409522014-02-01 03:16:11 -050046 # C++ and friends
47 r".*\.c$", r".*\.cc$", r".*\.cpp$", r".*\.h$", r".*\.m$", r".*\.mm$",
48 r".*\.inl$", r".*\.asm$", r".*\.hxx$", r".*\.hpp$", r".*\.s$", r".*\.S$",
49 # Scripts
50 r".*\.js$", r".*\.py$", r".*\.sh$", r".*\.rb$", r".*\.pl$", r".*\.pm$",
51 # No extension at all, note that ALL CAPS files are black listed in
52 # COMMON_EXCLUDED_LIST below.
53 r"(^|.*[\\\/])[^.]+$",
54 # Other
55 r".*\.java$", r".*\.mk$", r".*\.am$",
Brian Norrisfdbac8e2016-06-16 11:09:05 -070056 r".*\.policy$", r".*\.conf$",
Ryan Cuiec4d6332011-05-02 14:15:25 -070057]
58
Ryan Cui1562fb82011-05-09 11:01:31 -070059
Ryan Cuiec4d6332011-05-02 14:15:25 -070060COMMON_EXCLUDED_PATHS = [
Daniel Erate3ea3fc2015-02-13 15:27:52 -070061 # Avoid doing source file checks for kernel.
Mike Frysingerae409522014-02-01 03:16:11 -050062 r"/src/third_party/kernel/",
63 r"/src/third_party/kernel-next/",
64 r"/src/third_party/ktop/",
65 r"/src/third_party/punybench/",
66 r".*\bexperimental[\\\/].*",
67 r".*\b[A-Z0-9_]{2,}$",
68 r".*[\\\/]debian[\\\/]rules$",
Daniel Erate3ea3fc2015-02-13 15:27:52 -070069
70 # For ebuild trees, ignore any caches and manifest data.
Mike Frysingerae409522014-02-01 03:16:11 -050071 r".*/Manifest$",
72 r".*/metadata/[^/]*cache[^/]*/[^/]+/[^/]+$",
Doug Anderson5bfb6792011-10-25 16:45:41 -070073
Daniel Erate3ea3fc2015-02-13 15:27:52 -070074 # Ignore profiles data (like overlay-tegra2/profiles).
Mike Frysinger94a670c2014-09-19 12:46:26 -040075 r"(^|.*/)overlay-.*/profiles/.*",
Mike Frysinger98638102014-08-28 00:15:08 -040076 r"^profiles/.*$",
77
C Shapiro8f90e9b2017-06-28 09:54:50 -060078 # Ignore config files in ebuild setup.
79 r"(^|.*/)overlay-.*/chromeos-base/chromeos-bsp.*/files/.*",
80 r"^chromeos-base/chromeos-bsp.*/files/.*",
81
Daniel Erate3ea3fc2015-02-13 15:27:52 -070082 # Ignore minified js and jquery.
Mike Frysingerae409522014-02-01 03:16:11 -050083 r".*\.min\.js",
84 r".*jquery.*\.js",
Mike Frysinger33a458d2014-03-03 17:00:51 -050085
86 # Ignore license files as the content is often taken verbatim.
Daniel Erate3ea3fc2015-02-13 15:27:52 -070087 r".*/licenses/.*",
Ryan Cuiec4d6332011-05-02 14:15:25 -070088]
Mandeep Singh Baines116ad102011-04-27 15:16:37 -070089
Ryan Cui1562fb82011-05-09 11:01:31 -070090
Ryan Cui9b651632011-05-11 11:38:58 -070091_CONFIG_FILE = 'PRESUBMIT.cfg'
92
93
Daniel Erate3ea3fc2015-02-13 15:27:52 -070094# File containing wildcards, one per line, matching files that should be
95# excluded from presubmit checks. Lines beginning with '#' are ignored.
96_IGNORE_FILE = '.presubmitignore'
97
98
Doug Anderson44a644f2011-11-02 10:37:37 -070099# Exceptions
100
101
102class BadInvocation(Exception):
103 """An Exception indicating a bad invocation of the program."""
104 pass
105
106
Ryan Cui1562fb82011-05-09 11:01:31 -0700107# General Helpers
108
Sean Paulba01d402011-05-05 11:36:23 -0400109
Alex Deymo643ac4c2015-09-03 10:40:50 -0700110Project = collections.namedtuple('Project', ['name', 'dir', 'remote'])
111
112
Rahul Chaudhry0e515342015-08-07 12:00:43 -0700113# pylint: disable=redefined-builtin
114def _run_command(cmd, cwd=None, input=None,
115 redirect_stderr=False, combine_stdout_stderr=False):
Doug Anderson44a644f2011-11-02 10:37:37 -0700116 """Executes the passed in command and returns raw stdout output.
117
118 Args:
119 cmd: The command to run; should be a list of strings.
120 cwd: The directory to switch to for running the command.
Rahul Chaudhry0e515342015-08-07 12:00:43 -0700121 input: The data to pipe into this command through stdin. If a file object
122 or file descriptor, stdin will be connected directly to that.
123 redirect_stderr: Redirect stderr away from console.
124 combine_stdout_stderr: Combines stdout and stderr streams into stdout.
Doug Anderson44a644f2011-11-02 10:37:37 -0700125
126 Returns:
Rahul Chaudhry0e515342015-08-07 12:00:43 -0700127 The stdout from the process (discards stderr and returncode).
Doug Anderson44a644f2011-11-02 10:37:37 -0700128 """
Rahul Chaudhry0e515342015-08-07 12:00:43 -0700129 return cros_build_lib.RunCommand(cmd=cmd,
130 cwd=cwd,
131 print_cmd=False,
132 input=input,
133 stdout_to_pipe=True,
134 redirect_stderr=redirect_stderr,
135 combine_stdout_stderr=combine_stdout_stderr,
136 error_code_ok=True).output
137# pylint: enable=redefined-builtin
Ryan Cui72834d12011-05-05 14:51:33 -0700138
Ryan Cui1562fb82011-05-09 11:01:31 -0700139
Mandeep Singh Baines116ad102011-04-27 15:16:37 -0700140def _get_hooks_dir():
Ryan Cuiec4d6332011-05-02 14:15:25 -0700141 """Returns the absolute path to the repohooks directory."""
Doug Anderson44a644f2011-11-02 10:37:37 -0700142 if __name__ == '__main__':
143 # Works when file is run on its own (__file__ is defined)...
144 return os.path.abspath(os.path.dirname(__file__))
145 else:
146 # We need to do this when we're run through repo. Since repo executes
147 # us with execfile(), we don't get __file__ defined.
148 cmd = ['repo', 'forall', 'chromiumos/repohooks', '-c', 'pwd']
149 return _run_command(cmd).strip()
Mandeep Singh Baines116ad102011-04-27 15:16:37 -0700150
Ryan Cui1562fb82011-05-09 11:01:31 -0700151
Ryan Cuiec4d6332011-05-02 14:15:25 -0700152def _match_regex_list(subject, expressions):
153 """Try to match a list of regular expressions to a string.
154
155 Args:
156 subject: The string to match regexes on
157 expressions: A list of regular expressions to check for matches with.
158
159 Returns:
160 Whether the passed in subject matches any of the passed in regexes.
161 """
162 for expr in expressions:
Mike Frysingerae409522014-02-01 03:16:11 -0500163 if re.search(expr, subject):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700164 return True
165 return False
166
Ryan Cui1562fb82011-05-09 11:01:31 -0700167
Mike Frysingerae409522014-02-01 03:16:11 -0500168def _filter_files(files, include_list, exclude_list=()):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700169 """Filter out files based on the conditions passed in.
170
171 Args:
172 files: list of filepaths to filter
173 include_list: list of regex that when matched with a file path will cause it
174 to be added to the output list unless the file is also matched with a
175 regex in the exclude_list.
176 exclude_list: list of regex that when matched with a file will prevent it
177 from being added to the output list, even if it is also matched with a
178 regex in the include_list.
179
180 Returns:
181 A list of filepaths that contain files matched in the include_list and not
182 in the exclude_list.
183 """
184 filtered = []
185 for f in files:
186 if (_match_regex_list(f, include_list) and
187 not _match_regex_list(f, exclude_list)):
188 filtered.append(f)
189 return filtered
190
Ryan Cuiec4d6332011-05-02 14:15:25 -0700191
192# Git Helpers
Ryan Cui1562fb82011-05-09 11:01:31 -0700193
194
Ryan Cui4725d952011-05-05 15:41:19 -0700195def _get_upstream_branch():
196 """Returns the upstream tracking branch of the current branch.
197
198 Raises:
199 Error if there is no tracking branch
200 """
201 current_branch = _run_command(['git', 'symbolic-ref', 'HEAD']).strip()
202 current_branch = current_branch.replace('refs/heads/', '')
203 if not current_branch:
Ryan Cui1562fb82011-05-09 11:01:31 -0700204 raise VerifyException('Need to be on a tracking branch')
Ryan Cui4725d952011-05-05 15:41:19 -0700205
206 cfg_option = 'branch.' + current_branch + '.%s'
207 full_upstream = _run_command(['git', 'config', cfg_option % 'merge']).strip()
208 remote = _run_command(['git', 'config', cfg_option % 'remote']).strip()
209 if not remote or not full_upstream:
Ryan Cui1562fb82011-05-09 11:01:31 -0700210 raise VerifyException('Need to be on a tracking branch')
Ryan Cui4725d952011-05-05 15:41:19 -0700211
212 return full_upstream.replace('heads', 'remotes/' + remote)
213
Ryan Cui1562fb82011-05-09 11:01:31 -0700214
Che-Liang Chiou5ce2d7b2013-03-22 18:47:55 -0700215def _get_patch(commit):
216 """Returns the patch for this commit."""
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700217 if commit == PRE_SUBMIT:
218 return _run_command(['git', 'diff', '--cached', 'HEAD'])
219 else:
220 return _run_command(['git', 'format-patch', '--stdout', '-1', commit])
Mandeep Singh Baines116ad102011-04-27 15:16:37 -0700221
Ryan Cui1562fb82011-05-09 11:01:31 -0700222
Jon Salz98255932012-08-18 14:48:02 +0800223def _try_utf8_decode(data):
224 """Attempts to decode a string as UTF-8.
225
226 Returns:
227 The decoded Unicode object, or the original string if parsing fails.
228 """
229 try:
230 return unicode(data, 'utf-8', 'strict')
231 except UnicodeDecodeError:
232 return data
233
234
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500235def _get_file_content(path, commit):
236 """Returns the content of a file at a specific commit.
237
238 We can't rely on the file as it exists in the filesystem as people might be
239 uploading a series of changes which modifies the file multiple times.
240
241 Note: The "content" of a symlink is just the target. So if you're expecting
242 a full file, you should check that first. One way to detect is that the
243 content will not have any newlines.
244 """
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700245 if commit == PRE_SUBMIT:
246 return _run_command(['git', 'diff', 'HEAD', path])
247 else:
248 return _run_command(['git', 'show', '%s:%s' % (commit, path)])
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500249
250
Mike Frysingerae409522014-02-01 03:16:11 -0500251def _get_file_diff(path, commit):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700252 """Returns a list of (linenum, lines) tuples that the commit touched."""
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700253 if commit == PRE_SUBMIT:
Prathmesh Prabhua9de1722016-12-22 14:56:40 -0800254 command = ['git', 'diff', '-p', '--pretty=format:', '--no-ext-diff', 'HEAD',
255 path]
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700256 else:
Prathmesh Prabhua9de1722016-12-22 14:56:40 -0800257 command = ['git', 'show', '-p', '--pretty=format:', '--no-ext-diff', commit,
258 path]
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700259 output = _run_command(command)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700260
261 new_lines = []
262 line_num = 0
263 for line in output.splitlines():
264 m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
265 if m:
266 line_num = int(m.groups(1)[0])
267 continue
268 if line.startswith('+') and not line.startswith('++'):
Jon Salz98255932012-08-18 14:48:02 +0800269 new_lines.append((line_num, _try_utf8_decode(line[1:])))
Ryan Cuiec4d6332011-05-02 14:15:25 -0700270 if not line.startswith('-'):
271 line_num += 1
272 return new_lines
273
Ryan Cui1562fb82011-05-09 11:01:31 -0700274
Daniel Erate3ea3fc2015-02-13 15:27:52 -0700275def _get_ignore_wildcards(directory, cache):
276 """Get wildcards listed in a directory's _IGNORE_FILE.
277
278 Args:
279 directory: A string containing a directory path.
280 cache: A dictionary (opaque to caller) caching previously-read wildcards.
281
282 Returns:
283 A list of wildcards from _IGNORE_FILE or an empty list if _IGNORE_FILE
284 wasn't present.
285 """
286 # In the cache, keys are directories and values are lists of wildcards from
287 # _IGNORE_FILE within those directories (and empty if no file was present).
288 if directory not in cache:
289 wildcards = []
290 dotfile_path = os.path.join(directory, _IGNORE_FILE)
291 if os.path.exists(dotfile_path):
292 # TODO(derat): Consider using _get_file_content() to get the file as of
293 # this commit instead of the on-disk version. This may have a noticeable
294 # performance impact, as each call to _get_file_content() runs git.
295 with open(dotfile_path, 'r') as dotfile:
296 for line in dotfile.readlines():
297 line = line.strip()
298 if line.startswith('#'):
299 continue
300 if line.endswith('/'):
301 line += '*'
302 wildcards.append(line)
303 cache[directory] = wildcards
304
305 return cache[directory]
306
307
308def _path_is_ignored(path, cache):
309 """Check whether a path is ignored by _IGNORE_FILE.
310
311 Args:
312 path: A string containing a path.
313 cache: A dictionary (opaque to caller) caching previously-read wildcards.
314
315 Returns:
316 True if a file named _IGNORE_FILE in one of the passed-in path's parent
317 directories contains a wildcard matching the path.
318 """
319 # Skip ignore files.
320 if os.path.basename(path) == _IGNORE_FILE:
321 return True
322
323 path = os.path.abspath(path)
324 base = os.getcwd()
325
326 prefix = os.path.dirname(path)
327 while prefix.startswith(base):
328 rel_path = path[len(prefix) + 1:]
329 for wildcard in _get_ignore_wildcards(prefix, cache):
330 if fnmatch.fnmatch(rel_path, wildcard):
331 return True
332 prefix = os.path.dirname(prefix)
333
334 return False
335
336
Mike Frysinger292b45d2014-11-25 01:17:10 -0500337def _get_affected_files(commit, include_deletes=False, relative=False,
338 include_symlinks=False, include_adds=True,
Daniel Erate3ea3fc2015-02-13 15:27:52 -0700339 full_details=False, use_ignore_files=True):
Peter Ammon811f6702014-06-12 15:45:38 -0700340 """Returns list of file paths that were modified/added, excluding symlinks.
341
342 Args:
343 commit: The commit
344 include_deletes: If true, we'll include deleted files in the result
345 relative: Whether to return relative or full paths to files
Mike Frysinger292b45d2014-11-25 01:17:10 -0500346 include_symlinks: If true, we'll include symlinks in the result
347 include_adds: If true, we'll include new files in the result
348 full_details: If False, return filenames, else return structured results.
Daniel Erate3ea3fc2015-02-13 15:27:52 -0700349 use_ignore_files: Whether we ignore files matched by _IGNORE_FILE files.
Peter Ammon811f6702014-06-12 15:45:38 -0700350
351 Returns:
352 A list of modified/added (and perhaps deleted) files
353 """
Mike Frysinger292b45d2014-11-25 01:17:10 -0500354 if not relative and full_details:
355 raise ValueError('full_details only supports relative paths currently')
356
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700357 if commit == PRE_SUBMIT:
358 return _run_command(['git', 'diff-index', '--cached',
359 '--name-only', 'HEAD']).split()
Mike Frysingerd3bd32c2014-11-24 23:34:29 -0500360
361 path = os.getcwd()
362 files = git.RawDiff(path, '%s^!' % commit)
363
364 # Filter out symlinks.
Mike Frysinger292b45d2014-11-25 01:17:10 -0500365 if not include_symlinks:
366 files = [x for x in files if not stat.S_ISLNK(int(x.dst_mode, 8))]
Mike Frysingerd3bd32c2014-11-24 23:34:29 -0500367
368 if not include_deletes:
369 files = [x for x in files if x.status != 'D']
370
Mike Frysinger292b45d2014-11-25 01:17:10 -0500371 if not include_adds:
372 files = [x for x in files if x.status != 'A']
373
Daniel Erate3ea3fc2015-02-13 15:27:52 -0700374 if use_ignore_files:
375 cache = {}
376 is_ignored = lambda x: _path_is_ignored(x.dst_file or x.src_file, cache)
377 files = [x for x in files if not is_ignored(x)]
378
Mike Frysinger292b45d2014-11-25 01:17:10 -0500379 if full_details:
380 # Caller wants the raw objects to parse status/etc... themselves.
Mike Frysingerd3bd32c2014-11-24 23:34:29 -0500381 return files
382 else:
Mike Frysinger292b45d2014-11-25 01:17:10 -0500383 # Caller only cares about filenames.
384 files = [x.dst_file if x.dst_file else x.src_file for x in files]
385 if relative:
386 return files
387 else:
388 return [os.path.join(path, x) for x in files]
Peter Ammon811f6702014-06-12 15:45:38 -0700389
390
Mandeep Singh Bainesb9ed1402011-04-29 15:32:06 -0700391def _get_commits():
Ryan Cuiec4d6332011-05-02 14:15:25 -0700392 """Returns a list of commits for this review."""
Ryan Cui4725d952011-05-05 15:41:19 -0700393 cmd = ['git', 'log', '%s..' % _get_upstream_branch(), '--format=%H']
Ryan Cui72834d12011-05-05 14:51:33 -0700394 return _run_command(cmd).split()
Mandeep Singh Bainesb9ed1402011-04-29 15:32:06 -0700395
Ryan Cui1562fb82011-05-09 11:01:31 -0700396
Ryan Cuiec4d6332011-05-02 14:15:25 -0700397def _get_commit_desc(commit):
398 """Returns the full commit message of a commit."""
Vadim Bendebury2b62d742014-06-22 13:14:51 -0700399 if commit == PRE_SUBMIT:
400 return ''
Sean Paul23a2c582011-05-06 13:10:44 -0400401 return _run_command(['git', 'log', '--format=%s%n%n%b', commit + '^!'])
Ryan Cuiec4d6332011-05-02 14:15:25 -0700402
403
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800404def _check_lines_in_diff(commit, files, check_callable, error_description):
405 """Checks given file for errors via the given check.
406
407 This is a convenience function for common per-line checks. It goes through all
408 files and returns a HookFailure with the error description listing all the
409 failures.
410
411 Args:
412 commit: The commit we're working on.
413 files: The files to check.
414 check_callable: A callable that takes a line and returns True if this line
415 _fails_ the check.
416 error_description: A string describing the error.
417 """
418 errors = []
419 for afile in files:
420 for line_num, line in _get_file_diff(afile, commit):
421 if check_callable(line):
422 errors.append('%s, line %s' % (afile, line_num))
423 if errors:
424 return HookFailure(error_description, errors)
425
426
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900427def _parse_common_inclusion_options(options):
428 """Parses common hook options for including/excluding files.
429
430 Args:
431 options: Option string list.
432
433 Returns:
434 (included, excluded) where each one is a list of regex strings.
435 """
436 parser = argparse.ArgumentParser()
437 parser.add_argument('--exclude_regex', action='append')
438 parser.add_argument('--include_regex', action='append')
439 opts = parser.parse_args(options)
440 included = opts.include_regex or []
441 excluded = opts.exclude_regex or []
442 return included, excluded
443
444
Ryan Cuiec4d6332011-05-02 14:15:25 -0700445# Common Hooks
446
Ryan Cui1562fb82011-05-09 11:01:31 -0700447
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900448def _check_no_long_lines(_project, commit, options=()):
Mike Frysinger55f85b52014-12-18 14:45:21 -0500449 """Checks there are no lines longer than MAX_LEN in any of the text files."""
450
Ryan Cuiec4d6332011-05-02 14:15:25 -0700451 MAX_LEN = 80
Jon Salz98255932012-08-18 14:48:02 +0800452 SKIP_REGEXP = re.compile('|'.join([
453 r'https?://',
454 r'^#\s*(define|include|import|pragma|if|endif)\b']))
Ryan Cuiec4d6332011-05-02 14:15:25 -0700455
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900456 included, excluded = _parse_common_inclusion_options(options)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700457 files = _filter_files(_get_affected_files(commit),
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900458 included + COMMON_INCLUDED_PATHS,
459 excluded + COMMON_EXCLUDED_PATHS)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700460
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900461 errors = []
Ryan Cuiec4d6332011-05-02 14:15:25 -0700462 for afile in files:
463 for line_num, line in _get_file_diff(afile, commit):
464 # Allow certain lines to exceed the maxlen rule.
Mike Frysingerae409522014-02-01 03:16:11 -0500465 if len(line) <= MAX_LEN or SKIP_REGEXP.search(line):
Jon Salz98255932012-08-18 14:48:02 +0800466 continue
467
468 errors.append('%s, line %s, %s chars' % (afile, line_num, len(line)))
469 if len(errors) == 5: # Just show the first 5 errors.
470 break
Ryan Cuiec4d6332011-05-02 14:15:25 -0700471
472 if errors:
473 msg = 'Found lines longer than %s characters (first 5 shown):' % MAX_LEN
Ryan Cui1562fb82011-05-09 11:01:31 -0700474 return HookFailure(msg, errors)
475
Ryan Cuiec4d6332011-05-02 14:15:25 -0700476
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900477def _check_no_stray_whitespace(_project, commit, options=()):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700478 """Checks that there is no stray whitespace at source lines end."""
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900479 included, excluded = _parse_common_inclusion_options(options)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700480 files = _filter_files(_get_affected_files(commit),
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900481 included + COMMON_INCLUDED_PATHS,
482 excluded + COMMON_EXCLUDED_PATHS)
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800483 return _check_lines_in_diff(commit, files,
484 lambda line: line.rstrip() != line,
485 'Found line ending with white space in:')
Ryan Cui1562fb82011-05-09 11:01:31 -0700486
Ryan Cuiec4d6332011-05-02 14:15:25 -0700487
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900488def _check_no_tabs(_project, commit, options=()):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700489 """Checks there are no unexpanded tabs."""
Mike Frysingercd134512017-10-26 04:36:33 -0400490 # Don't add entire repos here. Update the PRESUBMIT.cfg in each repo instead.
491 # We only whitelist known specific filetypes here that show up in all repos.
Ryan Cuiec4d6332011-05-02 14:15:25 -0700492 TAB_OK_PATHS = [
Ryan Cuiec4d6332011-05-02 14:15:25 -0700493 r".*\.ebuild$",
494 r".*\.eclass$",
Elly Jones5ab34192011-11-15 14:57:06 -0500495 r".*/[M|m]akefile$",
496 r".*\.mk$"
Ryan Cuiec4d6332011-05-02 14:15:25 -0700497 ]
498
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900499 included, excluded = _parse_common_inclusion_options(options)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700500 files = _filter_files(_get_affected_files(commit),
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900501 included + COMMON_INCLUDED_PATHS,
502 excluded + COMMON_EXCLUDED_PATHS + TAB_OK_PATHS)
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800503 return _check_lines_in_diff(commit, files,
504 lambda line: '\t' in line,
505 'Found a tab character in:')
Ryan Cuiec4d6332011-05-02 14:15:25 -0700506
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800507
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900508def _check_tabbed_indents(_project, commit, options=()):
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800509 """Checks that indents use tabs only."""
510 TABS_REQUIRED_PATHS = [
511 r".*\.ebuild$",
512 r".*\.eclass$",
513 ]
514 LEADING_SPACE_RE = re.compile('[\t]* ')
515
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900516 included, excluded = _parse_common_inclusion_options(options)
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800517 files = _filter_files(_get_affected_files(commit),
Shuhei Takahashiabc20f32017-07-10 19:35:45 +0900518 included + TABS_REQUIRED_PATHS,
519 excluded + COMMON_EXCLUDED_PATHS)
Prathmesh Prabhuc5254652016-12-22 12:58:05 -0800520 return _check_lines_in_diff(
521 commit, files,
522 lambda line: LEADING_SPACE_RE.match(line) is not None,
523 'Found a space in indentation (must be all tabs):')
Ryan Cui1562fb82011-05-09 11:01:31 -0700524
Ryan Cuiec4d6332011-05-02 14:15:25 -0700525
Rahul Chaudhry09f61372015-07-31 17:14:26 -0700526def _check_gofmt(_project, commit):
527 """Checks that Go files are formatted with gofmt."""
528 errors = []
529 files = _filter_files(_get_affected_files(commit, relative=True),
530 [r'\.go$'])
531
532 for gofile in files:
533 contents = _get_file_content(gofile, commit)
Rahul Chaudhry0e515342015-08-07 12:00:43 -0700534 output = _run_command(cmd=['gofmt', '-l'], input=contents,
535 combine_stdout_stderr=True)
Rahul Chaudhry09f61372015-07-31 17:14:26 -0700536 if output:
537 errors.append(gofile)
538 if errors:
539 return HookFailure('Files not formatted with gofmt:', errors)
540
541
Mike Frysingerae409522014-02-01 03:16:11 -0500542def _check_change_has_test_field(_project, commit):
Ryan Cuiec4d6332011-05-02 14:15:25 -0700543 """Check for a non-empty 'TEST=' field in the commit message."""
David McMahon8f6553e2011-06-10 15:46:36 -0700544 TEST_RE = r'\nTEST=\S+'
Ryan Cuiec4d6332011-05-02 14:15:25 -0700545
Mandeep Singh Baines96a53be2011-05-03 11:10:25 -0700546 if not re.search(TEST_RE, _get_commit_desc(commit)):
Ryan Cui1562fb82011-05-09 11:01:31 -0700547 msg = 'Changelist description needs TEST field (after first line)'
548 return HookFailure(msg)
549
Ryan Cuiec4d6332011-05-02 14:15:25 -0700550
Mike Frysingerae409522014-02-01 03:16:11 -0500551def _check_change_has_valid_cq_depend(_project, commit):
David Jamesc3b68b32013-04-03 09:17:03 -0700552 """Check for a correctly formatted CQ-DEPEND field in the commit message."""
553 msg = 'Changelist has invalid CQ-DEPEND target.'
554 example = 'Example: CQ-DEPEND=CL:1234, CL:2345'
555 try:
556 patch.GetPaladinDeps(_get_commit_desc(commit))
557 except ValueError as ex:
558 return HookFailure(msg, [example, str(ex)])
559
560
Bernie Thompsonf8fea992016-01-14 10:27:18 -0800561def _check_change_is_contribution(_project, commit):
562 """Check that the change is a contribution."""
563 NO_CONTRIB = 'not a contribution'
564 if NO_CONTRIB in _get_commit_desc(commit).lower():
565 msg = ('Changelist is not a contribution, this cannot be accepted.\n'
566 'Please remove the "%s" text from the commit message.') % NO_CONTRIB
567 return HookFailure(msg)
568
569
Alex Deymo643ac4c2015-09-03 10:40:50 -0700570def _check_change_has_bug_field(project, commit):
David McMahon8f6553e2011-06-10 15:46:36 -0700571 """Check for a correctly formatted 'BUG=' field in the commit message."""
David James5c0073d2013-04-03 08:48:52 -0700572 OLD_BUG_RE = r'\nBUG=.*chromium-os'
573 if re.search(OLD_BUG_RE, _get_commit_desc(commit)):
574 msg = ('The chromium-os bug tracker is now deprecated. Please use\n'
575 'the chromium tracker in your BUG= line now.')
576 return HookFailure(msg)
Ryan Cuiec4d6332011-05-02 14:15:25 -0700577
Alex Deymo643ac4c2015-09-03 10:40:50 -0700578 # Android internal and external projects use "Bug: " to track bugs in
579 # buganizer.
580 BUG_COLON_REMOTES = (
581 'aosp',
582 'goog',
583 )
584 if project.remote in BUG_COLON_REMOTES:
585 BUG_RE = r'\nBug: ?([Nn]one|\d+)'
586 if not re.search(BUG_RE, _get_commit_desc(commit)):
587 msg = ('Changelist description needs BUG field (after first line):\n'
588 'Bug: 9999 (for buganizer)\n'
589 'BUG=None')
590 return HookFailure(msg)
591 else:
Jorge Lucangeli Obesdce214e2017-10-25 15:04:39 -0400592 BUG_RE = r'\nBUG=([Nn]one|(chromium|b):\d+)'
Alex Deymo643ac4c2015-09-03 10:40:50 -0700593 if not re.search(BUG_RE, _get_commit_desc(commit)):
594 msg = ('Changelist description needs BUG field (after first line):\n'
Alex Deymo643ac4c2015-09-03 10:40:50 -0700595 'BUG=chromium:9999 (for public tracker)\n'
Alex Deymo643ac4c2015-09-03 10:40:50 -0700596 'BUG=b:9999 (for buganizer)\n'
597 'BUG=None')
598 return HookFailure(msg)
Ryan Cui1562fb82011-05-09 11:01:31 -0700599
Ryan Cuiec4d6332011-05-02 14:15:25 -0700600
Mike Frysinger292b45d2014-11-25 01:17:10 -0500601def _check_for_uprev(project, commit, project_top=None):
Doug Anderson42b8a052013-06-26 10:45:36 -0700602 """Check that we're not missing a revbump of an ebuild in the given commit.
603
604 If the given commit touches files in a directory that has ebuilds somewhere
605 up the directory hierarchy, it's very likely that we need an ebuild revbump
606 in order for those changes to take effect.
607
608 It's not totally trivial to detect a revbump, so at least detect that an
609 ebuild with a revision number in it was touched. This should handle the
610 common case where we use a symlink to do the revbump.
611
612 TODO: it would be nice to enhance this hook to:
613 * Handle cases where people revbump with a slightly different syntax. I see
614 one ebuild (puppy) that revbumps with _pN. This is a false positive.
615 * Catches cases where people aren't using symlinks for revbumps. If they
616 edit a revisioned file directly (and are expected to rename it for revbump)
617 we'll miss that. Perhaps we could detect that the file touched is a
618 symlink?
619
620 If a project doesn't use symlinks we'll potentially miss a revbump, but we're
621 still better off than without this check.
622
623 Args:
Alex Deymo643ac4c2015-09-03 10:40:50 -0700624 project: The Project to look at
Doug Anderson42b8a052013-06-26 10:45:36 -0700625 commit: The commit to look at
Mike Frysinger292b45d2014-11-25 01:17:10 -0500626 project_top: Top dir to process commits in
Doug Anderson42b8a052013-06-26 10:45:36 -0700627
628 Returns:
629 A HookFailure or None.
630 """
Mike Frysinger011af942014-01-17 16:12:22 -0500631 # If this is the portage-stable overlay, then ignore the check. It's rare
632 # that we're doing anything other than importing files from upstream, so
633 # forcing a rev bump makes no sense.
634 whitelist = (
635 'chromiumos/overlays/portage-stable',
636 )
Alex Deymo643ac4c2015-09-03 10:40:50 -0700637 if project.name in whitelist:
Mike Frysinger011af942014-01-17 16:12:22 -0500638 return None
639
Mike Frysinger292b45d2014-11-25 01:17:10 -0500640 def FinalName(obj):
641 # If the file is being deleted, then the dst_file is not set.
642 if obj.dst_file is None:
643 return obj.src_file
644 else:
645 return obj.dst_file
646
647 affected_path_objs = _get_affected_files(
648 commit, include_deletes=True, include_symlinks=True, relative=True,
649 full_details=True)
Doug Anderson42b8a052013-06-26 10:45:36 -0700650
651 # Don't yell about changes to whitelisted files...
Aviv Keshet272f2e52016-04-25 14:49:44 -0700652 whitelist = ('ChangeLog', 'Manifest', 'metadata.xml', 'COMMIT-QUEUE.ini')
Mike Frysinger292b45d2014-11-25 01:17:10 -0500653 affected_path_objs = [x for x in affected_path_objs
654 if os.path.basename(FinalName(x)) not in whitelist]
655 if not affected_path_objs:
Doug Anderson42b8a052013-06-26 10:45:36 -0700656 return None
657
658 # If we've touched any file named with a -rN.ebuild then we'll say we're
659 # OK right away. See TODO above about enhancing this.
Mike Frysinger292b45d2014-11-25 01:17:10 -0500660 touched_revved_ebuild = any(re.search(r'-r\d*\.ebuild$', FinalName(x))
661 for x in affected_path_objs)
Doug Anderson42b8a052013-06-26 10:45:36 -0700662 if touched_revved_ebuild:
663 return None
664
Mike Frysinger292b45d2014-11-25 01:17:10 -0500665 # If we're creating new ebuilds from scratch, then we don't need an uprev.
666 # Find all the dirs that new ebuilds and ignore their files/.
667 ebuild_dirs = [os.path.dirname(FinalName(x)) + '/' for x in affected_path_objs
668 if FinalName(x).endswith('.ebuild') and x.status == 'A']
669 affected_path_objs = [obj for obj in affected_path_objs
670 if not any(FinalName(obj).startswith(x)
671 for x in ebuild_dirs)]
672 if not affected_path_objs:
673 return
674
Doug Anderson42b8a052013-06-26 10:45:36 -0700675 # We want to examine the current contents of all directories that are parents
676 # of files that were touched (up to the top of the project).
677 #
678 # ...note: we use the current directory contents even though it may have
679 # changed since the commit we're looking at. This is just a heuristic after
680 # all. Worst case we don't flag a missing revbump.
Mike Frysinger292b45d2014-11-25 01:17:10 -0500681 if project_top is None:
682 project_top = os.getcwd()
Doug Anderson42b8a052013-06-26 10:45:36 -0700683 dirs_to_check = set([project_top])
Mike Frysinger292b45d2014-11-25 01:17:10 -0500684 for obj in affected_path_objs:
685 path = os.path.join(project_top, os.path.dirname(FinalName(obj)))
Doug Anderson42b8a052013-06-26 10:45:36 -0700686 while os.path.exists(path) and not os.path.samefile(path, project_top):
687 dirs_to_check.add(path)
688 path = os.path.dirname(path)
689
690 # Look through each directory. If it's got an ebuild in it then we'll
691 # consider this as a case when we need a revbump.
Gwendal Grignoua3086c32014-12-09 11:17:22 -0800692 affected_paths = set(os.path.join(project_top, FinalName(x))
693 for x in affected_path_objs)
Doug Anderson42b8a052013-06-26 10:45:36 -0700694 for dir_path in dirs_to_check:
695 contents = os.listdir(dir_path)
696 ebuilds = [os.path.join(dir_path, path)
697 for path in contents if path.endswith('.ebuild')]
698 ebuilds_9999 = [path for path in ebuilds if path.endswith('-9999.ebuild')]
699
C Shapiroae157ae2017-09-18 16:24:03 -0600700 affected_paths_under_9999_ebuilds = set()
701 for affected_path in affected_paths:
702 for ebuild_9999 in ebuilds_9999:
703 ebuild_dir = os.path.dirname(ebuild_9999)
704 if affected_path.startswith(ebuild_dir):
705 affected_paths_under_9999_ebuilds.add(affected_path)
706
707 # If every file changed exists under a 9999 ebuild, then skip
708 if len(affected_paths_under_9999_ebuilds) == len(affected_paths):
709 continue
710
Doug Anderson42b8a052013-06-26 10:45:36 -0700711 # If the -9999.ebuild file was touched the bot will uprev for us.
712 # ...we'll use a simple intersection here as a heuristic...
Mike Frysinger292b45d2014-11-25 01:17:10 -0500713 if set(ebuilds_9999) & affected_paths:
Doug Anderson42b8a052013-06-26 10:45:36 -0700714 continue
715
716 if ebuilds:
Mike Frysinger292b45d2014-11-25 01:17:10 -0500717 return HookFailure('Changelist probably needs a revbump of an ebuild, '
718 'or a -r1.ebuild symlink if this is a new ebuild:\n'
719 '%s' % dir_path)
Doug Anderson42b8a052013-06-26 10:45:36 -0700720
721 return None
722
723
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500724def _check_ebuild_eapi(project, commit):
725 """Make sure we have people use EAPI=4 or newer with custom ebuilds.
726
727 We want to get away from older EAPI's as it makes life confusing and they
728 have less builtin error checking.
729
730 Args:
Alex Deymo643ac4c2015-09-03 10:40:50 -0700731 project: The Project to look at
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500732 commit: The commit to look at
733
734 Returns:
735 A HookFailure or None.
736 """
737 # If this is the portage-stable overlay, then ignore the check. It's rare
Mike Frysingercd6adfc2014-02-06 01:03:56 -0500738 # that we're doing anything other than importing files from upstream, and
739 # we shouldn't be rewriting things fundamentally anyways.
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500740 whitelist = (
741 'chromiumos/overlays/portage-stable',
742 )
Alex Deymo643ac4c2015-09-03 10:40:50 -0700743 if project.name in whitelist:
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500744 return None
745
746 BAD_EAPIS = ('0', '1', '2', '3')
747
748 get_eapi = re.compile(r'^\s*EAPI=[\'"]?([^\'"]+)')
749
750 ebuilds_re = [r'\.ebuild$']
751 ebuilds = _filter_files(_get_affected_files(commit, relative=True),
752 ebuilds_re)
753 bad_ebuilds = []
754
755 for ebuild in ebuilds:
756 # If the ebuild does not specify an EAPI, it defaults to 0.
757 eapi = '0'
758
759 lines = _get_file_content(ebuild, commit).splitlines()
760 if len(lines) == 1:
761 # This is most likely a symlink, so skip it entirely.
762 continue
763
764 for line in lines:
765 m = get_eapi.match(line)
766 if m:
767 # Once we hit the first EAPI line in this ebuild, stop processing.
768 # The spec requires that there only be one and it be first, so
769 # checking all possible values is pointless. We also assume that
770 # it's "the" EAPI line and not something in the middle of a heredoc.
771 eapi = m.group(1)
772 break
773
774 if eapi in BAD_EAPIS:
775 bad_ebuilds.append((ebuild, eapi))
776
777 if bad_ebuilds:
778 # pylint: disable=C0301
779 url = 'http://dev.chromium.org/chromium-os/how-tos-and-troubleshooting/upgrade-ebuild-eapis'
780 # pylint: enable=C0301
781 return HookFailure(
Mike Frysingercd6adfc2014-02-06 01:03:56 -0500782 'These ebuilds are using old EAPIs. If these are imported from\n'
783 'Gentoo, then you may ignore and upload once with the --no-verify\n'
784 'flag. Otherwise, please update to 4 or newer.\n'
Mike Frysingerbf8b91c2014-02-01 02:50:27 -0500785 '\t%s\n'
786 'See this guide for more details:\n%s\n' %
787 ('\n\t'.join(['%s: EAPI=%s' % x for x in bad_ebuilds]), url))
788
789
Mike Frysinger89bdb852014-02-01 05:26:26 -0500790def _check_ebuild_keywords(_project, commit):
Mike Frysingerc51ece72014-01-17 16:23:40 -0500791 """Make sure we use the new style KEYWORDS when possible in ebuilds.
792
793 If an ebuild generally does not care about the arch it is running on, then
794 ebuilds should flag it with one of:
795 KEYWORDS="*" # A stable ebuild.
796 KEYWORDS="~*" # An unstable ebuild.
797 KEYWORDS="-* ..." # Is known to only work on specific arches.
798
799 Args:
Alex Deymo643ac4c2015-09-03 10:40:50 -0700800 project: The Project to look at
Mike Frysingerc51ece72014-01-17 16:23:40 -0500801 commit: The commit to look at
802
803 Returns:
804 A HookFailure or None.
805 """
806 WHITELIST = set(('*', '-*', '~*'))
807
808 get_keywords = re.compile(r'^\s*KEYWORDS="(.*)"')
809
Mike Frysinger89bdb852014-02-01 05:26:26 -0500810 ebuilds_re = [r'\.ebuild$']
811 ebuilds = _filter_files(_get_affected_files(commit, relative=True),
812 ebuilds_re)
813
Mike Frysinger8d42d742014-09-22 15:50:21 -0400814 bad_ebuilds = []
Mike Frysingerc51ece72014-01-17 16:23:40 -0500815 for ebuild in ebuilds:
Mike Frysinger5c9e58d2014-09-09 03:32:50 -0400816 # We get the full content rather than a diff as the latter does not work
817 # on new files (like when adding new ebuilds).
818 lines = _get_file_content(ebuild, commit).splitlines()
819 for line in lines:
Mike Frysingerc51ece72014-01-17 16:23:40 -0500820 m = get_keywords.match(line)
821 if m:
822 keywords = set(m.group(1).split())
823 if not keywords or WHITELIST - keywords != WHITELIST:
824 continue
825
Mike Frysinger8d42d742014-09-22 15:50:21 -0400826 bad_ebuilds.append(ebuild)
827
828 if bad_ebuilds:
829 return HookFailure(
830 '%s\n'
831 'Please update KEYWORDS to use a glob:\n'
832 'If the ebuild should be marked stable (normal for non-9999 ebuilds):\n'
833 ' KEYWORDS="*"\n'
834 'If the ebuild should be marked unstable (normal for '
835 'cros-workon / 9999 ebuilds):\n'
836 ' KEYWORDS="~*"\n'
Mike Frysingerde8efea2015-05-17 03:42:26 -0400837 'If the ebuild needs to be marked for only specific arches, '
Mike Frysinger8d42d742014-09-22 15:50:21 -0400838 'then use -* like so:\n'
839 ' KEYWORDS="-* arm ..."\n' % '\n* '.join(bad_ebuilds))
Mike Frysingerc51ece72014-01-17 16:23:40 -0500840
841
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -0800842def _check_ebuild_licenses(_project, commit):
843 """Check if the LICENSE field in the ebuild is correct."""
Brian Norris7a610e82016-02-17 12:24:54 -0800844 affected_paths = _get_affected_files(commit, relative=True)
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -0800845 touched_ebuilds = [x for x in affected_paths if x.endswith('.ebuild')]
846
847 # A list of licenses to ignore for now.
Yu-Ju Hongc0963fa2014-03-03 12:36:52 -0800848 LICENSES_IGNORE = ['||', '(', ')']
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -0800849
850 for ebuild in touched_ebuilds:
851 # Skip virutal packages.
852 if ebuild.split('/')[-3] == 'virtual':
853 continue
854
855 try:
Brian Norris7a610e82016-02-17 12:24:54 -0800856 ebuild_content = _get_file_content(ebuild, commit)
857 license_types = licenses_lib.GetLicenseTypesFromEbuild(ebuild_content)
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -0800858 except ValueError as e:
859 return HookFailure(e.message, [ebuild])
860
861 # Also ignore licenses ending with '?'
862 for license_type in [x for x in license_types
863 if x not in LICENSES_IGNORE and not x.endswith('?')]:
864 try:
Mike Frysinger2ec70ed2014-08-17 19:28:34 -0400865 licenses_lib.Licensing.FindLicenseType(license_type)
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -0800866 except AssertionError as e:
867 return HookFailure(e.message, [ebuild])
868
869
Mike Frysingercd363c82014-02-01 05:20:18 -0500870def _check_ebuild_virtual_pv(project, commit):
871 """Enforce the virtual PV policies."""
872 # If this is the portage-stable overlay, then ignore the check.
873 # We want to import virtuals as-is from upstream Gentoo.
874 whitelist = (
875 'chromiumos/overlays/portage-stable',
876 )
Alex Deymo643ac4c2015-09-03 10:40:50 -0700877 if project.name in whitelist:
Mike Frysingercd363c82014-02-01 05:20:18 -0500878 return None
879
880 # We assume the repo name is the same as the dir name on disk.
881 # It would be dumb to not have them match though.
Alex Deymo643ac4c2015-09-03 10:40:50 -0700882 project_base = os.path.basename(project.name)
Mike Frysingercd363c82014-02-01 05:20:18 -0500883
884 is_variant = lambda x: x.startswith('overlay-variant-')
885 is_board = lambda x: x.startswith('overlay-')
886 is_private = lambda x: x.endswith('-private')
887
888 get_pv = re.compile(r'(.*?)virtual/([^/]+)/\2-([^/]*)\.ebuild$')
889
890 ebuilds_re = [r'\.ebuild$']
891 ebuilds = _filter_files(_get_affected_files(commit, relative=True),
892 ebuilds_re)
893 bad_ebuilds = []
894
895 for ebuild in ebuilds:
896 m = get_pv.match(ebuild)
897 if m:
898 overlay = m.group(1)
899 if not overlay or not is_board(overlay):
Alex Deymo643ac4c2015-09-03 10:40:50 -0700900 overlay = project_base
Mike Frysingercd363c82014-02-01 05:20:18 -0500901
902 pv = m.group(3).split('-', 1)[0]
903
Bernie Thompsone5ee1822016-01-12 14:22:23 -0800904 # Virtual versions >= 4 are special cases used above the standard
905 # versioning structure, e.g. if one has a board inheriting a board.
906 if float(pv) >= 4:
907 want_pv = pv
908 elif is_private(overlay):
Mike Frysingercd363c82014-02-01 05:20:18 -0500909 want_pv = '3.5' if is_variant(overlay) else '3'
910 elif is_board(overlay):
911 want_pv = '2.5' if is_variant(overlay) else '2'
912 else:
913 want_pv = '1'
914
915 if pv != want_pv:
916 bad_ebuilds.append((ebuild, pv, want_pv))
917
918 if bad_ebuilds:
919 # pylint: disable=C0301
920 url = 'http://dev.chromium.org/chromium-os/how-tos-and-troubleshooting/portage-build-faq#TOC-Virtuals-and-central-management'
921 # pylint: enable=C0301
922 return HookFailure(
923 'These virtuals have incorrect package versions (PVs). Please adjust:\n'
924 '\t%s\n'
925 'If this is an upstream Gentoo virtual, then you may ignore this\n'
926 'check (and re-run w/--no-verify). Otherwise, please see this\n'
927 'page for more details:\n%s\n' %
928 ('\n\t'.join(['%s:\n\t\tPV is %s but should be %s' % x
929 for x in bad_ebuilds]), url))
930
931
Daniel Erat9d203ff2015-02-17 10:12:21 -0700932def _check_portage_make_use_var(_project, commit):
933 """Verify that $USE is set correctly in make.conf and make.defaults."""
934 files = _filter_files(_get_affected_files(commit, relative=True),
935 [r'(^|/)make.(conf|defaults)$'])
936
937 errors = []
938 for path in files:
939 basename = os.path.basename(path)
940
941 # Has a USE= line already been encountered in this file?
942 saw_use = False
943
944 for i, line in enumerate(_get_file_content(path, commit).splitlines(), 1):
945 if not line.startswith('USE='):
946 continue
947
948 preserves_use = '${USE}' in line or '$USE' in line
949
950 if (basename == 'make.conf' or
951 (basename == 'make.defaults' and saw_use)) and not preserves_use:
952 errors.append('%s:%d: missing ${USE}' % (path, i))
953 elif basename == 'make.defaults' and not saw_use and preserves_use:
954 errors.append('%s:%d: ${USE} referenced in initial declaration' %
955 (path, i))
956
957 saw_use = True
958
959 if errors:
960 return HookFailure(
961 'One or more Portage make files appear to set USE incorrectly.\n'
962 '\n'
963 'All USE assignments in make.conf and all assignments after the\n'
964 'initial declaration in make.defaults should contain "${USE}" to\n'
965 'preserve previously-set flags.\n'
966 '\n'
967 'The initial USE declaration in make.defaults should not contain\n'
968 '"${USE}".\n',
969 errors)
970
971
Mike Frysingerae409522014-02-01 03:16:11 -0500972def _check_change_has_proper_changeid(_project, commit):
Mandeep Singh Bainesa23eb5f2011-05-04 13:43:25 -0700973 """Verify that Change-ID is present in last paragraph of commit message."""
Mike Frysinger4a22bf02014-10-31 13:53:35 -0400974 CHANGE_ID_RE = r'\nChange-Id: I[a-f0-9]+\n'
Mandeep Singh Bainesa23eb5f2011-05-04 13:43:25 -0700975 desc = _get_commit_desc(commit)
Mike Frysinger4a22bf02014-10-31 13:53:35 -0400976 m = re.search(CHANGE_ID_RE, desc)
Mike Frysinger02b88bd2014-11-21 00:29:38 -0500977 if not m:
Vadim Bendebury20532ba2017-05-23 17:26:15 -0700978 return HookFailure('Last paragraph of description must include Change-Id.')
Ryan Cui1562fb82011-05-09 11:01:31 -0700979
Vadim Bendebury20532ba2017-05-23 17:26:15 -0700980 # S-o-b tags always allowed to follow Change-ID.
981 allowed_tags = ['Signed-off-by']
982
Mike Frysinger02b88bd2014-11-21 00:29:38 -0500983 end = desc[m.end():].strip().splitlines()
Vadim Bendebury6504aea2017-09-13 18:35:49 -0700984 cherry_pick_marker = 'cherry picked from commit'
985
986 if end and cherry_pick_marker in end[-1]:
Vadim Bendebury20532ba2017-05-23 17:26:15 -0700987 # Cherry picked patches allow more tags in the last paragraph.
Vadim Bendebury6504aea2017-09-13 18:35:49 -0700988 allowed_tags += ['Commit-Queue', 'Commit-Ready', 'Reviewed-by',
989 'Reviewed-on', 'Tested-by']
Vadim Bendebury20532ba2017-05-23 17:26:15 -0700990 end = end[:-1]
991
Vadim Bendebury6504aea2017-09-13 18:35:49 -0700992 # Note that descriptions could have multiple cherry pick markers.
993 tag_search = r'^(%s:|\(%s) ' % (':|'.join(allowed_tags), cherry_pick_marker)
Vadim Bendebury20532ba2017-05-23 17:26:15 -0700994
995 if [x for x in end if not re.search(tag_search, x)]:
996 return HookFailure('Only "%s:" tag(s) may follow the Change-Id.' %
997 ':", "'.join(allowed_tags))
Mike Frysinger02b88bd2014-11-21 00:29:38 -0500998
Mandeep Singh Bainesa23eb5f2011-05-04 13:43:25 -0700999
Mike Frysinger36b2ebc2014-10-31 14:02:03 -04001000def _check_commit_message_style(_project, commit):
1001 """Verify that the commit message matches our style.
1002
1003 We do not check for BUG=/TEST=/etc... lines here as that is handled by other
1004 commit hooks.
1005 """
1006 desc = _get_commit_desc(commit)
1007
1008 # The first line should be by itself.
1009 lines = desc.splitlines()
1010 if len(lines) > 1 and lines[1]:
1011 return HookFailure('The second line of the commit message must be blank.')
1012
1013 # The first line should be one sentence.
1014 if '. ' in lines[0]:
1015 return HookFailure('The first line cannot be more than one sentence.')
1016
1017 # The first line cannot be too long.
1018 MAX_FIRST_LINE_LEN = 100
1019 if len(lines[0]) > MAX_FIRST_LINE_LEN:
1020 return HookFailure('The first line must be less than %i chars.' %
1021 MAX_FIRST_LINE_LEN)
1022
1023
Filipe Brandenburger4b542b12015-10-09 12:46:31 -07001024def _check_cros_license(_project, commit, options=()):
Alex Deymof5792ce2015-08-24 22:50:08 -07001025 """Verifies the Chromium OS license/copyright header.
Ryan Cuiec4d6332011-05-02 14:15:25 -07001026
Mike Frysinger98638102014-08-28 00:15:08 -04001027 Should be following the spec:
1028 http://dev.chromium.org/developers/coding-style#TOC-File-headers
1029 """
1030 # For older years, be a bit more flexible as our policy says leave them be.
1031 LICENSE_HEADER = (
1032 r'.* Copyright( \(c\))? 20[-0-9]{2,7} The Chromium OS Authors\. '
Mike Frysingerb81102f2014-11-21 00:33:35 -05001033 r'All rights reserved\.' r'\n'
Mike Frysinger98638102014-08-28 00:15:08 -04001034 r'.* Use of this source code is governed by a BSD-style license that can '
Mike Frysingerb81102f2014-11-21 00:33:35 -05001035 r'be\n'
Mike Frysinger98638102014-08-28 00:15:08 -04001036 r'.* found in the LICENSE file\.'
Mike Frysingerb81102f2014-11-21 00:33:35 -05001037 r'\n'
Mike Frysinger98638102014-08-28 00:15:08 -04001038 )
1039 license_re = re.compile(LICENSE_HEADER, re.MULTILINE)
1040
1041 # For newer years, be stricter.
1042 COPYRIGHT_LINE = (
1043 r'.* Copyright \(c\) 20(1[5-9]|[2-9][0-9]) The Chromium OS Authors\. '
Mike Frysingerb81102f2014-11-21 00:33:35 -05001044 r'All rights reserved\.' r'\n'
Mike Frysinger98638102014-08-28 00:15:08 -04001045 )
1046 copyright_re = re.compile(COPYRIGHT_LINE)
1047
Shuhei Takahashiabc20f32017-07-10 19:35:45 +09001048 included, excluded = _parse_common_inclusion_options(options)
Filipe Brandenburger4b542b12015-10-09 12:46:31 -07001049
Mike Frysinger98638102014-08-28 00:15:08 -04001050 bad_files = []
1051 bad_copyright_files = []
1052 files = _filter_files(_get_affected_files(commit, relative=True),
Filipe Brandenburger4b542b12015-10-09 12:46:31 -07001053 included + COMMON_INCLUDED_PATHS,
1054 excluded + COMMON_EXCLUDED_PATHS)
Mike Frysinger98638102014-08-28 00:15:08 -04001055
1056 for f in files:
1057 contents = _get_file_content(f, commit)
1058 if not contents:
1059 # Ignore empty files.
1060 continue
1061
1062 if not license_re.search(contents):
1063 bad_files.append(f)
1064 elif copyright_re.search(contents):
1065 bad_copyright_files.append(f)
1066
1067 if bad_files:
1068 msg = '%s:\n%s\n%s' % (
1069 'License must match', license_re.pattern,
1070 'Found a bad header in these files:')
1071 return HookFailure(msg, bad_files)
1072
1073 if bad_copyright_files:
1074 msg = 'Do not use (c) in copyright headers in new files:'
1075 return HookFailure(msg, bad_copyright_files)
Ryan Cuiec4d6332011-05-02 14:15:25 -07001076
1077
Alex Deymof5792ce2015-08-24 22:50:08 -07001078def _check_aosp_license(_project, commit):
1079 """Verifies the AOSP license/copyright header.
1080
1081 AOSP uses the Apache2 License:
1082 https://source.android.com/source/licenses.html
1083 """
1084 LICENSE_HEADER = (
1085 r"""^[#/\*]*
1086[#/\*]* ?Copyright( \([cC]\))? 20[-0-9]{2,7} The Android Open Source Project
1087[#/\*]* ?
1088[#/\*]* ?Licensed under the Apache License, Version 2.0 \(the "License"\);
1089[#/\*]* ?you may not use this file except in compliance with the License\.
1090[#/\*]* ?You may obtain a copy of the License at
1091[#/\*]* ?
1092[#/\*]* ? http://www\.apache\.org/licenses/LICENSE-2\.0
1093[#/\*]* ?
1094[#/\*]* ?Unless required by applicable law or agreed to in writing, software
1095[#/\*]* ?distributed under the License is distributed on an "AS IS" BASIS,
1096[#/\*]* ?WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or """
1097 r"""implied\.
1098[#/\*]* ?See the License for the specific language governing permissions and
1099[#/\*]* ?limitations under the License\.
1100[#/\*]*$
1101"""
1102 )
1103 license_re = re.compile(LICENSE_HEADER, re.MULTILINE)
1104
1105 files = _filter_files(_get_affected_files(commit, relative=True),
1106 COMMON_INCLUDED_PATHS,
1107 COMMON_EXCLUDED_PATHS)
1108
1109 bad_files = []
1110 for f in files:
1111 contents = _get_file_content(f, commit)
1112 if not contents:
1113 # Ignore empty files.
1114 continue
1115
1116 if not license_re.search(contents):
1117 bad_files.append(f)
1118
1119 if bad_files:
1120 msg = ('License must match:\n%s\nFound a bad header in these files:' %
1121 license_re.pattern)
1122 return HookFailure(msg, bad_files)
1123
1124
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001125def _check_layout_conf(_project, commit):
1126 """Verifies the metadata/layout.conf file."""
1127 repo_name = 'profiles/repo_name'
Mike Frysinger94a670c2014-09-19 12:46:26 -04001128 repo_names = []
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001129 layout_path = 'metadata/layout.conf'
Mike Frysinger94a670c2014-09-19 12:46:26 -04001130 layout_paths = []
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001131
Mike Frysinger94a670c2014-09-19 12:46:26 -04001132 # Handle multiple overlays in a single commit (like the public tree).
1133 for f in _get_affected_files(commit, relative=True):
1134 if f.endswith(repo_name):
1135 repo_names.append(f)
1136 elif f.endswith(layout_path):
1137 layout_paths.append(f)
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001138
1139 # Disallow new repos with the repo_name file.
Mike Frysinger94a670c2014-09-19 12:46:26 -04001140 if repo_names:
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001141 return HookFailure('%s: use "repo-name" in %s instead' %
Mike Frysinger94a670c2014-09-19 12:46:26 -04001142 (repo_names, layout_path))
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001143
Mike Frysinger94a670c2014-09-19 12:46:26 -04001144 # Gather all the errors in one pass so we show one full message.
1145 all_errors = {}
1146 for layout_path in layout_paths:
1147 all_errors[layout_path] = errors = []
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001148
Mike Frysinger94a670c2014-09-19 12:46:26 -04001149 # Make sure the config file is sorted.
1150 data = [x for x in _get_file_content(layout_path, commit).splitlines()
1151 if x and x[0] != '#']
1152 if sorted(data) != data:
1153 errors += ['keep lines sorted']
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001154
Mike Frysinger94a670c2014-09-19 12:46:26 -04001155 # Require people to set specific values all the time.
1156 settings = (
1157 # TODO: Enable this for everyone. http://crbug.com/408038
1158 #('fast caching', 'cache-format = md5-dict'),
1159 ('fast manifests', 'thin-manifests = true'),
Mike Frysingerd7734522015-02-26 16:12:43 -05001160 ('extra features', 'profile-formats = portage-2 profile-default-eapi'),
1161 ('newer eapi', 'profile_eapi_when_unspecified = 5-progress'),
Mike Frysinger94a670c2014-09-19 12:46:26 -04001162 )
1163 for reason, line in settings:
1164 if line not in data:
1165 errors += ['enable %s with: %s' % (reason, line)]
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001166
Mike Frysinger94a670c2014-09-19 12:46:26 -04001167 # Require one of these settings.
Mike Frysinger5fae62d2015-11-11 20:12:15 -05001168 if 'use-manifests = strict' not in data:
1169 errors += ['enable file checking with: use-manifests = strict']
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001170
Mike Frysinger94a670c2014-09-19 12:46:26 -04001171 # Require repo-name to be set.
Mike Frysinger324cf682014-09-22 15:52:50 -04001172 for line in data:
1173 if line.startswith('repo-name = '):
1174 break
1175 else:
Mike Frysinger94a670c2014-09-19 12:46:26 -04001176 errors += ['set the board name with: repo-name = $BOARD']
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001177
Mike Frysinger94a670c2014-09-19 12:46:26 -04001178 # Summarize all the errors we saw (if any).
1179 lines = ''
1180 for layout_path, errors in all_errors.items():
1181 if errors:
1182 lines += '\n\t- '.join(['\n* %s:' % layout_path] + errors)
1183 if lines:
1184 lines = 'See the portage(5) man page for layout.conf details' + lines + '\n'
1185 return HookFailure(lines)
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001186
1187
Ryan Cuiec4d6332011-05-02 14:15:25 -07001188# Project-specific hooks
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001189
Ryan Cui1562fb82011-05-09 11:01:31 -07001190
Luis Hector Chavezb50391d2017-09-26 15:48:15 -07001191def _check_clang_format(_project, commit, options=()):
1192 """Runs clang-format on the given project"""
1193 hooks_dir = _get_hooks_dir()
1194 options = list(options)
1195 if commit == PRE_SUBMIT:
1196 options.append('--commit=HEAD')
1197 else:
1198 options.extend(['--commit', commit])
1199 cmd = ['%s/clang-format.py' % hooks_dir] + options
1200 cmd_result = cros_build_lib.RunCommand(cmd=cmd,
1201 print_cmd=False,
1202 input=_get_patch(commit),
1203 stdout_to_pipe=True,
1204 combine_stdout_stderr=True,
1205 error_code_ok=True)
1206 if cmd_result.returncode:
1207 return HookFailure('clang-format.py errors/warnings\n\n' +
1208 cmd_result.output)
1209
1210
Mike Frysingerae409522014-02-01 03:16:11 -05001211def _run_checkpatch(_project, commit, options=()):
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001212 """Runs checkpatch.pl on the given project"""
1213 hooks_dir = _get_hooks_dir()
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001214 options = list(options)
1215 if commit == PRE_SUBMIT:
1216 # The --ignore option must be present and include 'MISSING_SIGN_OFF' in
1217 # this case.
1218 options.append('--ignore=MISSING_SIGN_OFF')
Filipe Brandenburger28d48d62015-10-07 09:48:54 -07001219 # Always ignore the check for the MAINTAINERS file. We do not track that
1220 # information on that file in our source trees, so let's suppress the
1221 # warning.
1222 options.append('--ignore=FILE_PATH_CHANGES')
Filipe Brandenburgerce978322015-10-09 10:04:15 -07001223 # Do not complain about the Change-Id: fields, since we use Gerrit.
1224 # Upstream does not want those lines (since they do not use Gerrit), but
1225 # we always do, so disable the check globally.
Daisuke Nojiri4844f892015-10-08 09:54:33 -07001226 options.append('--ignore=GERRIT_CHANGE_ID')
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001227 cmd = ['%s/checkpatch.pl' % hooks_dir] + options + ['-']
Rahul Chaudhry0e515342015-08-07 12:00:43 -07001228 cmd_result = cros_build_lib.RunCommand(cmd=cmd,
1229 print_cmd=False,
1230 input=_get_patch(commit),
1231 stdout_to_pipe=True,
1232 combine_stdout_stderr=True,
1233 error_code_ok=True)
1234 if cmd_result.returncode:
1235 return HookFailure('checkpatch.pl errors/warnings\n\n' + cmd_result.output)
Ryan Cuiec4d6332011-05-02 14:15:25 -07001236
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001237
Mike Frysingerae409522014-02-01 03:16:11 -05001238def _kernel_configcheck(_project, commit):
Olof Johanssona96810f2012-09-04 16:20:03 -07001239 """Makes sure kernel config changes are not mixed with code changes"""
1240 files = _get_affected_files(commit)
1241 if not len(_filter_files(files, [r'chromeos/config'])) in [0, len(files)]:
1242 return HookFailure('Changes to chromeos/config/ and regular files must '
1243 'be in separate commits:\n%s' % '\n'.join(files))
Anton Staaf815d6852011-08-22 10:08:45 -07001244
Mike Frysingerae409522014-02-01 03:16:11 -05001245
1246def _run_json_check(_project, commit):
Dale Curtis2975c432011-05-03 17:25:20 -07001247 """Checks that all JSON files are syntactically valid."""
Dale Curtisa039cfd2011-05-04 12:01:05 -07001248 for f in _filter_files(_get_affected_files(commit), [r'.*\.json']):
Dale Curtis2975c432011-05-03 17:25:20 -07001249 try:
1250 json.load(open(f))
1251 except Exception, e:
Ryan Cui1562fb82011-05-09 11:01:31 -07001252 return HookFailure('Invalid JSON in %s: %s' % (f, e))
Dale Curtis2975c432011-05-03 17:25:20 -07001253
1254
Mike Frysingerae409522014-02-01 03:16:11 -05001255def _check_manifests(_project, commit):
Mike Frysinger52b537e2013-08-22 22:59:53 -04001256 """Make sure Manifest files only have DIST lines"""
1257 paths = []
1258
1259 for path in _get_affected_files(commit):
1260 if os.path.basename(path) != 'Manifest':
1261 continue
1262 if not os.path.exists(path):
1263 continue
1264
1265 with open(path, 'r') as f:
1266 for line in f.readlines():
1267 if not line.startswith('DIST '):
1268 paths.append(path)
1269 break
1270
1271 if paths:
1272 return HookFailure('Please remove lines that do not start with DIST:\n%s' %
1273 ('\n'.join(paths),))
1274
1275
Mike Frysingerae409522014-02-01 03:16:11 -05001276def _check_change_has_branch_field(_project, commit):
Puneet Kumarc80e3f62012-08-13 19:01:18 -07001277 """Check for a non-empty 'BRANCH=' field in the commit message."""
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001278 if commit == PRE_SUBMIT:
1279 return
Puneet Kumarc80e3f62012-08-13 19:01:18 -07001280 BRANCH_RE = r'\nBRANCH=\S+'
1281
1282 if not re.search(BRANCH_RE, _get_commit_desc(commit)):
1283 msg = ('Changelist description needs BRANCH field (after first line)\n'
1284 'E.g. BRANCH=none or BRANCH=link,snow')
1285 return HookFailure(msg)
1286
1287
Mike Frysingerae409522014-02-01 03:16:11 -05001288def _check_change_has_signoff_field(_project, commit):
Shawn Nematbakhsh51e16ac2014-01-28 15:31:07 -08001289 """Check for a non-empty 'Signed-off-by:' field in the commit message."""
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001290 if commit == PRE_SUBMIT:
1291 return
Shawn Nematbakhsh51e16ac2014-01-28 15:31:07 -08001292 SIGNOFF_RE = r'\nSigned-off-by: \S+'
1293
1294 if not re.search(SIGNOFF_RE, _get_commit_desc(commit)):
1295 msg = ('Changelist description needs Signed-off-by: field\n'
1296 'E.g. Signed-off-by: My Name <me@chromium.org>')
1297 return HookFailure(msg)
1298
1299
Aviv Keshet5ac59522017-01-31 14:28:27 -08001300def _check_cq_ini_well_formed(_project, commit):
1301 """Check that any modified COMMIT-QUEUE.ini files are well formed."""
1302 pattern = '.*' + constants.CQ_CONFIG_FILENAME
1303 files = _filter_files(_get_affected_files(commit), (pattern,))
1304
1305 # TODO(akeshet): Check not only that the file is parseable, but that all the
1306 # pre-cq configs it requests are existing ones.
1307 for f in files:
1308 try:
1309 parser = ConfigParser.SafeConfigParser()
1310 # Prior to python3, ConfigParser has no read_string method, so we must
1311 # pass it either a file path or file like object. And we must use
1312 # _get_file_content to fetch file contents to ensure we are examining the
1313 # commit diff, rather than whatever's on disk.
1314 contents = _get_file_content(f, commit)
1315 parser.readfp(StringIO.StringIO(contents))
1316 except ConfigParser.Error as e:
1317 msg = ('Unable to parse COMMIT-QUEUE.ini file at %s due to %s.' %
1318 (f, e))
1319 return HookFailure(msg)
1320
1321
Jon Salz3ee59de2012-08-18 13:54:22 +08001322def _run_project_hook_script(script, project, commit):
1323 """Runs a project hook script.
1324
1325 The script is run with the following environment variables set:
1326 PRESUBMIT_PROJECT: The affected project
1327 PRESUBMIT_COMMIT: The affected commit
1328 PRESUBMIT_FILES: A newline-separated list of affected files
1329
1330 The script is considered to fail if the exit code is non-zero. It should
1331 write an error message to stdout.
1332 """
1333 env = dict(os.environ)
Alex Deymo643ac4c2015-09-03 10:40:50 -07001334 env['PRESUBMIT_PROJECT'] = project.name
Jon Salz3ee59de2012-08-18 13:54:22 +08001335 env['PRESUBMIT_COMMIT'] = commit
1336
1337 # Put affected files in an environment variable
1338 files = _get_affected_files(commit)
1339 env['PRESUBMIT_FILES'] = '\n'.join(files)
1340
Rahul Chaudhry0e515342015-08-07 12:00:43 -07001341 cmd_result = cros_build_lib.RunCommand(cmd=script,
1342 env=env,
1343 shell=True,
1344 print_cmd=False,
1345 input=os.devnull,
1346 stdout_to_pipe=True,
1347 combine_stdout_stderr=True,
1348 error_code_ok=True)
1349 if cmd_result.returncode:
1350 stdout = cmd_result.output
Jon Salz7b618af2012-08-31 06:03:16 +08001351 if stdout:
1352 stdout = re.sub('(?m)^', ' ', stdout)
1353 return HookFailure('Hook script "%s" failed with code %d%s' %
Rahul Chaudhry0e515342015-08-07 12:00:43 -07001354 (script, cmd_result.returncode,
Jon Salz3ee59de2012-08-18 13:54:22 +08001355 ':\n' + stdout if stdout else ''))
1356
1357
Bertrand SIMONNET0022fff2014-07-07 09:52:15 -07001358def _check_project_prefix(_project, commit):
Mike Frysinger55f85b52014-12-18 14:45:21 -05001359 """Require the commit message have a project specific prefix as needed."""
Bertrand SIMONNET0022fff2014-07-07 09:52:15 -07001360
1361 files = _get_affected_files(commit, relative=True)
1362 prefix = os.path.commonprefix(files)
1363 prefix = os.path.dirname(prefix)
1364
1365 # If there is no common prefix, the CL span multiple projects.
Daniel Erata350fd32014-09-29 14:02:34 -07001366 if not prefix:
Bertrand SIMONNET0022fff2014-07-07 09:52:15 -07001367 return
1368
1369 project_name = prefix.split('/')[0]
Daniel Erata350fd32014-09-29 14:02:34 -07001370
1371 # The common files may all be within a subdirectory of the main project
1372 # directory, so walk up the tree until we find an alias file.
1373 # _get_affected_files() should return relative paths, but check against '/' to
1374 # ensure that this loop terminates even if it receives an absolute path.
1375 while prefix and prefix != '/':
1376 alias_file = os.path.join(prefix, '.project_alias')
1377
1378 # If an alias exists, use it.
1379 if os.path.isfile(alias_file):
1380 project_name = osutils.ReadFile(alias_file).strip()
1381
1382 prefix = os.path.dirname(prefix)
Bertrand SIMONNET0022fff2014-07-07 09:52:15 -07001383
1384 if not _get_commit_desc(commit).startswith(project_name + ': '):
1385 return HookFailure('The commit title for changes affecting only %s'
1386 ' should start with \"%s: \"'
1387 % (project_name, project_name))
1388
1389
Mike Frysingerf9d41b32017-02-23 15:20:04 -05001390def _check_exec_files(_project, commit):
1391 """Make +x bits on files."""
1392 # List of files that should never be +x.
1393 NO_EXEC = (
1394 'ChangeLog*',
1395 'COPYING',
1396 'make.conf',
1397 'make.defaults',
1398 'Manifest',
1399 'OWNERS',
1400 'package.use',
1401 'package.keywords',
1402 'package.mask',
1403 'parent',
1404 'README',
1405 'TODO',
1406 '.gitignore',
1407 '*.[achly]',
1408 '*.[ch]xx',
1409 '*.boto',
1410 '*.cc',
1411 '*.cfg',
1412 '*.conf',
1413 '*.config',
1414 '*.cpp',
1415 '*.css',
1416 '*.ebuild',
1417 '*.eclass',
1418 '*.gyp',
1419 '*.gypi',
1420 '*.htm',
1421 '*.html',
1422 '*.ini',
1423 '*.js',
1424 '*.json',
1425 '*.md',
1426 '*.mk',
1427 '*.patch',
1428 '*.policy',
1429 '*.proto',
1430 '*.raw',
1431 '*.rules',
1432 '*.service',
1433 '*.target',
1434 '*.txt',
1435 '*.xml',
1436 '*.yaml',
1437 )
1438
1439 def FinalName(obj):
1440 # If the file is being deleted, then the dst_file is not set.
1441 if obj.dst_file is None:
1442 return obj.src_file
1443 else:
1444 return obj.dst_file
1445
1446 bad_files = []
1447 files = _get_affected_files(commit, relative=True, full_details=True)
1448 for f in files:
1449 mode = int(f.dst_mode, 8)
1450 if not mode & 0o111:
1451 continue
1452 name = FinalName(f)
1453 for no_exec in NO_EXEC:
1454 if fnmatch.fnmatch(name, no_exec):
1455 bad_files.append(name)
1456 break
1457
1458 if bad_files:
1459 return HookFailure('These files should not be executable. '
1460 'Please `chmod -x` them.', bad_files)
1461
1462
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001463# Base
1464
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001465# A list of hooks which are not project specific and check patch description
1466# (as opposed to patch body).
1467_PATCH_DESCRIPTION_HOOKS = [
Ryan Cui9b651632011-05-11 11:38:58 -07001468 _check_change_has_bug_field,
David Jamesc3b68b32013-04-03 09:17:03 -07001469 _check_change_has_valid_cq_depend,
Ryan Cui9b651632011-05-11 11:38:58 -07001470 _check_change_has_test_field,
1471 _check_change_has_proper_changeid,
Mike Frysinger36b2ebc2014-10-31 14:02:03 -04001472 _check_commit_message_style,
Bernie Thompsonf8fea992016-01-14 10:27:18 -08001473 _check_change_is_contribution,
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001474]
1475
1476
1477# A list of hooks that are not project-specific
1478_COMMON_HOOKS = [
Aviv Keshet5ac59522017-01-31 14:28:27 -08001479 _check_cq_ini_well_formed,
1480 _check_cros_license,
Mike Frysingerbf8b91c2014-02-01 02:50:27 -05001481 _check_ebuild_eapi,
Mike Frysinger89bdb852014-02-01 05:26:26 -05001482 _check_ebuild_keywords,
Yu-Ju Hong5e0efa72013-11-19 16:28:10 -08001483 _check_ebuild_licenses,
Mike Frysingercd363c82014-02-01 05:20:18 -05001484 _check_ebuild_virtual_pv,
Mike Frysingerf9d41b32017-02-23 15:20:04 -05001485 _check_exec_files,
Daniel Erat9d203ff2015-02-17 10:12:21 -07001486 _check_for_uprev,
Rahul Chaudhry09f61372015-07-31 17:14:26 -07001487 _check_gofmt,
Mike Frysinger998c2cc2014-08-27 05:20:23 -04001488 _check_layout_conf,
Daniel Erat9d203ff2015-02-17 10:12:21 -07001489 _check_no_long_lines,
1490 _check_no_stray_whitespace,
Ryan Cui9b651632011-05-11 11:38:58 -07001491 _check_no_tabs,
Daniel Erat9d203ff2015-02-17 10:12:21 -07001492 _check_portage_make_use_var,
Aviv Keshet5ac59522017-01-31 14:28:27 -08001493 _check_tabbed_indents,
Ryan Cui9b651632011-05-11 11:38:58 -07001494]
Ryan Cuiec4d6332011-05-02 14:15:25 -07001495
Ryan Cui1562fb82011-05-09 11:01:31 -07001496
Ryan Cui9b651632011-05-11 11:38:58 -07001497# A dictionary of project-specific hooks(callbacks), indexed by project name.
1498# dict[project] = [callback1, callback2]
1499_PROJECT_SPECIFIC_HOOKS = {
Bertrand SIMONNET0022fff2014-07-07 09:52:15 -07001500 "chromiumos/platform2": [_check_project_prefix],
Mike Frysingeraf891292015-03-25 19:46:53 -04001501 "chromiumos/third_party/kernel": [_kernel_configcheck],
1502 "chromiumos/third_party/kernel-next": [_kernel_configcheck],
Ryan Cui9b651632011-05-11 11:38:58 -07001503}
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001504
Ryan Cui1562fb82011-05-09 11:01:31 -07001505
Ryan Cui9b651632011-05-11 11:38:58 -07001506# A dictionary of flags (keys) that can appear in the config file, and the hook
Mike Frysinger3554bc92015-03-11 04:59:21 -04001507# that the flag controls (value).
1508_HOOK_FLAGS = {
Luis Hector Chavezb50391d2017-09-26 15:48:15 -07001509 'clang_format_check': _check_clang_format,
Mike Frysingera7642f52015-03-25 18:31:42 -04001510 'checkpatch_check': _run_checkpatch,
Ryan Cui9b651632011-05-11 11:38:58 -07001511 'stray_whitespace_check': _check_no_stray_whitespace,
Mike Frysingerff6c7d62015-03-24 13:49:46 -04001512 'json_check': _run_json_check,
Ryan Cui9b651632011-05-11 11:38:58 -07001513 'long_line_check': _check_no_long_lines,
Alex Deymof5792ce2015-08-24 22:50:08 -07001514 'cros_license_check': _check_cros_license,
1515 'aosp_license_check': _check_aosp_license,
Ryan Cui9b651632011-05-11 11:38:58 -07001516 'tab_check': _check_no_tabs,
Prathmesh Prabhuc5254652016-12-22 12:58:05 -08001517 'tabbed_indent_required_check': _check_tabbed_indents,
Puneet Kumarc80e3f62012-08-13 19:01:18 -07001518 'branch_check': _check_change_has_branch_field,
Shawn Nematbakhsh51e16ac2014-01-28 15:31:07 -08001519 'signoff_check': _check_change_has_signoff_field,
Josh Triplett0e8fc7f2014-04-23 16:00:00 -07001520 'bug_field_check': _check_change_has_bug_field,
1521 'test_field_check': _check_change_has_test_field,
Steve Fung49ec7e92015-03-23 16:07:12 -07001522 'manifest_check': _check_manifests,
Bernie Thompsonf8fea992016-01-14 10:27:18 -08001523 'contribution_check': _check_change_is_contribution,
Ryan Cui9b651632011-05-11 11:38:58 -07001524}
1525
1526
Mike Frysinger3554bc92015-03-11 04:59:21 -04001527def _get_override_hooks(config):
1528 """Returns a set of hooks controlled by the current project's config file.
Ryan Cui9b651632011-05-11 11:38:58 -07001529
1530 Expects to be called within the project root.
Jon Salz3ee59de2012-08-18 13:54:22 +08001531
1532 Args:
1533 config: A ConfigParser for the project's config file.
Ryan Cui9b651632011-05-11 11:38:58 -07001534 """
1535 SECTION = 'Hook Overrides'
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001536 SECTION_OPTIONS = 'Hook Overrides Options'
Jon Salz3ee59de2012-08-18 13:54:22 +08001537 if not config.has_section(SECTION):
Mike Frysinger3554bc92015-03-11 04:59:21 -04001538 return set(), set()
Ryan Cui9b651632011-05-11 11:38:58 -07001539
Mike Frysinger3554bc92015-03-11 04:59:21 -04001540 valid_keys = set(_HOOK_FLAGS.iterkeys())
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001541 hooks = _HOOK_FLAGS.copy()
Mike Frysinger3554bc92015-03-11 04:59:21 -04001542
1543 enable_flags = []
Ryan Cui9b651632011-05-11 11:38:58 -07001544 disable_flags = []
Jon Salz3ee59de2012-08-18 13:54:22 +08001545 for flag in config.options(SECTION):
Mike Frysinger3554bc92015-03-11 04:59:21 -04001546 if flag not in valid_keys:
1547 raise ValueError('Error: unknown key "%s" in hook section of "%s"' %
1548 (flag, _CONFIG_FILE))
1549
Ryan Cui9b651632011-05-11 11:38:58 -07001550 try:
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001551 enabled = config.getboolean(SECTION, flag)
Ryan Cui9b651632011-05-11 11:38:58 -07001552 except ValueError as e:
Mike Frysinger3554bc92015-03-11 04:59:21 -04001553 raise ValueError('Error: parsing flag "%s" in "%s" failed: %s' %
1554 (flag, _CONFIG_FILE, e))
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001555 if enabled:
1556 enable_flags.append(flag)
1557 else:
1558 disable_flags.append(flag)
Ryan Cui9b651632011-05-11 11:38:58 -07001559
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001560 # See if this hook has custom options.
1561 if enabled:
1562 try:
1563 options = config.get(SECTION_OPTIONS, flag)
1564 hooks[flag] = functools.partial(hooks[flag], options=options.split())
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001565 hooks[flag].__name__ = flag
Mike Frysingerf8ce1712015-03-25 18:32:33 -04001566 except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
1567 pass
1568
1569 enabled_hooks = set(hooks[x] for x in enable_flags)
1570 disabled_hooks = set(hooks[x] for x in disable_flags)
Mike Frysinger3554bc92015-03-11 04:59:21 -04001571 return enabled_hooks, disabled_hooks
Ryan Cui9b651632011-05-11 11:38:58 -07001572
1573
Jon Salz3ee59de2012-08-18 13:54:22 +08001574def _get_project_hook_scripts(config):
1575 """Returns a list of project-specific hook scripts.
1576
1577 Args:
1578 config: A ConfigParser for the project's config file.
1579 """
1580 SECTION = 'Hook Scripts'
1581 if not config.has_section(SECTION):
1582 return []
1583
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001584 return config.items(SECTION)
Jon Salz3ee59de2012-08-18 13:54:22 +08001585
1586
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001587def _get_project_hooks(project, presubmit):
Ryan Cui9b651632011-05-11 11:38:58 -07001588 """Returns a list of hooks that need to be run for a project.
1589
1590 Expects to be called from within the project root.
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001591
1592 Args:
1593 project: A string, name of the project.
1594 presubmit: A Boolean, True if the check is run as a git pre-submit script.
Ryan Cui9b651632011-05-11 11:38:58 -07001595 """
Jon Salz3ee59de2012-08-18 13:54:22 +08001596 config = ConfigParser.RawConfigParser()
1597 try:
1598 config.read(_CONFIG_FILE)
1599 except ConfigParser.Error:
1600 # Just use an empty config file
1601 config = ConfigParser.RawConfigParser()
1602
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001603 if presubmit:
Filipe Brandenburgerf70d32c2015-10-09 13:35:45 -07001604 hooks = _COMMON_HOOKS
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001605 else:
Filipe Brandenburgerf70d32c2015-10-09 13:35:45 -07001606 hooks = _PATCH_DESCRIPTION_HOOKS + _COMMON_HOOKS
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001607
Mike Frysinger3554bc92015-03-11 04:59:21 -04001608 enabled_hooks, disabled_hooks = _get_override_hooks(config)
Filipe Brandenburgerf70d32c2015-10-09 13:35:45 -07001609 hooks = [hook for hook in hooks if hook not in disabled_hooks]
1610
1611 # If a list is both in _COMMON_HOOKS and also enabled explicitly through an
1612 # override, keep the override only. Note that the override may end up being
1613 # a functools.partial, in which case we need to extract the .func to compare
1614 # it to the common hooks.
1615 unwrapped_hooks = [getattr(hook, 'func', hook) for hook in enabled_hooks]
1616 hooks = [hook for hook in hooks if hook not in unwrapped_hooks]
1617
1618 hooks = list(enabled_hooks) + hooks
Ryan Cui9b651632011-05-11 11:38:58 -07001619
1620 if project in _PROJECT_SPECIFIC_HOOKS:
Puneet Kumarc80e3f62012-08-13 19:01:18 -07001621 hooks.extend(hook for hook in _PROJECT_SPECIFIC_HOOKS[project]
1622 if hook not in disabled_hooks)
Ryan Cui9b651632011-05-11 11:38:58 -07001623
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001624 for name, script in _get_project_hook_scripts(config):
1625 func = functools.partial(_run_project_hook_script, script)
1626 func.__name__ = name
1627 hooks.append(func)
Jon Salz3ee59de2012-08-18 13:54:22 +08001628
Ryan Cui9b651632011-05-11 11:38:58 -07001629 return hooks
1630
1631
Alex Deymo643ac4c2015-09-03 10:40:50 -07001632def _run_project_hooks(project_name, proj_dir=None,
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001633 commit_list=None, presubmit=False):
Ryan Cui1562fb82011-05-09 11:01:31 -07001634 """For each project run its project specific hook from the hooks dictionary.
1635
1636 Args:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001637 project_name: The name of project to run hooks for.
Doug Anderson44a644f2011-11-02 10:37:37 -07001638 proj_dir: If non-None, this is the directory the project is in. If None,
1639 we'll ask repo.
Doug Anderson14749562013-06-26 13:38:29 -07001640 commit_list: A list of commits to run hooks against. If None or empty list
1641 then we'll automatically get the list of commits that would be uploaded.
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001642 presubmit: A Boolean, True if the check is run as a git pre-submit script.
Ryan Cui1562fb82011-05-09 11:01:31 -07001643
1644 Returns:
1645 Boolean value of whether any errors were ecountered while running the hooks.
1646 """
Doug Anderson44a644f2011-11-02 10:37:37 -07001647 if proj_dir is None:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001648 proj_dirs = _run_command(
1649 ['repo', 'forall', project_name, '-c', 'pwd']).split()
David James2edd9002013-10-11 14:09:19 -07001650 if len(proj_dirs) == 0:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001651 print('%s cannot be found.' % project_name, file=sys.stderr)
David James2edd9002013-10-11 14:09:19 -07001652 print('Please specify a valid project.', file=sys.stderr)
1653 return True
1654 if len(proj_dirs) > 1:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001655 print('%s is associated with multiple directories.' % project_name,
David James2edd9002013-10-11 14:09:19 -07001656 file=sys.stderr)
1657 print('Please specify a directory to help disambiguate.', file=sys.stderr)
1658 return True
1659 proj_dir = proj_dirs[0]
Doug Anderson44a644f2011-11-02 10:37:37 -07001660
Ryan Cuiec4d6332011-05-02 14:15:25 -07001661 pwd = os.getcwd()
1662 # hooks assume they are run from the root of the project
1663 os.chdir(proj_dir)
1664
Alex Deymo643ac4c2015-09-03 10:40:50 -07001665 remote_branch = _run_command(['git', 'rev-parse', '--abbrev-ref',
1666 '--symbolic-full-name', '@{u}']).strip()
1667 if not remote_branch:
1668 print('Your project %s doesn\'t track any remote repo.' % project_name,
1669 file=sys.stderr)
1670 remote = None
1671 else:
1672 remote, _branch = remote_branch.split('/', 1)
1673
1674 project = Project(name=project_name, dir=proj_dir, remote=remote)
1675
Doug Anderson14749562013-06-26 13:38:29 -07001676 if not commit_list:
1677 try:
1678 commit_list = _get_commits()
1679 except VerifyException as e:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001680 PrintErrorForProject(project.name, HookFailure(str(e)))
Doug Anderson14749562013-06-26 13:38:29 -07001681 os.chdir(pwd)
1682 return True
Ryan Cuifa55df52011-05-06 11:16:55 -07001683
Alex Deymo643ac4c2015-09-03 10:40:50 -07001684 hooks = _get_project_hooks(project.name, presubmit)
Ryan Cui1562fb82011-05-09 11:01:31 -07001685 error_found = False
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001686 commit_count = len(commit_list)
1687 for i, commit in enumerate(commit_list):
Ryan Cui1562fb82011-05-09 11:01:31 -07001688 error_list = []
Ryan Cui9b651632011-05-11 11:38:58 -07001689 for hook in hooks:
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001690 output = ('PRESUBMIT.cfg: [%i/%i]: %s: Running %s' %
1691 (i + 1, commit_count, commit, hook.__name__))
1692 print(output, end='\r')
1693 sys.stdout.flush()
Ryan Cui1562fb82011-05-09 11:01:31 -07001694 hook_error = hook(project, commit)
Mike Frysingerb7d552e2017-11-23 11:50:47 -05001695 print(' ' * len(output), end='\r')
1696 sys.stdout.flush()
Ryan Cui1562fb82011-05-09 11:01:31 -07001697 if hook_error:
1698 error_list.append(hook_error)
1699 error_found = True
1700 if error_list:
Alex Deymo643ac4c2015-09-03 10:40:50 -07001701 PrintErrorsForCommit(project.name, commit, _get_commit_desc(commit),
Ryan Cui1562fb82011-05-09 11:01:31 -07001702 error_list)
Don Garrettdba548a2011-05-05 15:17:14 -07001703
Ryan Cuiec4d6332011-05-02 14:15:25 -07001704 os.chdir(pwd)
Ryan Cui1562fb82011-05-09 11:01:31 -07001705 return error_found
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001706
Mike Frysingerae409522014-02-01 03:16:11 -05001707
Mandeep Singh Baines116ad102011-04-27 15:16:37 -07001708# Main
Mandeep Singh Baines69e470e2011-04-06 10:34:52 -07001709
Ryan Cui1562fb82011-05-09 11:01:31 -07001710
Mike Frysingerae409522014-02-01 03:16:11 -05001711def main(project_list, worktree_list=None, **_kwargs):
Doug Anderson06456632012-01-05 11:02:14 -08001712 """Main function invoked directly by repo.
1713
1714 This function will exit directly upon error so that repo doesn't print some
1715 obscure error message.
1716
1717 Args:
1718 project_list: List of projects to run on.
David James2edd9002013-10-11 14:09:19 -07001719 worktree_list: A list of directories. It should be the same length as
1720 project_list, so that each entry in project_list matches with a directory
1721 in worktree_list. If None, we will attempt to calculate the directories
1722 automatically.
Doug Anderson06456632012-01-05 11:02:14 -08001723 kwargs: Leave this here for forward-compatibility.
1724 """
Ryan Cui1562fb82011-05-09 11:01:31 -07001725 found_error = False
David James2edd9002013-10-11 14:09:19 -07001726 if not worktree_list:
1727 worktree_list = [None] * len(project_list)
1728 for project, worktree in zip(project_list, worktree_list):
1729 if _run_project_hooks(project, proj_dir=worktree):
Ryan Cui1562fb82011-05-09 11:01:31 -07001730 found_error = True
1731
Mike Frysingerae409522014-02-01 03:16:11 -05001732 if found_error:
Ryan Cui1562fb82011-05-09 11:01:31 -07001733 msg = ('Preupload failed due to errors in project(s). HINTS:\n'
Ryan Cui9b651632011-05-11 11:38:58 -07001734 '- To disable some source style checks, and for other hints, see '
1735 '<checkout_dir>/src/repohooks/README\n'
1736 '- To upload only current project, run \'repo upload .\'')
Mike Frysinger09d6a3d2013-10-08 22:21:03 -04001737 print(msg, file=sys.stderr)
Don Garrettdba548a2011-05-05 15:17:14 -07001738 sys.exit(1)
Anush Elangovan63afad72011-03-23 00:41:27 -07001739
Ryan Cui1562fb82011-05-09 11:01:31 -07001740
Doug Anderson44a644f2011-11-02 10:37:37 -07001741def _identify_project(path):
1742 """Identify the repo project associated with the given path.
1743
1744 Returns:
1745 A string indicating what project is associated with the path passed in or
1746 a blank string upon failure.
1747 """
1748 return _run_command(['repo', 'forall', '.', '-c', 'echo ${REPO_PROJECT}'],
Rahul Chaudhry0e515342015-08-07 12:00:43 -07001749 redirect_stderr=True, cwd=path).strip()
Doug Anderson44a644f2011-11-02 10:37:37 -07001750
1751
Mike Frysinger55f85b52014-12-18 14:45:21 -05001752def direct_main(argv):
Doug Anderson44a644f2011-11-02 10:37:37 -07001753 """Run hooks directly (outside of the context of repo).
1754
Doug Anderson44a644f2011-11-02 10:37:37 -07001755 Args:
Mike Frysinger55f85b52014-12-18 14:45:21 -05001756 argv: The command line args to process
Doug Anderson44a644f2011-11-02 10:37:37 -07001757
1758 Returns:
1759 0 if no pre-upload failures, 1 if failures.
1760
1761 Raises:
1762 BadInvocation: On some types of invocation errors.
1763 """
Mike Frysinger66142932014-12-18 14:55:57 -05001764 parser = commandline.ArgumentParser(description=__doc__)
1765 parser.add_argument('--dir', default=None,
1766 help='The directory that the project lives in. If not '
1767 'specified, use the git project root based on the cwd.')
1768 parser.add_argument('--project', default=None,
1769 help='The project repo path; this can affect how the '
1770 'hooks get run, since some hooks are project-specific. '
1771 'For chromite this is chromiumos/chromite. If not '
1772 'specified, the repo tool will be used to figure this '
1773 'out based on the dir.')
1774 parser.add_argument('--rerun-since', default=None,
1775 help='Rerun hooks on old commits since the given date. '
1776 'The date should match git log\'s concept of a date. '
1777 'e.g. 2012-06-20. This option is mutually exclusive '
1778 'with --pre-submit.')
1779 parser.add_argument('--pre-submit', action="store_true",
1780 help='Run the check against the pending commit. '
1781 'This option should be used at the \'git commit\' '
1782 'phase as opposed to \'repo upload\'. This option '
1783 'is mutually exclusive with --rerun-since.')
1784 parser.add_argument('commits', nargs='*',
1785 help='Check specific commits')
1786 opts = parser.parse_args(argv)
Doug Anderson44a644f2011-11-02 10:37:37 -07001787
Doug Anderson14749562013-06-26 13:38:29 -07001788 if opts.rerun_since:
Mike Frysinger66142932014-12-18 14:55:57 -05001789 if opts.commits:
Doug Anderson14749562013-06-26 13:38:29 -07001790 raise BadInvocation('Can\'t pass commits and use rerun-since: %s' %
Mike Frysinger66142932014-12-18 14:55:57 -05001791 ' '.join(opts.commits))
Doug Anderson14749562013-06-26 13:38:29 -07001792
1793 cmd = ['git', 'log', '--since="%s"' % opts.rerun_since, '--pretty=%H']
1794 all_commits = _run_command(cmd).splitlines()
1795 bot_commits = _run_command(cmd + ['--author=chrome-bot']).splitlines()
1796
1797 # Eliminate chrome-bot commits but keep ordering the same...
1798 bot_commits = set(bot_commits)
Mike Frysinger66142932014-12-18 14:55:57 -05001799 opts.commits = [c for c in all_commits if c not in bot_commits]
Doug Anderson14749562013-06-26 13:38:29 -07001800
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001801 if opts.pre_submit:
1802 raise BadInvocation('rerun-since and pre-submit can not be '
1803 'used together')
1804 if opts.pre_submit:
Mike Frysinger66142932014-12-18 14:55:57 -05001805 if opts.commits:
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001806 raise BadInvocation('Can\'t pass commits and use pre-submit: %s' %
Mike Frysinger66142932014-12-18 14:55:57 -05001807 ' '.join(opts.commits))
1808 opts.commits = [PRE_SUBMIT,]
Doug Anderson44a644f2011-11-02 10:37:37 -07001809
1810 # Check/normlaize git dir; if unspecified, we'll use the root of the git
1811 # project from CWD
1812 if opts.dir is None:
1813 git_dir = _run_command(['git', 'rev-parse', '--git-dir'],
Rahul Chaudhry0e515342015-08-07 12:00:43 -07001814 redirect_stderr=True).strip()
Doug Anderson44a644f2011-11-02 10:37:37 -07001815 if not git_dir:
1816 raise BadInvocation('The current directory is not part of a git project.')
1817 opts.dir = os.path.dirname(os.path.abspath(git_dir))
1818 elif not os.path.isdir(opts.dir):
1819 raise BadInvocation('Invalid dir: %s' % opts.dir)
1820 elif not os.path.isdir(os.path.join(opts.dir, '.git')):
1821 raise BadInvocation('Not a git directory: %s' % opts.dir)
1822
1823 # Identify the project if it wasn't specified; this _requires_ the repo
1824 # tool to be installed and for the project to be part of a repo checkout.
1825 if not opts.project:
1826 opts.project = _identify_project(opts.dir)
1827 if not opts.project:
1828 raise BadInvocation("Repo couldn't identify the project of %s" % opts.dir)
1829
Doug Anderson14749562013-06-26 13:38:29 -07001830 found_error = _run_project_hooks(opts.project, proj_dir=opts.dir,
Mike Frysinger66142932014-12-18 14:55:57 -05001831 commit_list=opts.commits,
Vadim Bendebury2b62d742014-06-22 13:14:51 -07001832 presubmit=opts.pre_submit)
Doug Anderson44a644f2011-11-02 10:37:37 -07001833 if found_error:
1834 return 1
1835 return 0
1836
1837
Mandeep Singh Baines69e470e2011-04-06 10:34:52 -07001838if __name__ == '__main__':
Mike Frysinger55f85b52014-12-18 14:45:21 -05001839 sys.exit(direct_main(sys.argv[1:]))