presubmit_canned_checks.py

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Generic presubmit checks that can be reused by other presubmit checks."""

import os as _os
_HERE = _os.path.dirname(_os.path.abspath(__file__))

# Justifications for each filter:
#
# - build/include       : Too many; fix in the future.
# - build/include_order : Not happening; #ifdefed includes.
# - build/namespace     : I'm surprised by how often we violate this rule.
# - readability/casting : Mistakes a whole bunch of function pointer.
# - runtime/int         : Can be fixed long term; volume of errors too high
# - runtime/virtual     : Broken now, but can be fixed in the future?
# - whitespace/braces   : We have a lot of explicit scoping in chrome code.
DEFAULT_LINT_FILTERS = [
  '-build/include',
  '-build/include_order',
  '-build/namespace',
  '-readability/casting',
  '-runtime/int',
  '-runtime/virtual',
  '-whitespace/braces',
]

# These filters will always be removed, even if the caller specifies a filter
# set, as they are problematic or broken in some way.
#
# Justifications for each filter:
# - build/c++11         : Rvalue ref checks are unreliable (false positives),
#                         include file and feature blacklists are
#                         google3-specific.
BLACKLIST_LINT_FILTERS = [
  '-build/c++11',
]

### Description checks

def CheckChangeHasTestField(input_api, output_api):
  """Requires that the changelist have a TEST= field."""
  if input_api.change.TEST:
    return []
  else:
    return [output_api.PresubmitNotifyResult(
        'If this change requires manual test instructions to QA team, add '
        'TEST=[instructions].')]


def CheckChangeHasBugField(input_api, output_api):
  """Requires that the changelist have a BUG= field."""
  if input_api.change.BUG:
    return []
  else:
    return [output_api.PresubmitNotifyResult(
        'If this change has an associated bug, add BUG=[bug number].')]


def CheckChangeHasTestedField(input_api, output_api):
  """Requires that the changelist have a TESTED= field."""
  if input_api.change.TESTED:
    return []
  else:
    return [output_api.PresubmitError('Changelist must have a TESTED= field.')]


def CheckChangeHasQaField(input_api, output_api):
  """Requires that the changelist have a QA= field."""
  if input_api.change.QA:
    return []
  else:
    return [output_api.PresubmitError('Changelist must have a QA= field.')]


def CheckDoNotSubmitInDescription(input_api, output_api):
  """Checks that the user didn't add 'DO NOT ''SUBMIT' to the CL description.
  """
  keyword = 'DO NOT ''SUBMIT'
  if keyword in input_api.change.DescriptionText():
    return [output_api.PresubmitError(
        keyword + ' is present in the changelist description.')]
  else:
    return []


def CheckChangeHasDescription(input_api, output_api):
  """Checks the CL description is not empty."""
  text = input_api.change.DescriptionText()
  if text.strip() == '':
    if input_api.is_committing:
      return [output_api.PresubmitError('Add a description to the CL.')]
    else:
      return [output_api.PresubmitNotifyResult('Add a description to the CL.')]
  return []


def CheckChangeWasUploaded(input_api, output_api):
  """Checks that the issue was uploaded before committing."""
  if input_api.is_committing and not input_api.change.issue:
    return [output_api.PresubmitError(
      'Issue wasn\'t uploaded. Please upload first.')]
  return []


### Content checks

def CheckDoNotSubmitInFiles(input_api, output_api):
  """Checks that the user didn't add 'DO NOT ''SUBMIT' to any files."""
  # We want to check every text file, not just source files.
  file_filter = lambda x : x
  keyword = 'DO NOT ''SUBMIT'
  errors = _FindNewViolationsOfRule(lambda _, line : keyword not in line,
                                    input_api, file_filter)
  text = '\n'.join('Found %s in %s' % (keyword, loc) for loc in errors)
  if text:
    return [output_api.PresubmitError(text)]
  return []


def CheckChangeLintsClean(input_api, output_api, source_file_filter=None,
                          lint_filters=None, verbose_level=None):
  """Checks that all '.cc' and '.h' files pass cpplint.py."""
  _RE_IS_TEST = input_api.re.compile(r'.*tests?.(cc|h)$')
  result = []

  cpplint = input_api.cpplint
  # Access to a protected member _XX of a client class
  # pylint: disable=W0212
  cpplint._cpplint_state.ResetErrorCounts()

  lint_filters = lint_filters or DEFAULT_LINT_FILTERS
  lint_filters.extend(BLACKLIST_LINT_FILTERS)
  cpplint._SetFilters(','.join(lint_filters))

  # We currently are more strict with normal code than unit tests; 4 and 5 are
  # the verbosity level that would normally be passed to cpplint.py through
  # --verbose=#. Hopefully, in the future, we can be more verbose.
  files = [f.AbsoluteLocalPath() for f in
           input_api.AffectedSourceFiles(source_file_filter)]
  for file_name in files:
    if _RE_IS_TEST.match(file_name):
      level = 5
    else:
      level = 4

    verbose_level = verbose_level or level
    cpplint.ProcessFile(file_name, verbose_level)

  if cpplint._cpplint_state.error_count > 0:
    if input_api.is_committing:
      res_type = output_api.PresubmitError
    else:
      res_type = output_api.PresubmitPromptWarning
    result = [res_type('Changelist failed cpplint.py check.')]

  return result


def CheckChangeHasNoCR(input_api, output_api, source_file_filter=None):
  """Checks no '\r' (CR) character is in any source files."""
  cr_files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    if '\r' in input_api.ReadFile(f, 'rb'):
      cr_files.append(f.LocalPath())
  if cr_files:
    return [output_api.PresubmitPromptWarning(
        'Found a CR character in these files:', items=cr_files)]
  return []


def CheckChangeHasOnlyOneEol(input_api, output_api, source_file_filter=None):
  """Checks the files ends with one and only one \n (LF)."""
  eof_files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    contents = input_api.ReadFile(f, 'rb')
    # Check that the file ends in one and only one newline character.
    if len(contents) > 1 and (contents[-1:] != '\n' or contents[-2:-1] == '\n'):
      eof_files.append(f.LocalPath())

  if eof_files:
    return [output_api.PresubmitPromptWarning(
      'These files should end in one (and only one) newline character:',
      items=eof_files)]
  return []


def CheckChangeHasNoCrAndHasOnlyOneEol(input_api, output_api,
                                       source_file_filter=None):
  """Runs both CheckChangeHasNoCR and CheckChangeHasOnlyOneEOL in one pass.

  It is faster because it is reading the file only once.
  """
  cr_files = []
  eof_files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    contents = input_api.ReadFile(f, 'rb')
    if '\r' in contents:
      cr_files.append(f.LocalPath())
    # Check that the file ends in one and only one newline character.
    if len(contents) > 1 and (contents[-1:] != '\n' or contents[-2:-1] == '\n'):
      eof_files.append(f.LocalPath())
  outputs = []
  if cr_files:
    outputs.append(output_api.PresubmitPromptWarning(
        'Found a CR character in these files:', items=cr_files))
  if eof_files:
    outputs.append(output_api.PresubmitPromptWarning(
      'These files should end in one (and only one) newline character:',
      items=eof_files))
  return outputs

def CheckGenderNeutral(input_api, output_api, source_file_filter=None):
  """Checks that there are no gendered pronouns in any of the text files to be
  submitted.
  """
  gendered_re = input_api.re.compile(
      '(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')

  errors = []
  for f in input_api.AffectedFiles(include_deletes=False,
                                   file_filter=source_file_filter):
    for line_num, line in f.ChangedContents():
      if gendered_re.search(line):
        errors.append('%s (%d): %s' % (f.LocalPath(), line_num, line))

  if len(errors):
    return [output_api.PresubmitPromptWarning('Found a gendered pronoun in:',
                                              long_text='\n'.join(errors))]
  return []



def _ReportErrorFileAndLine(filename, line_num, dummy_line):
  """Default error formatter for _FindNewViolationsOfRule."""
  return '%s:%s' % (filename, line_num)


def _FindNewViolationsOfRule(callable_rule, input_api, source_file_filter=None,
                             error_formatter=_ReportErrorFileAndLine):
  """Find all newly introduced violations of a per-line rule (a callable).

  Arguments:
    callable_rule: a callable taking a file extension and line of input and
      returning True if the rule is satisfied and False if there was a problem.
    input_api: object to enumerate the affected files.
    source_file_filter: a filter to be passed to the input api.
    error_formatter: a callable taking (filename, line_number, line) and
      returning a formatted error string.

  Returns:
    A list of the newly-introduced violations reported by the rule.
  """
  errors = []
  for f in input_api.AffectedFiles(include_deletes=False,
                                   file_filter=source_file_filter):
    # For speed, we do two passes, checking first the full file.  Shelling out
    # to the SCM to determine the changed region can be quite expensive on
    # Win32.  Assuming that most files will be kept problem-free, we can
    # skip the SCM operations most of the time.
    extension = str(f.LocalPath()).rsplit('.', 1)[-1]
    if all(callable_rule(extension, line) for line in f.NewContents()):
      continue  # No violation found in full text: can skip considering diff.

    for line_num, line in f.ChangedContents():
      if not callable_rule(extension, line):
        errors.append(error_formatter(f.LocalPath(), line_num, line))

  return errors


def CheckChangeHasNoTabs(input_api, output_api, source_file_filter=None):
  """Checks that there are no tab characters in any of the text files to be
  submitted.
  """
  # In addition to the filter, make sure that makefiles are blacklisted.
  if not source_file_filter:
    # It's the default filter.
    source_file_filter = input_api.FilterSourceFile
  def filter_more(affected_file):
    basename = input_api.os_path.basename(affected_file.LocalPath())
    return (not (basename in ('Makefile', 'makefile') or
                 basename.endswith('.mk')) and
            source_file_filter(affected_file))

  tabs = _FindNewViolationsOfRule(lambda _, line : '\t' not in line,
                                  input_api, filter_more)

  if tabs:
    return [output_api.PresubmitPromptWarning('Found a tab character in:',
                                              long_text='\n'.join(tabs))]
  return []


def CheckChangeTodoHasOwner(input_api, output_api, source_file_filter=None):
  """Checks that the user didn't add TODO(name) without an owner."""

  unowned_todo = input_api.re.compile('TO''DO[^(]')
  errors = _FindNewViolationsOfRule(lambda _, x : not unowned_todo.search(x),
                                    input_api, source_file_filter)
  errors = ['Found TO''DO with no owner in ' + x for x in errors]
  if errors:
    return [output_api.PresubmitPromptWarning('\n'.join(errors))]
  return []


def CheckChangeHasNoStrayWhitespace(input_api, output_api,
                                    source_file_filter=None):
  """Checks that there is no stray whitespace at source lines end."""
  errors = _FindNewViolationsOfRule(lambda _, line : line.rstrip() == line,
                                    input_api, source_file_filter)
  if errors:
    return [output_api.PresubmitPromptWarning(
        'Found line ending with white spaces in:',
        long_text='\n'.join(errors))]
  return []


def CheckLongLines(input_api, output_api, maxlen, source_file_filter=None):
  """Checks that there aren't any lines longer than maxlen characters in any of
  the text files to be submitted.
  """
  maxlens = {
      'java': 100,
      # This is specifically for Android's handwritten makefiles (Android.mk).
      'mk': 200,
      '': maxlen,
  }

  # Language specific exceptions to max line length.
  # '.h' is considered an obj-c file extension, since OBJC_EXCEPTIONS are a
  # superset of CPP_EXCEPTIONS.
  CPP_FILE_EXTS = ('c', 'cc')
  CPP_EXCEPTIONS = ('#define', '#endif', '#if', '#include', '#pragma')
  JAVA_FILE_EXTS = ('java',)
  JAVA_EXCEPTIONS = ('import ', 'package ')
  JS_FILE_EXTS = ('js',)
  JS_EXCEPTIONS = ("GEN('#include",)
  OBJC_FILE_EXTS = ('h', 'm', 'mm')
  OBJC_EXCEPTIONS = ('#define', '#endif', '#if', '#import', '#include',
                     '#pragma')
  PY_FILE_EXTS = ('py',)
  PY_EXCEPTIONS = ('import', 'from')

  LANGUAGE_EXCEPTIONS = [
    (CPP_FILE_EXTS, CPP_EXCEPTIONS),
    (JAVA_FILE_EXTS, JAVA_EXCEPTIONS),
    (JS_FILE_EXTS, JS_EXCEPTIONS),
    (OBJC_FILE_EXTS, OBJC_EXCEPTIONS),
    (PY_FILE_EXTS, PY_EXCEPTIONS),
  ]

  def no_long_lines(file_extension, line):
    # Check for language specific exceptions.
    if any(file_extension in exts and line.startswith(exceptions)
           for exts, exceptions in LANGUAGE_EXCEPTIONS):
      return True

    file_maxlen = maxlens.get(file_extension, maxlens[''])
    # Stupidly long symbols that needs to be worked around if takes 66% of line.
    long_symbol = file_maxlen * 2 / 3
    # Hard line length limit at 50% more.
    extra_maxlen = file_maxlen * 3 / 2

    line_len = len(line)
    if line_len <= file_maxlen:
      return True

    # Allow long URLs of any length.
    if any((url in line) for url in ('file://', 'http://', 'https://')):
      return True

    # If 'line-too-long' is explicitly suppressed for the line, any length is
    # acceptable.
    if 'pylint: disable=line-too-long' in line and file_extension == 'py':
      return True

    if line_len > extra_maxlen:
      return False

    if 'url(' in line and file_extension == 'css':
      return True

    if '<include' in line and file_extension in ('css', 'html', 'js'):
      return True

    return input_api.re.match(
        r'.*[A-Za-z][A-Za-z_0-9]{%d,}.*' % long_symbol, line)

  def format_error(filename, line_num, line):
    return '%s, line %s, %s chars' % (filename, line_num, len(line))

  errors = _FindNewViolationsOfRule(no_long_lines, input_api,
                                    source_file_filter,
                                    error_formatter=format_error)
  if errors:
    msg = 'Found lines longer than %s characters (first 5 shown).' % maxlen
    return [output_api.PresubmitPromptWarning(msg, items=errors[:5])]
  else:
    return []


def CheckLicense(input_api, output_api, license_re, source_file_filter=None,
    accept_empty_files=True):
  """Verifies the license header.
  """
  license_re = input_api.re.compile(license_re, input_api.re.MULTILINE)
  bad_files = []
  for f in input_api.AffectedSourceFiles(source_file_filter):
    contents = input_api.ReadFile(f, 'rb')
    if accept_empty_files and not contents:
      continue
    if not license_re.search(contents):
      bad_files.append(f.LocalPath())
  if bad_files:
    return [output_api.PresubmitPromptWarning(
        'License must match:\n%s\n' % license_re.pattern +
        'Found a bad license header in these files:', items=bad_files)]
  return []


def CheckChangeSvnEolStyle(input_api, output_api, source_file_filter=None):
  """Checks that the source files have svn:eol-style=LF."""
  return CheckSvnProperty(input_api, output_api,
                          'svn:eol-style', 'LF',
                          input_api.AffectedSourceFiles(source_file_filter))


def CheckSvnForCommonMimeTypes(input_api, output_api):
  """Checks that common binary file types have the correct svn:mime-type."""
  output = []
  files = input_api.AffectedFiles(include_deletes=False)
  def IsExts(x, exts):
    path = x.LocalPath()
    for extension in exts:
      if path.endswith(extension):
        return True
    return False
  def FilterFiles(extension):
    return filter(lambda x: IsExts(x, extension), files)
  def RunCheck(mime_type, files):
    output.extend(CheckSvnProperty(input_api, output_api, 'svn:mime-type',
                                   mime_type, files))
  RunCheck('application/pdf', FilterFiles(['.pdf']))
  RunCheck('image/bmp', FilterFiles(['.bmp']))
  RunCheck('image/gif', FilterFiles(['.gif']))
  RunCheck('image/png', FilterFiles(['.png']))
  RunCheck('image/jpeg', FilterFiles(['.jpg', '.jpeg', '.jpe']))
  RunCheck('image/vnd.microsoft.icon', FilterFiles(['.ico']))
  return output


def CheckSvnProperty(input_api, output_api, prop, expected, affected_files):
  """Checks that affected_files files have prop=expected."""
  if input_api.change.scm != 'svn':
    return []

  bad = filter(lambda f: f.Property(prop) != expected, affected_files)
  if bad:
    if input_api.is_committing:
      res_type = output_api.PresubmitError
    else:
      res_type = output_api.PresubmitNotifyResult
    message = 'Run the command: svn pset %s %s \\' % (prop, expected)
    return [res_type(message, items=bad)]
  return []


### Other checks

def CheckDoNotSubmit(input_api, output_api):
  return (
      CheckDoNotSubmitInDescription(input_api, output_api) +
      CheckDoNotSubmitInFiles(input_api, output_api)
      )


def CheckTreeIsOpen(input_api, output_api,
                    url=None, closed=None, json_url=None):
  """Check whether to allow commit without prompt.

  Supports two styles:
    1. Checks that an url's content doesn't match a regexp that would mean that
       the tree is closed. (old)
    2. Check the json_url to decide whether to allow commit without prompt.
  Args:
    input_api: input related apis.
    output_api: output related apis.
    url: url to use for regex based tree status.
    closed: regex to match for closed status.
    json_url: url to download json style status.
  """
  if not input_api.is_committing:
    return []
  try:
    if json_url:
      connection = input_api.urllib2.urlopen(json_url)
      status = input_api.json.loads(connection.read())
      connection.close()
      if not status['can_commit_freely']:
        short_text = 'Tree state is: ' + status['general_state']
        long_text = status['message'] + '\n' + json_url
        return [output_api.PresubmitError(short_text, long_text=long_text)]
    else:
      # TODO(bradnelson): drop this once all users are gone.
      connection = input_api.urllib2.urlopen(url)
      status = connection.read()
      connection.close()
      if input_api.re.match(closed, status):
        long_text = status + '\n' + url
        return [output_api.PresubmitError('The tree is closed.',
                                          long_text=long_text)]
  except IOError as e:
    return [output_api.PresubmitError('Error fetching tree status.',
                                      long_text=str(e))]
  return []

def GetUnitTestsInDirectory(
    input_api, output_api, directory, whitelist=None, blacklist=None, env=None):
  """Lists all files in a directory and runs them. Doesn't recurse.

  It's mainly a wrapper for RunUnitTests. Use whitelist and blacklist to filter
  tests accordingly.
  """
  unit_tests = []
  test_path = input_api.os_path.abspath(
      input_api.os_path.join(input_api.PresubmitLocalPath(), directory))

  def check(filename, filters):
    return any(True for i in filters if input_api.re.match(i, filename))

  to_run = found = 0
  for filename in input_api.os_listdir(test_path):
    found += 1
    fullpath = input_api.os_path.join(test_path, filename)
    if not input_api.os_path.isfile(fullpath):
      continue
    if whitelist and not check(filename, whitelist):
      continue
    if blacklist and check(filename, blacklist):
      continue
    unit_tests.append(input_api.os_path.join(directory, filename))
    to_run += 1
  input_api.logging.debug('Found %d files, running %d' % (found, to_run))
  if not to_run:
    return [
        output_api.PresubmitPromptWarning(
          'Out of %d files, found none that matched w=%r, b=%r in directory %s'
          % (found, whitelist, blacklist, directory))
    ]
  return GetUnitTests(input_api, output_api, unit_tests, env)


def GetUnitTests(input_api, output_api, unit_tests, env=None):
  """Runs all unit tests in a directory.

  On Windows, sys.executable is used for unit tests ending with ".py".
  """
  # We don't want to hinder users from uploading incomplete patches.
  if input_api.is_committing:
    message_type = output_api.PresubmitError
  else:
    message_type = output_api.PresubmitPromptWarning

  results = []
  for unit_test in unit_tests:
    cmd = []
    if input_api.platform == 'win32' and unit_test.endswith('.py'):
      # Windows needs some help.
      cmd = [input_api.python_executable]
    cmd.append(unit_test)
    if input_api.verbose:
      cmd.append('--verbose')
    kwargs = {'cwd': input_api.PresubmitLocalPath()}
    if env:
      kwargs['env'] = env
    results.append(input_api.Command(
        name=unit_test,
        cmd=cmd,
        kwargs=kwargs,
        message=message_type))
  return results


def GetUnitTestsRecursively(input_api, output_api, directory,
                            whitelist, blacklist):
  """Gets all files in the directory tree (git repo) that match the whitelist.

  Restricts itself to only find files within the Change's source repo, not
  dependencies.
  """
  def check(filename):
    return (any(input_api.re.match(f, filename) for f in whitelist) and
            not any(input_api.re.match(f, filename) for f in blacklist))

  tests = []

  to_run = found = 0
  for filepath in input_api.change.AllFiles(directory):
    found += 1
    if check(filepath):
      to_run += 1
      tests.append(filepath)
  input_api.logging.debug('Found %d files, running %d' % (found, to_run))
  if not to_run:
    return [
        output_api.PresubmitPromptWarning(
          'Out of %d files, found none that matched w=%r, b=%r in directory %s'
          % (found, whitelist, blacklist, directory))
    ]

  return GetUnitTests(input_api, output_api, tests)


def GetPythonUnitTests(input_api, output_api, unit_tests):
  """Run the unit tests out of process, capture the output and use the result
  code to determine success.

  DEPRECATED.
  """
  # We don't want to hinder users from uploading incomplete patches.
  if input_api.is_committing:
    message_type = output_api.PresubmitError
  else:
    message_type = output_api.PresubmitNotifyResult
  results = []
  for unit_test in unit_tests:
    # Run the unit tests out of process. This is because some unit tests
    # stub out base libraries and don't clean up their mess. It's too easy to
    # get subtle bugs.
    cwd = None
    env = None
    unit_test_name = unit_test
    # 'python -m test.unit_test' doesn't work. We need to change to the right
    # directory instead.
    if '.' in unit_test:
      # Tests imported in submodules (subdirectories) assume that the current
      # directory is in the PYTHONPATH. Manually fix that.
      unit_test = unit_test.replace('.', '/')
      cwd = input_api.os_path.dirname(unit_test)
      unit_test = input_api.os_path.basename(unit_test)
      env = input_api.environ.copy()
      # At least on Windows, it seems '.' must explicitly be in PYTHONPATH
      backpath = [
          '.', input_api.os_path.pathsep.join(['..'] * (cwd.count('/') + 1))
        ]
      if env.get('PYTHONPATH'):
        backpath.append(env.get('PYTHONPATH'))
      env['PYTHONPATH'] = input_api.os_path.pathsep.join((backpath))
    cmd = [input_api.python_executable, '-m', '%s' % unit_test]
    results.append(input_api.Command(
        name=unit_test_name,
        cmd=cmd,
        kwargs={'env': env, 'cwd': cwd},
        message=message_type))
  return results


def RunUnitTestsInDirectory(input_api, *args, **kwargs):
  """Run tests in a directory serially.

  For better performance, use GetUnitTestsInDirectory and then
  pass to input_api.RunTests.
  """
  return input_api.RunTests(
      GetUnitTestsInDirectory(input_api, *args, **kwargs), False)


def RunUnitTests(input_api, *args, **kwargs):
  """Run tests serially.

  For better performance, use GetUnitTests and then pass to
  input_api.RunTests.
  """
  return input_api.RunTests(GetUnitTests(input_api, *args, **kwargs), False)


def RunPythonUnitTests(input_api, *args, **kwargs):
  """Run python tests in a directory serially.

  DEPRECATED
  """
  return input_api.RunTests(
      GetPythonUnitTests(input_api, *args, **kwargs), False)


def _FetchAllFiles(input_api, white_list, black_list):
  """Hack to fetch all files."""
  # We cannot use AffectedFiles here because we want to test every python
  # file on each single python change. It's because a change in a python file
  # can break another unmodified file.
  # Use code similar to InputApi.FilterSourceFile()
  def Find(filepath, filters):
    for item in filters:
      if input_api.re.match(item, filepath):
        return True
    return False

  files = []
  path_len = len(input_api.PresubmitLocalPath())
  for dirpath, dirnames, filenames in input_api.os_walk(
      input_api.PresubmitLocalPath()):
    # Passes dirnames in black list to speed up search.
    for item in dirnames[:]:
      filepath = input_api.os_path.join(dirpath, item)[path_len + 1:]
      if Find(filepath, black_list):
        dirnames.remove(item)
    for item in filenames:
      filepath = input_api.os_path.join(dirpath, item)[path_len + 1:]
      if Find(filepath, white_list) and not Find(filepath, black_list):
        files.append(filepath)
  return files


def GetPylint(input_api, output_api, white_list=None, black_list=None,
              disabled_warnings=None, extra_paths_list=None, pylintrc=None):
  """Run pylint on python files.

  The default white_list enforces looking only at *.py files.
  """
  white_list = tuple(white_list or ('.*\.py$',))
  black_list = tuple(black_list or input_api.DEFAULT_BLACK_LIST)
  extra_paths_list = extra_paths_list or []

  if input_api.is_committing:
    error_type = output_api.PresubmitError
  else:
    error_type = output_api.PresubmitPromptWarning

  # Only trigger if there is at least one python file affected.
  def rel_path(regex):
    """Modifies a regex for a subject to accept paths relative to root."""
    def samefile(a, b):
      # Default implementation for platforms lacking os.path.samefile
      # (like Windows).
      return input_api.os_path.abspath(a) == input_api.os_path.abspath(b)
    samefile = getattr(input_api.os_path, 'samefile', samefile)
    if samefile(input_api.PresubmitLocalPath(),
                input_api.change.RepositoryRoot()):
      return regex

    prefix = input_api.os_path.join(input_api.os_path.relpath(
        input_api.PresubmitLocalPath(), input_api.change.RepositoryRoot()), '')
    return input_api.re.escape(prefix) + regex
  src_filter = lambda x: input_api.FilterSourceFile(
      x, map(rel_path, white_list), map(rel_path, black_list))
  if not input_api.AffectedSourceFiles(src_filter):
    input_api.logging.info('Skipping pylint: no matching changes.')
    return []

  if pylintrc is not None:
    pylintrc = input_api.os_path.join(input_api.PresubmitLocalPath(), pylintrc)
  else:
    pylintrc = input_api.os_path.join(_HERE, 'pylintrc')
  extra_args = ['--rcfile=%s' % pylintrc]
  if disabled_warnings:
    extra_args.extend(['-d', ','.join(disabled_warnings)])

  files = _FetchAllFiles(input_api, white_list, black_list)
  if not files:
    return []
  files.sort()

  input_api.logging.info('Running pylint on %d files', len(files))
  input_api.logging.debug('Running pylint on: %s', files)
  # Copy the system path to the environment so pylint can find the right
  # imports.
  env = input_api.environ.copy()
  import sys
  env['PYTHONPATH'] = input_api.os_path.pathsep.join(
      extra_paths_list + sys.path).encode('utf8')

  def GetPylintCmd(flist, extra, parallel):
    # Windows needs help running python files so we explicitly specify
    # the interpreter to use. It also has limitations on the size of
    # the command-line, so we pass arguments via a pipe.
    cmd = [input_api.python_executable,
           input_api.os_path.join(_HERE, 'third_party', 'pylint.py'),
           '--args-on-stdin']
    if len(flist) == 1:
      description = flist[0]
    else:
      description = '%s files' % len(flist)

    args = extra_args[:]
    if extra:
      args.extend(extra)
      description += ' using %s' % (extra,)
    if parallel:
      args.append('--jobs=%s' % input_api.cpu_count)
      description += ' on %d cores' % input_api.cpu_count

    return input_api.Command(
        name='Pylint (%s)' % description,
        cmd=cmd,
        kwargs={'env': env, 'stdin': '\n'.join(args + flist)},
        message=error_type)

  # Always run pylint and pass it all the py files at once.
  # Passing py files one at time is slower and can produce
  # different results.  input_api.verbose used to be used
  # to enable this behaviour but differing behaviour in
  # verbose mode is not desirable.
  # Leave this unreachable code in here so users can make
  # a quick local edit to diagnose pylint issues more
  # easily.
  if True:
    # pylint's cycle detection doesn't work in parallel, so spawn a second,
    # single-threaded job for just that check.

    # Some PRESUBMITs explicitly mention cycle detection.
    if not any('R0401' in a or 'cyclic-import' in a for a in extra_args):
      return [
        GetPylintCmd(files, ["--disable=cyclic-import"], True),
        GetPylintCmd(files, ["--disable=all", "--enable=cyclic-import"], False)
      ]
    else:
      return [ GetPylintCmd(files, [], True) ]

  else:
    return map(lambda x: GetPylintCmd([x], [], 1), files)


def RunPylint(input_api, *args, **kwargs):
  """Legacy presubmit function.

  For better performance, get all tests and then pass to
  input_api.RunTests.
  """
  return input_api.RunTests(GetPylint(input_api, *args, **kwargs), False)


def CheckRietveldTryJobExecution(dummy_input_api, output_api,
                                 dummy_host_url, dummy_platforms,
                                 dummy_owner):
  return [
    output_api.PresubmitNotifyResult(
        'CheckRietveldTryJobExecution is deprecated, please remove it.')
  ]


def CheckBuildbotPendingBuilds(input_api, output_api, url, max_pendings,
    ignored):
  try:
    connection = input_api.urllib2.urlopen(url)
    raw_data = connection.read()
    connection.close()
  except IOError:
    return [output_api.PresubmitNotifyResult('%s is not accessible' % url)]

  try:
    data = input_api.json.loads(raw_data)
  except ValueError:
    return [output_api.PresubmitNotifyResult('Received malformed json while '
                                             'looking up buildbot status')]

  out = []
  for (builder_name, builder) in data.iteritems():
    if builder_name in ignored:
      continue
    if builder.get('state', '') == 'offline':
      continue
    pending_builds_len = len(builder.get('pending_builds', []))
    if pending_builds_len > max_pendings:
      out.append('%s has %d build(s) pending' %
                  (builder_name, pending_builds_len))
  if out:
    return [output_api.PresubmitPromptWarning(
        'Build(s) pending. It is suggested to wait that no more than %d '
            'builds are pending.' % max_pendings,
        long_text='\n'.join(out))]
  return []


def CheckOwners(input_api, output_api, source_file_filter=None):
  if input_api.is_committing:
    if input_api.tbr:
      return [output_api.PresubmitNotifyResult(
          '--tbr was specified, skipping OWNERS check')]
    if input_api.change.issue:
      if input_api.dry_run:
        return [output_api.PresubmitNotifyResult(
            'This is a dry run, skipping OWNERS check')]
    else:
      return [output_api.PresubmitError("OWNERS check failed: this change has "
          "no Rietveld issue number, so we can't check it for approvals.")]
    needed = 'LGTM from an OWNER'
    output = output_api.PresubmitError
  else:
    needed = 'OWNER reviewers'
    output = output_api.PresubmitNotifyResult

  affected_files = set([f.LocalPath() for f in
      input_api.change.AffectedFiles(file_filter=source_file_filter)])

  owners_db = input_api.owners_db
  owner_email, reviewers = GetCodereviewOwnerAndReviewers(
      input_api,
      owners_db.email_regexp,
      approval_needed=input_api.is_committing)

  owner_email = owner_email or input_api.change.author_email

  if owner_email:
    reviewers_plus_owner = set([owner_email]).union(reviewers)
    missing_files = owners_db.files_not_covered_by(affected_files,
        reviewers_plus_owner)
  else:
    missing_files = owners_db.files_not_covered_by(affected_files, reviewers)

  if missing_files:
    output_list = [
        output('Missing %s for these files:\n    %s' %
               (needed, '\n    '.join(sorted(missing_files))))]
    if not input_api.is_committing:
      suggested_owners = owners_db.reviewers_for(missing_files, owner_email)
      output_list.append(output('Suggested OWNERS: ' +
          '(Use "git-cl owners" to interactively select owners.)\n    %s' %
          ('\n    '.join(suggested_owners or []))))
    return output_list

  if input_api.is_committing and not reviewers:
    return [output('Missing LGTM from someone other than %s' % owner_email)]
  return []

def GetCodereviewOwnerAndReviewers(input_api, email_regexp, approval_needed):
  """Return the owner and reviewers of a change, if any.

  If approval_needed is True, only reviewers who have approved the change
  will be returned.
  """
  # Rietveld is default.
  func = _RietveldOwnerAndReviewers
  if input_api.gerrit:
    func = _GerritOwnerAndReviewers
  return func(input_api, email_regexp, approval_needed)


def _GetRietveldIssueProps(input_api, messages):
  """Gets the issue properties from rietveld."""
  issue = input_api.change.issue
  if issue and input_api.rietveld:
    return input_api.rietveld.get_issue_properties(
        issue=int(issue), messages=messages)


def _ReviewersFromChange(change):
  """Return the reviewers specified in the |change|, if any."""
  reviewers = set()
  if change.R:
    reviewers.update(set([r.strip() for r in change.R.split(',')]))
  if change.TBR:
    reviewers.update(set([r.strip() for r in change.TBR.split(',')]))

  # Drop reviewers that aren't specified in email address format.
  return set(reviewer for reviewer in reviewers if '@' in reviewer)


def _match_reviewer_email(r, owner_email, email_regexp):
  return email_regexp.match(r) and r != owner_email

def _RietveldOwnerAndReviewers(input_api, email_regexp, approval_needed=False):
  """Return the owner and reviewers of a change, if any.

  If approval_needed is True, only reviewers who have approved the change
  will be returned.
  """
  issue_props = _GetRietveldIssueProps(input_api, True)
  if not issue_props:
    return None, (set() if approval_needed else
                  _ReviewersFromChange(input_api.change))

  if not approval_needed:
    return issue_props['owner_email'], set(issue_props['reviewers'])

  owner_email = issue_props['owner_email']

  messages = issue_props.get('messages', [])
  approvers = set(
      m['sender'] for m in messages
      if m.get('approval') and _match_reviewer_email(m['sender'], owner_email,
                                                     email_regexp))
  return owner_email, approvers


def _GerritOwnerAndReviewers(input_api, email_regexp, approval_needed=False):
  """Return the owner and reviewers of a change, if any.

  If approval_needed is True, only reviewers who have approved the change
  will be returned.
  """
  issue = input_api.change.issue
  if not issue:
    return None, (set() if approval_needed else
                  _ReviewersFromChange(input_api.change))

  owner_email = input_api.gerrit.GetChangeOwner(issue)
  reviewers = set(
      r for r in input_api.gerrit.GetChangeReviewers(issue, approval_needed)
      if _match_reviewer_email(r, owner_email, email_regexp))
  return owner_email, reviewers


def _CheckConstNSObject(input_api, output_api, source_file_filter):
  """Checks to make sure no objective-c files have |const NSSomeClass*|."""
  pattern = input_api.re.compile(
    r'(?<!reinterpret_cast<)'
    r'const\s+NS(?!(Point|Range|Rect|Size)\s*\*)\w*\s*\*')

  def objective_c_filter(f):
    return (source_file_filter(f) and
            input_api.os_path.splitext(f.LocalPath())[1] in ('.h', '.m', '.mm'))

  files = []
  for f in input_api.AffectedSourceFiles(objective_c_filter):
    contents = input_api.ReadFile(f)
    if pattern.search(contents):
      files.append(f)

  if files:
    if input_api.is_committing:
      res_type = output_api.PresubmitPromptWarning
    else:
      res_type = output_api.PresubmitNotifyResult
    return [ res_type('|const NSClass*| is wrong, see ' +
                      'http://dev.chromium.org/developers/clang-mac',
                      files) ]
  return []


def CheckSingletonInHeaders(input_api, output_api, source_file_filter=None):
  """Deprecated, must be removed."""
  return [
    output_api.PresubmitNotifyResult(
        'CheckSingletonInHeaders is deprecated, please remove it.')
  ]


def PanProjectChecks(input_api, output_api,
                     excluded_paths=None, text_files=None,
                     license_header=None, project_name=None,
                     owners_check=True, maxlen=80):
  """Checks that ALL chromium orbit projects should use.

  These are checks to be run on all Chromium orbit project, including:
    Chromium
    Native Client
    V8
  When you update this function, please take this broad scope into account.
  Args:
    input_api: Bag of input related interfaces.
    output_api: Bag of output related interfaces.
    excluded_paths: Don't include these paths in common checks.
    text_files: Which file are to be treated as documentation text files.
    license_header: What license header should be on files.
    project_name: What is the name of the project as it appears in the license.
  Returns:
    A list of warning or error objects.
  """
  excluded_paths = tuple(excluded_paths or [])
  text_files = tuple(text_files or (
      r'.+\.txt$',
      r'.+\.json$',
  ))
  project_name = project_name or 'Chromium'

  # Accept any year number from 2006 to the current year, or the special
  # 2006-20xx string used on the oldest files. 2006-20xx is deprecated, but
  # tolerated on old files.
  current_year = int(input_api.time.strftime('%Y'))
  allowed_years = (str(s) for s in reversed(xrange(2006, current_year + 1)))
  years_re = '(' + '|'.join(allowed_years) + '|2006-2008|2006-2009|2006-2010)'

  # The (c) is deprecated, but tolerate it until it's removed from all files.
  license_header = license_header or (
      r'.*? Copyright (\(c\) )?%(year)s The %(project)s Authors\. '
        r'All rights reserved\.\n'
      r'.*? Use of this source code is governed by a BSD-style license that '
        r'can be\n'
      r'.*? found in the LICENSE file\.(?: \*/)?\n'
  ) % {
      'year': years_re,
      'project': project_name,
  }

  results = []
  # This code loads the default black list (e.g. third_party, experimental, etc)
  # and add our black list (breakpad, skia and v8 are still not following
  # google style and are not really living this repository).
  # See presubmit_support.py InputApi.FilterSourceFile for the (simple) usage.
  black_list = input_api.DEFAULT_BLACK_LIST + excluded_paths
  white_list = input_api.DEFAULT_WHITE_LIST + text_files
  sources = lambda x: input_api.FilterSourceFile(x, black_list=black_list)
  text_files = lambda x: input_api.FilterSourceFile(
      x, black_list=black_list, white_list=white_list)

  snapshot_memory = []
  def snapshot(msg):
    """Measures & prints performance warning if a rule is running slow."""
    dt2 = input_api.time.clock()
    if snapshot_memory:
      delta_ms = int(1000*(dt2 - snapshot_memory[0]))
      if delta_ms > 500:
        print "  %s took a long time: %dms" % (snapshot_memory[1], delta_ms)
    snapshot_memory[:] = (dt2, msg)

  if owners_check:
    snapshot("checking owners")
    results.extend(input_api.canned_checks.CheckOwners(
        input_api, output_api, source_file_filter=None))

  snapshot("checking long lines")
  results.extend(input_api.canned_checks.CheckLongLines(
      input_api, output_api, maxlen, source_file_filter=sources))
  snapshot( "checking tabs")
  results.extend(input_api.canned_checks.CheckChangeHasNoTabs(
      input_api, output_api, source_file_filter=sources))
  snapshot( "checking stray whitespace")
  results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
      input_api, output_api, source_file_filter=sources))
  snapshot("checking nsobjects")
  results.extend(_CheckConstNSObject(
      input_api, output_api, source_file_filter=sources))
  snapshot("checking eol style")
  results.extend(input_api.canned_checks.CheckChangeSvnEolStyle(
      input_api, output_api, source_file_filter=text_files))
  snapshot("checking license")
  results.extend(input_api.canned_checks.CheckLicense(
      input_api, output_api, license_header, source_file_filter=sources))

  if input_api.is_committing:
    snapshot("checking svn mime types")
    results.extend(input_api.canned_checks.CheckSvnForCommonMimeTypes(
        input_api, output_api))
    snapshot("checking was uploaded")
    results.extend(input_api.canned_checks.CheckChangeWasUploaded(
        input_api, output_api))
    snapshot("checking description")
    results.extend(input_api.canned_checks.CheckChangeHasDescription(
        input_api, output_api))
    results.extend(input_api.canned_checks.CheckDoNotSubmitInDescription(
        input_api, output_api))
    snapshot("checking do not submit in files")
    results.extend(input_api.canned_checks.CheckDoNotSubmitInFiles(
        input_api, output_api))
  snapshot("done")
  return results


def CheckPatchFormatted(input_api, output_api):
  import git_cl
  cmd = ['cl', 'format', '--dry-run', input_api.PresubmitLocalPath()]
  code, _ = git_cl.RunGitWithCode(cmd, suppress_stderr=True)
  if code == 2:
    short_path = input_api.basename(input_api.PresubmitLocalPath())
    full_path = input_api.os_path.relpath(input_api.PresubmitLocalPath(),
                                          input_api.change.RepositoryRoot())
    return [output_api.PresubmitPromptWarning(
      'The %s directory requires source formatting. '
      'Please run git cl format %s' %
      (short_path, full_path))]
  # As this is just a warning, ignore all other errors if the user
  # happens to have a broken clang-format, doesn't use git, etc etc.
  return []


def CheckGNFormatted(input_api, output_api):
  import gn
  affected_files = input_api.AffectedFiles(
      include_deletes=False,
      file_filter=lambda x: x.LocalPath().endswith('.gn') or
                            x.LocalPath().endswith('.gni') or
                            x.LocalPath().endswith('.typemap'))
  warnings = []
  for f in affected_files:
    cmd = ['gn', 'format', '--dry-run', f.AbsoluteLocalPath()]
    rc = gn.main(cmd)
    if rc == 2:
      warnings.append(output_api.PresubmitPromptWarning(
          '%s requires formatting. Please run:\n  gn format %s' % (
              f.AbsoluteLocalPath(), f.LocalPath())))
  # It's just a warning, so ignore other types of failures assuming they'll be
  # caught elsewhere.
  return warnings