autoninja.py

#!/usr/bin/env python3
# Copyright (c) 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
This script (intended to be invoked by autoninja or autoninja.bat) detects
whether a build is accelerated using a service like goma. If so, it runs with a
large -j value, and otherwise it chooses a small one. This auto-adjustment
makes using remote build acceleration simpler and safer, and avoids errors that
can cause slow goma builds or swap-storms on unaccelerated builds.
"""

import multiprocessing
import os
import platform
import re
import subprocess
import sys

if sys.platform == 'darwin':
  import resource

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))


def main(args):
  # The -t tools are incompatible with -j
  t_specified = False
  j_specified = False
  offline = False
  output_dir = '.'
  input_args = args
  # On Windows the autoninja.bat script passes along the arguments enclosed in
  # double quotes. This prevents multiple levels of parsing of the special '^'
  # characters needed when compiling a single file but means that this script
  # gets called with a single argument containing all of the actual arguments,
  # separated by spaces. When this case is detected we need to do argument
  # splitting ourselves. This means that arguments containing actual spaces are
  # not supported by autoninja, but that is not a real limitation.
  if (sys.platform.startswith('win') and len(args) == 2
      and input_args[1].count(' ') > 0):
    input_args = args[:1] + args[1].split()

  # Ninja uses getopt_long, which allow to intermix non-option arguments.
  # To leave non supported parameters untouched, we do not use getopt.
  for index, arg in enumerate(input_args[1:]):
    if arg.startswith('-j'):
      j_specified = True
    if arg.startswith('-t'):
      t_specified = True
    if arg == '-C':
      # + 1 to get the next argument and +1 because we trimmed off input_args[0]
      output_dir = input_args[index + 2]
    elif arg.startswith('-C'):
      # Support -Cout/Default
      output_dir = arg[2:]
    elif arg in ('-o', '--offline'):
      offline = True
    elif arg == '-h':
      print('autoninja: Use -o/--offline to temporary disable goma.',
            file=sys.stderr)
      print(file=sys.stderr)

  use_goma = False
  use_remoteexec = False
  use_rbe = False
  use_siso = False

  # Attempt to auto-detect remote build acceleration. We support gn-based
  # builds, where we look for args.gn in the build tree, and cmake-based builds
  # where we look for rules.ninja.
  if os.path.exists(os.path.join(output_dir, 'args.gn')):
    with open(os.path.join(output_dir, 'args.gn')) as file_handle:
      for line in file_handle:
        # use_goma, use_remoteexec, or use_rbe will activate build acceleration.
        #
        # This test can match multi-argument lines. Examples of this are:
        # is_debug=false use_goma=true is_official_build=false
        # use_goma=false# use_goma=true This comment is ignored
        #
        # Anything after a comment is not consider a valid argument.
        line_without_comment = line.split('#')[0]
        if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
                     line_without_comment):
          use_goma = True
          continue
        if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)',
                     line_without_comment):
          use_remoteexec = True
          continue
        if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)', line_without_comment):
          use_rbe = True
          continue
        if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)',
                     line_without_comment):
          use_siso = True
          continue

    siso_marker = os.path.join(output_dir, '.siso_deps')
    if use_siso:
      ninja_marker = os.path.join(output_dir, '.ninja_log')
      # autosiso generates a .ninja_log file so the mere existence of a
      # .ninja_log file doesn't imply that a ninja build was done. However if
      # there is a .ninja_log but no .siso_deps then that implies a ninja build.
      if os.path.exists(ninja_marker) and not os.path.exists(siso_marker):
        return ('echo Run gn clean before switching from ninja to siso in %s' %
                output_dir)
      siso = ['autosiso'] if use_remoteexec else ['siso', 'ninja']
      if sys.platform.startswith('win'):
        # An explicit 'call' is needed to make sure the invocation of autosiso
        # returns to autoninja.bat, and the command prompt title gets reset.
        siso = ['call'] + siso
      return ' '.join(siso + input_args[1:])

    if os.path.exists(siso_marker):
      return ('echo Run gn clean before switching from siso to ninja in %s' %
              output_dir)

  else:
    for relative_path in [
        '',  # GN keeps them in the root of output_dir
        'CMakeFiles'
    ]:
      path = os.path.join(output_dir, relative_path, 'rules.ninja')
      if os.path.exists(path):
        with open(path) as file_handle:
          for line in file_handle:
            if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
              use_goma = True
              break

  # Strip -o/--offline so ninja doesn't see them.
  input_args = [arg for arg in input_args if arg not in ('-o', '--offline')]

  # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
  # (case-insensitive) then gomacc will use the local compiler instead of doing
  # a goma compile. This is convenient if you want to briefly disable goma. It
  # avoids having to rebuild the world when transitioning between goma/non-goma
  # builds. However, it is not as fast as doing a "normal" non-goma build
  # because an extra process is created for each compile step. Checking this
  # environment variable ensures that autoninja uses an appropriate -j value in
  # this situation.
  goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
  if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
    use_goma = False

  if use_goma:
    gomacc_file = 'gomacc.exe' if sys.platform.startswith('win') else 'gomacc'
    goma_dir = os.environ.get('GOMA_DIR', os.path.join(SCRIPT_DIR, '.cipd_bin'))
    gomacc_path = os.path.join(goma_dir, gomacc_file)
    # Don't invoke gomacc if it doesn't exist.
    if os.path.exists(gomacc_path):
      # Check to make sure that goma is running. If not, don't start the build.
      status = subprocess.call([gomacc_path, 'port'],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               shell=False)
      if status == 1:
        print('Goma is not running. Use "goma_ctl ensure_start" to start it.',
              file=sys.stderr)
        if sys.platform.startswith('win'):
          # Set an exit code of 1 in the batch file.
          print('cmd "/c exit 1"')
        else:
          # Set an exit code of 1 by executing 'false' in the bash script.
          print('false')
        sys.exit(1)

  # A large build (with or without goma) tends to hog all system resources.
  # Launching the ninja process with 'nice' priorities improves this situation.
  prefix_args = []
  if (sys.platform.startswith('linux')
      and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
    # nice -10 is process priority 10 lower than default 0
    # ionice -c 3 is IO priority IDLE
    prefix_args = ['nice'] + ['-10']

  # Tell goma or reclient to do local compiles. On Windows these environment
  # variables are set by the wrapper batch file.
  offline_env = ['RBE_remote_disabled=1', 'GOMA_DISABLED=1'
                 ] if offline and not sys.platform.startswith('win') else []

  # On macOS, the default limit of open file descriptors is too low (256).
  # This causes a large j value to result in 'Too many open files' errors.
  # Check whether the limit can be raised to a large enough value. If yes,
  # use `ulimit -n .... &&` as a prefix to increase the limit when running
  # ninja.
  if sys.platform == 'darwin':
    wanted_limit = 200000  # Large enough to avoid any risk of exhaustion.
    fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
    if fileno_limit <= wanted_limit:
      try:
        resource.setrlimit(resource.RLIMIT_NOFILE, (wanted_limit, hard_limit))
      except Exception as _:
        pass
      fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
      if fileno_limit >= wanted_limit:
        prefix_args = ['ulimit', '-n', f'{wanted_limit}', '&&'] + offline_env
        offline_env = []


  # Call ninja.py so that it can find ninja binary installed by DEPS or one in
  # PATH.
  ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py')
  # If using remoteexec, use ninja_reclient.py which wraps ninja.py with
  # starting and stopping reproxy.
  if use_remoteexec:
    ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py')

  args = offline_env + prefix_args + [sys.executable, ninja_path
                                      ] + input_args[1:]

  num_cores = multiprocessing.cpu_count()
  if not j_specified and not t_specified:
    if not offline and (use_goma or use_remoteexec or use_rbe):
      args.append('-j')
      default_core_multiplier = 80
      if platform.machine() in ('x86_64', 'AMD64'):
        # Assume simultaneous multithreading and therefore half as many cores as
        # logical processors.
        num_cores //= 2

      core_multiplier = int(
          os.environ.get('NINJA_CORE_MULTIPLIER', default_core_multiplier))
      j_value = num_cores * core_multiplier

      core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value))
      j_value = min(j_value, core_limit)

      if sys.platform.startswith('win'):
        # On windows, j value higher than 1000 does not improve build
        # performance.
        j_value = min(j_value, 1000)
      elif sys.platform == 'darwin':
        # If the number of open file descriptors is large enough (or it can be
        # raised to a large enough value), then set j value to 1000. This limit
        # comes from ninja which is limited to at most FD_SETSIZE (1024) open
        # file descriptors (using 1000 leave a bit of head room).
        #
        # If the number of open file descriptors cannot be raised, then use a
        # j value of 200 which is the maximum value that reliably work with
        # the default limit of 256.
        if fileno_limit >= wanted_limit:
          j_value = min(j_value, 1000)
        else:
          j_value = min(j_value, 200)

      args.append('%d' % j_value)
    else:
      j_value = num_cores
      # Ninja defaults to |num_cores + 2|
      j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
      args.append('-j')
      args.append('%d' % j_value)

  # On Windows, fully quote the path so that the command processor doesn't think
  # the whole output is the command.
  # On Linux and Mac, if people put depot_tools in directories with ' ',
  # shell would misunderstand ' ' as a path separation.
  # TODO(yyanagisawa): provide proper quoting for Windows.
  # see https://cs.chromium.org/chromium/src/tools/mb/mb.py
  for i in range(len(args)):
    if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
      args[i] = '"%s"' % args[i].replace('"', '\\"')

  if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
    args += ['-d', 'stats']

  return ' '.join(args)


if __name__ == '__main__':
  print(main(sys.argv))