blob: af8417c55c456ee6dd63adca87620453e19913e3 [file] [log] [blame]
Takuto Ikuta237cc462022-01-19 18:04:15 +00001#!/usr/bin/env python3
Bruce Dawsonebebd952017-05-31 14:24:38 -07002# Copyright (c) 2017 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
Bruce Dawsonebebd952017-05-31 14:24:38 -07005"""
6This script (intended to be invoked by autoninja or autoninja.bat) detects
Simeon Anfinrud5dba9c92020-09-03 20:02:05 +00007whether a build is accelerated using a service like goma. If so, it runs with a
8large -j value, and otherwise it chooses a small one. This auto-adjustment
9makes using remote build acceleration simpler and safer, and avoids errors that
10can cause slow goma builds or swap-storms on unaccelerated builds.
Bruce Dawson30c1cba2023-09-15 18:20:32 +000011
12autoninja tries to detect relevant build settings such as use_remoteexec, and it
13does handle import statements, but it can't handle conditional setting of build
14settings.
Bruce Dawsonebebd952017-05-31 14:24:38 -070015"""
16
Bruce Dawsone952fae2021-02-27 23:33:37 +000017import multiprocessing
Bruce Dawsonebebd952017-05-31 14:24:38 -070018import os
Takuto Ikuta6a1494e2022-05-06 01:22:16 +000019import platform
Bruce Dawsonebebd952017-05-31 14:24:38 -070020import re
Philipp Wollermann0b943402023-10-12 07:13:30 +000021import shlex
Bruce Dawsone952fae2021-02-27 23:33:37 +000022import subprocess
Bruce Dawsonebebd952017-05-31 14:24:38 -070023import sys
24
Philipp Wollermann0b943402023-10-12 07:13:30 +000025import autosiso
26import ninja
27import ninja_reclient
28import siso
29
Henrique Ferreiro8bde1642023-09-14 11:41:19 +000030if sys.platform in ['darwin', 'linux']:
Mike Frysinger124bb8e2023-09-06 05:48:55 +000031 import resource
Sylvain Defresne7b4ecc72023-07-27 16:24:54 +000032
Yoshisato Yanagisawa4b497072018-11-07 02:52:33 +000033SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
34
Philipp Wollermann0b943402023-10-12 07:13:30 +000035# See [1] and [2] for the painful details of this next section, which handles
36# escaping command lines so that they can be copied and pasted into a cmd
37# window.
38#
39# pylint: disable=line-too-long
40# [1] https://learn.microsoft.com/en-us/archive/blogs/twistylittlepassagesallalike/everyone-quotes-command-line-arguments-the-wrong-way # noqa
41# [2] https://web.archive.org/web/20150815000000*/https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/set.mspx # noqa
42_UNSAFE_FOR_CMD = set('^<>&|()%')
43_ALL_META_CHARS = _UNSAFE_FOR_CMD.union(set('"'))
44
45
46def _quote_for_cmd(arg):
47 # First, escape the arg so that CommandLineToArgvW will parse it properly.
48 if arg == '' or ' ' in arg or '"' in arg:
49 quote_re = re.compile(r'(\\*)"')
50 arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
51
52 # Then check to see if the arg contains any metacharacters other than
53 # double quotes; if it does, quote everything (including the double
54 # quotes) for safety.
55 if any(a in _UNSAFE_FOR_CMD for a in arg):
56 arg = ''.join('^' + a if a in _ALL_META_CHARS else a for a in arg)
57 return arg
58
59
60def _print_cmd(cmd):
61 shell_quoter = shlex.quote
62 if sys.platform.startswith('win'):
63 shell_quoter = _quote_for_cmd
64 print(*[shell_quoter(arg) for arg in cmd], file=sys.stderr)
65
Yoshisato Yanagisawaf66e5512018-11-15 00:40:39 +000066
Bruce Dawson30c1cba2023-09-15 18:20:32 +000067def _gn_lines(output_dir, path):
68 """
69 Generator function that returns args.gn lines one at a time, following
70 import directives as needed.
71 """
72 import_re = re.compile(r'\s*import\("(.*)"\)')
73 with open(path, encoding='utf-8') as f:
74 for line in f:
75 match = import_re.match(line)
76 if match:
Samuel Attard8a259982023-09-18 17:12:55 +000077 raw_import_path = match.groups()[0]
78 if raw_import_path[:2] == "//":
79 import_path = os.path.normpath(
80 os.path.join(output_dir, '..', '..',
81 raw_import_path[2:]))
82 else:
83 import_path = os.path.normpath(
84 os.path.join(os.path.dirname(path), raw_import_path))
Bruce Dawson30c1cba2023-09-15 18:20:32 +000085 for import_line in _gn_lines(output_dir, import_path):
86 yield import_line
87 else:
88 yield line
89
90
Takuto Ikuta381db682022-04-27 23:54:02 +000091def main(args):
Mike Frysinger124bb8e2023-09-06 05:48:55 +000092 # The -t tools are incompatible with -j
93 t_specified = False
94 j_specified = False
95 offline = False
96 output_dir = '.'
97 input_args = args
Philipp Wollermann0b943402023-10-12 07:13:30 +000098 summarize_build = os.environ.get('NINJA_SUMMARIZE_BUILD') == '1'
Mike Frysinger124bb8e2023-09-06 05:48:55 +000099 # On Windows the autoninja.bat script passes along the arguments enclosed in
100 # double quotes. This prevents multiple levels of parsing of the special '^'
101 # characters needed when compiling a single file but means that this script
102 # gets called with a single argument containing all of the actual arguments,
103 # separated by spaces. When this case is detected we need to do argument
104 # splitting ourselves. This means that arguments containing actual spaces
105 # are not supported by autoninja, but that is not a real limitation.
106 if (sys.platform.startswith('win') and len(args) == 2
107 and input_args[1].count(' ') > 0):
108 input_args = args[:1] + args[1].split()
Bruce Dawson655afeb2020-11-02 23:30:37 +0000109
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000110 # Ninja uses getopt_long, which allow to intermix non-option arguments.
111 # To leave non supported parameters untouched, we do not use getopt.
112 for index, arg in enumerate(input_args[1:]):
113 if arg.startswith('-j'):
114 j_specified = True
115 if arg.startswith('-t'):
116 t_specified = True
117 if arg == '-C':
118 # + 1 to get the next argument and +1 because we trimmed off
119 # input_args[0]
120 output_dir = input_args[index + 2]
121 elif arg.startswith('-C'):
122 # Support -Cout/Default
123 output_dir = arg[2:]
124 elif arg in ('-o', '--offline'):
125 offline = True
126 elif arg == '-h':
127 print('autoninja: Use -o/--offline to temporary disable goma.',
128 file=sys.stderr)
129 print(file=sys.stderr)
Bruce Dawsone952fae2021-02-27 23:33:37 +0000130
Takuto Ikuta381db682022-04-27 23:54:02 +0000131 use_goma = False
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000132 use_remoteexec = False
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000133 use_siso = False
Yoshisato Yanagisawa43a35d22018-11-15 03:00:51 +0000134
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000135 # Attempt to auto-detect remote build acceleration. We support gn-based
136 # builds, where we look for args.gn in the build tree, and cmake-based
137 # builds where we look for rules.ninja.
138 if os.path.exists(os.path.join(output_dir, 'args.gn')):
Bruce Dawson30c1cba2023-09-15 18:20:32 +0000139 for line in _gn_lines(output_dir, os.path.join(output_dir, 'args.gn')):
Takuto Ikuta5cbc5212023-11-16 02:38:31 +0000140 # use_goma, or use_remoteexec will activate build
Bruce Dawson30c1cba2023-09-15 18:20:32 +0000141 # acceleration.
142 #
143 # This test can match multi-argument lines. Examples of this
144 # are: is_debug=false use_goma=true is_official_build=false
145 # use_goma=false# use_goma=true This comment is ignored
146 #
147 # Anything after a comment is not consider a valid argument.
148 line_without_comment = line.split('#')[0]
149 if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
150 line_without_comment):
151 use_goma = True
152 continue
153 if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)',
154 line_without_comment):
155 use_remoteexec = True
156 continue
Bruce Dawson30c1cba2023-09-15 18:20:32 +0000157 if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)',
158 line_without_comment):
159 use_siso = True
160 continue
Bruce Dawsonb3b46a22019-09-06 15:57:52 +0000161
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000162 siso_marker = os.path.join(output_dir, '.siso_deps')
163 if use_siso:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000164 # autosiso generates a .ninja_log file so the mere existence of a
165 # .ninja_log file doesn't imply that a ninja build was done. However
166 # if there is a .ninja_log but no .siso_deps then that implies a
167 # ninja build.
Junji Watanabec5182992023-10-13 04:52:02 +0000168 ninja_marker = os.path.join(output_dir, '.ninja_log')
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000169 if os.path.exists(ninja_marker) and not os.path.exists(siso_marker):
Philipp Wollermann0b943402023-10-12 07:13:30 +0000170 print('Run gn clean before switching from ninja to siso in %s' %
171 output_dir,
172 file=sys.stderr)
173 return 1
Fumitoshi Ukai406be822023-10-18 04:18:24 +0000174 if use_goma:
175 print('Siso does not support Goma.', file=sys.stderr)
176 print('Do not use use_siso=true and use_goma=true',
177 file=sys.stderr)
178 return 1
Philipp Wollermann0b943402023-10-12 07:13:30 +0000179 if use_remoteexec:
180 return autosiso.main(['autosiso'] + input_args[1:])
Philipp Wollermann17debef2023-11-02 02:03:46 +0000181 return siso.main(['siso', 'ninja', '--offline'] + input_args[1:])
Michael Savigny20eda952021-01-20 01:16:27 +0000182
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000183 if os.path.exists(siso_marker):
Philipp Wollermann0b943402023-10-12 07:13:30 +0000184 print('Run gn clean before switching from siso to ninja in %s' %
185 output_dir,
186 file=sys.stderr)
187 return 1
Ben Segall467991e2023-08-09 19:02:09 +0000188
Takuto Ikuta381db682022-04-27 23:54:02 +0000189 else:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000190 for relative_path in [
191 '', # GN keeps them in the root of output_dir
192 'CMakeFiles'
193 ]:
194 path = os.path.join(output_dir, relative_path, 'rules.ninja')
195 if os.path.exists(path):
Bruce Dawson30c1cba2023-09-15 18:20:32 +0000196 with open(path, encoding='utf-8') as file_handle:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000197 for line in file_handle:
198 if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
199 use_goma = True
200 break
Takuto Ikuta381db682022-04-27 23:54:02 +0000201
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000202 # Strip -o/--offline so ninja doesn't see them.
203 input_args = [arg for arg in input_args if arg not in ('-o', '--offline')]
Takuto Ikuta381db682022-04-27 23:54:02 +0000204
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000205 # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
206 # (case-insensitive) then gomacc will use the local compiler instead of
207 # doing a goma compile. This is convenient if you want to briefly disable
208 # goma. It avoids having to rebuild the world when transitioning between
209 # goma/non-goma builds. However, it is not as fast as doing a "normal"
210 # non-goma build because an extra process is created for each compile step.
211 # Checking this environment variable ensures that autoninja uses an
212 # appropriate -j value in this situation.
213 goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
214 if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
215 use_goma = False
Takuto Ikuta381db682022-04-27 23:54:02 +0000216
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000217 if use_goma:
218 gomacc_file = 'gomacc.exe' if sys.platform.startswith(
219 'win') else 'gomacc'
220 goma_dir = os.environ.get('GOMA_DIR',
221 os.path.join(SCRIPT_DIR, '.cipd_bin'))
222 gomacc_path = os.path.join(goma_dir, gomacc_file)
223 # Don't invoke gomacc if it doesn't exist.
224 if os.path.exists(gomacc_path):
225 # Check to make sure that goma is running. If not, don't start the
226 # build.
227 status = subprocess.call([gomacc_path, 'port'],
228 stdout=subprocess.PIPE,
229 stderr=subprocess.PIPE,
230 shell=False)
231 if status == 1:
232 print(
233 'Goma is not running. Use "goma_ctl ensure_start" to start '
234 'it.',
235 file=sys.stderr)
236 if sys.platform.startswith('win'):
237 # Set an exit code of 1 in the batch file.
238 print('cmd "/c exit 1"')
239 else:
240 # Set an exit code of 1 by executing 'false' in the bash
241 # script.
242 print('false')
243 sys.exit(1)
244
245 # A large build (with or without goma) tends to hog all system resources.
Philipp Wollermann0b943402023-10-12 07:13:30 +0000246 # Depending on the operating system, we might have mechanisms available
247 # to run at a lower priority, which improves this situation.
248 if os.environ.get('NINJA_BUILD_IN_BACKGROUND') == '1':
249 if sys.platform in ['darwin', 'linux']:
250 # nice-level 10 is usually considered a good default for background
251 # tasks. The niceness is inherited by child processes, so we can
252 # just set it here for us and it'll apply to the build tool we
253 # spawn later.
254 os.nice(10)
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000255
Philipp Wollermann0b943402023-10-12 07:13:30 +0000256 # Tell goma or reclient to do local compiles.
257 if offline:
258 os.environ['RBE_remote_disabled'] = '1'
259 os.environ['GOMA_DISABLED'] = '1'
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000260
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000261 # On macOS and most Linux distributions, the default limit of open file
262 # descriptors is too low (256 and 1024, respectively).
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000263 # This causes a large j value to result in 'Too many open files' errors.
264 # Check whether the limit can be raised to a large enough value. If yes,
265 # use `ulimit -n .... &&` as a prefix to increase the limit when running
266 # ninja.
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000267 if sys.platform in ['darwin', 'linux']:
268 # Increase the number of allowed open file descriptors to the maximum.
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000269 fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000270 if fileno_limit < hard_limit:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000271 try:
272 resource.setrlimit(resource.RLIMIT_NOFILE,
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000273 (hard_limit, hard_limit))
Philipp Wollermann0b943402023-10-12 07:13:30 +0000274 except Exception:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000275 pass
276 fileno_limit, hard_limit = resource.getrlimit(
277 resource.RLIMIT_NOFILE)
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000278
279 # Call ninja.py so that it can find ninja binary installed by DEPS or one in
280 # PATH.
281 ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py')
282 # If using remoteexec, use ninja_reclient.py which wraps ninja.py with
283 # starting and stopping reproxy.
284 if use_remoteexec:
285 ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py')
286
Philipp Wollermann0b943402023-10-12 07:13:30 +0000287 args = [sys.executable, ninja_path] + input_args[1:]
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000288
289 num_cores = multiprocessing.cpu_count()
290 if not j_specified and not t_specified:
Takuto Ikuta5cbc5212023-11-16 02:38:31 +0000291 if not offline and (use_goma or use_remoteexec):
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000292 args.append('-j')
293 default_core_multiplier = 80
294 if platform.machine() in ('x86_64', 'AMD64'):
295 # Assume simultaneous multithreading and therefore half as many
296 # cores as logical processors.
297 num_cores //= 2
298
299 core_multiplier = int(
300 os.environ.get('NINJA_CORE_MULTIPLIER',
301 default_core_multiplier))
302 j_value = num_cores * core_multiplier
303
304 core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value))
305 j_value = min(j_value, core_limit)
306
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000307 # On Windows, a -j higher than 1000 doesn't improve build times.
Henrique Ferreiro7eb4e482023-09-20 20:25:20 +0000308 # On macOS, ninja is limited to at most FD_SETSIZE (1024) open file
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000309 # descriptors.
Henrique Ferreiro7eb4e482023-09-20 20:25:20 +0000310 if sys.platform in ['darwin', 'win32']:
311 j_value = min(j_value, 1000)
312
313 # Use a j value that reliably works with the open file descriptors
314 # limit.
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000315 if sys.platform in ['darwin', 'linux']:
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000316 j_value = min(j_value, int(fileno_limit * 0.8))
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000317
318 args.append('%d' % j_value)
319 else:
320 j_value = num_cores
321 # Ninja defaults to |num_cores + 2|
322 j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
323 args.append('-j')
324 args.append('%d' % j_value)
325
Philipp Wollermann0b943402023-10-12 07:13:30 +0000326 if summarize_build:
327 # Enable statistics collection in Ninja.
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000328 args += ['-d', 'stats']
Philipp Wollermann0b943402023-10-12 07:13:30 +0000329 # Print the command-line to reassure the user that the right settings
330 # are being used.
331 _print_cmd(args)
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000332
Philipp Wollermann0b943402023-10-12 07:13:30 +0000333 if use_remoteexec:
334 return ninja_reclient.main(args[1:])
335 return ninja.main(args[1:])
Takuto Ikuta381db682022-04-27 23:54:02 +0000336
337
338if __name__ == '__main__':
Philipp Wollermann0b943402023-10-12 07:13:30 +0000339 try:
340 sys.exit(main(sys.argv))
341 except KeyboardInterrupt:
342 sys.exit(1)