blob: 34db3c359d22f4d484e38ea949adb85844a94165 [file] [log] [blame]
Takuto Ikuta237cc462022-01-19 18:04:15 +00001#!/usr/bin/env python3
Bruce Dawsonebebd952017-05-31 14:24:38 -07002# Copyright (c) 2017 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
Bruce Dawsonebebd952017-05-31 14:24:38 -07005"""
6This script (intended to be invoked by autoninja or autoninja.bat) detects
Simeon Anfinrud5dba9c92020-09-03 20:02:05 +00007whether a build is accelerated using a service like goma. If so, it runs with a
8large -j value, and otherwise it chooses a small one. This auto-adjustment
9makes using remote build acceleration simpler and safer, and avoids errors that
10can cause slow goma builds or swap-storms on unaccelerated builds.
Bruce Dawson30c1cba2023-09-15 18:20:32 +000011
12autoninja tries to detect relevant build settings such as use_remoteexec, and it
13does handle import statements, but it can't handle conditional setting of build
14settings.
Bruce Dawsonebebd952017-05-31 14:24:38 -070015"""
16
Bruce Dawsone952fae2021-02-27 23:33:37 +000017import multiprocessing
Bruce Dawsonebebd952017-05-31 14:24:38 -070018import os
Takuto Ikuta6a1494e2022-05-06 01:22:16 +000019import platform
Bruce Dawsonebebd952017-05-31 14:24:38 -070020import re
Bruce Dawsone952fae2021-02-27 23:33:37 +000021import subprocess
Bruce Dawsonebebd952017-05-31 14:24:38 -070022import sys
23
Henrique Ferreiro8bde1642023-09-14 11:41:19 +000024if sys.platform in ['darwin', 'linux']:
Mike Frysinger124bb8e2023-09-06 05:48:55 +000025 import resource
Sylvain Defresne7b4ecc72023-07-27 16:24:54 +000026
Yoshisato Yanagisawa4b497072018-11-07 02:52:33 +000027SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
28
Yoshisato Yanagisawaf66e5512018-11-15 00:40:39 +000029
Bruce Dawson30c1cba2023-09-15 18:20:32 +000030def _gn_lines(output_dir, path):
31 """
32 Generator function that returns args.gn lines one at a time, following
33 import directives as needed.
34 """
35 import_re = re.compile(r'\s*import\("(.*)"\)')
36 with open(path, encoding='utf-8') as f:
37 for line in f:
38 match = import_re.match(line)
39 if match:
40 import_path = os.path.normpath(
41 os.path.join(output_dir, '..', '..',
42 match.groups()[0][2:]))
43 for import_line in _gn_lines(output_dir, import_path):
44 yield import_line
45 else:
46 yield line
47
48
Takuto Ikuta381db682022-04-27 23:54:02 +000049def main(args):
Mike Frysinger124bb8e2023-09-06 05:48:55 +000050 # The -t tools are incompatible with -j
51 t_specified = False
52 j_specified = False
53 offline = False
54 output_dir = '.'
55 input_args = args
56 # On Windows the autoninja.bat script passes along the arguments enclosed in
57 # double quotes. This prevents multiple levels of parsing of the special '^'
58 # characters needed when compiling a single file but means that this script
59 # gets called with a single argument containing all of the actual arguments,
60 # separated by spaces. When this case is detected we need to do argument
61 # splitting ourselves. This means that arguments containing actual spaces
62 # are not supported by autoninja, but that is not a real limitation.
63 if (sys.platform.startswith('win') and len(args) == 2
64 and input_args[1].count(' ') > 0):
65 input_args = args[:1] + args[1].split()
Bruce Dawson655afeb2020-11-02 23:30:37 +000066
Mike Frysinger124bb8e2023-09-06 05:48:55 +000067 # Ninja uses getopt_long, which allow to intermix non-option arguments.
68 # To leave non supported parameters untouched, we do not use getopt.
69 for index, arg in enumerate(input_args[1:]):
70 if arg.startswith('-j'):
71 j_specified = True
72 if arg.startswith('-t'):
73 t_specified = True
74 if arg == '-C':
75 # + 1 to get the next argument and +1 because we trimmed off
76 # input_args[0]
77 output_dir = input_args[index + 2]
78 elif arg.startswith('-C'):
79 # Support -Cout/Default
80 output_dir = arg[2:]
81 elif arg in ('-o', '--offline'):
82 offline = True
83 elif arg == '-h':
84 print('autoninja: Use -o/--offline to temporary disable goma.',
85 file=sys.stderr)
86 print(file=sys.stderr)
Bruce Dawsone952fae2021-02-27 23:33:37 +000087
Takuto Ikuta381db682022-04-27 23:54:02 +000088 use_goma = False
Mike Frysinger124bb8e2023-09-06 05:48:55 +000089 use_remoteexec = False
90 use_rbe = False
91 use_siso = False
Yoshisato Yanagisawa43a35d22018-11-15 03:00:51 +000092
Mike Frysinger124bb8e2023-09-06 05:48:55 +000093 # Attempt to auto-detect remote build acceleration. We support gn-based
94 # builds, where we look for args.gn in the build tree, and cmake-based
95 # builds where we look for rules.ninja.
96 if os.path.exists(os.path.join(output_dir, 'args.gn')):
Bruce Dawson30c1cba2023-09-15 18:20:32 +000097 for line in _gn_lines(output_dir, os.path.join(output_dir, 'args.gn')):
98 # use_goma, use_remoteexec, or use_rbe will activate build
99 # acceleration.
100 #
101 # This test can match multi-argument lines. Examples of this
102 # are: is_debug=false use_goma=true is_official_build=false
103 # use_goma=false# use_goma=true This comment is ignored
104 #
105 # Anything after a comment is not consider a valid argument.
106 line_without_comment = line.split('#')[0]
107 if re.search(r'(^|\s)(use_goma)\s*=\s*true($|\s)',
108 line_without_comment):
109 use_goma = True
110 continue
111 if re.search(r'(^|\s)(use_remoteexec)\s*=\s*true($|\s)',
112 line_without_comment):
113 use_remoteexec = True
114 continue
115 if re.search(r'(^|\s)(use_rbe)\s*=\s*true($|\s)',
116 line_without_comment):
117 use_rbe = True
118 continue
119 if re.search(r'(^|\s)(use_siso)\s*=\s*true($|\s)',
120 line_without_comment):
121 use_siso = True
122 continue
Bruce Dawsonb3b46a22019-09-06 15:57:52 +0000123
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000124 siso_marker = os.path.join(output_dir, '.siso_deps')
125 if use_siso:
126 ninja_marker = os.path.join(output_dir, '.ninja_log')
127 # autosiso generates a .ninja_log file so the mere existence of a
128 # .ninja_log file doesn't imply that a ninja build was done. However
129 # if there is a .ninja_log but no .siso_deps then that implies a
130 # ninja build.
131 if os.path.exists(ninja_marker) and not os.path.exists(siso_marker):
132 return (
133 'echo Run gn clean before switching from ninja to siso in '
134 '%s' % output_dir)
135 siso = ['autosiso'] if use_remoteexec else ['siso', 'ninja']
136 if sys.platform.startswith('win'):
137 # An explicit 'call' is needed to make sure the invocation of
138 # autosiso returns to autoninja.bat, and the command prompt
139 # title gets reset.
140 siso = ['call'] + siso
141 return ' '.join(siso + input_args[1:])
Michael Savigny20eda952021-01-20 01:16:27 +0000142
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000143 if os.path.exists(siso_marker):
144 return (
145 'echo Run gn clean before switching from siso to ninja in %s' %
146 output_dir)
Ben Segall467991e2023-08-09 19:02:09 +0000147
Takuto Ikuta381db682022-04-27 23:54:02 +0000148 else:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000149 for relative_path in [
150 '', # GN keeps them in the root of output_dir
151 'CMakeFiles'
152 ]:
153 path = os.path.join(output_dir, relative_path, 'rules.ninja')
154 if os.path.exists(path):
Bruce Dawson30c1cba2023-09-15 18:20:32 +0000155 with open(path, encoding='utf-8') as file_handle:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000156 for line in file_handle:
157 if re.match(r'^\s*command\s*=\s*\S+gomacc', line):
158 use_goma = True
159 break
Takuto Ikuta381db682022-04-27 23:54:02 +0000160
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000161 # Strip -o/--offline so ninja doesn't see them.
162 input_args = [arg for arg in input_args if arg not in ('-o', '--offline')]
Takuto Ikuta381db682022-04-27 23:54:02 +0000163
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000164 # If GOMA_DISABLED is set to "true", "t", "yes", "y", or "1"
165 # (case-insensitive) then gomacc will use the local compiler instead of
166 # doing a goma compile. This is convenient if you want to briefly disable
167 # goma. It avoids having to rebuild the world when transitioning between
168 # goma/non-goma builds. However, it is not as fast as doing a "normal"
169 # non-goma build because an extra process is created for each compile step.
170 # Checking this environment variable ensures that autoninja uses an
171 # appropriate -j value in this situation.
172 goma_disabled_env = os.environ.get('GOMA_DISABLED', '0').lower()
173 if offline or goma_disabled_env in ['true', 't', 'yes', 'y', '1']:
174 use_goma = False
Takuto Ikuta381db682022-04-27 23:54:02 +0000175
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000176 if use_goma:
177 gomacc_file = 'gomacc.exe' if sys.platform.startswith(
178 'win') else 'gomacc'
179 goma_dir = os.environ.get('GOMA_DIR',
180 os.path.join(SCRIPT_DIR, '.cipd_bin'))
181 gomacc_path = os.path.join(goma_dir, gomacc_file)
182 # Don't invoke gomacc if it doesn't exist.
183 if os.path.exists(gomacc_path):
184 # Check to make sure that goma is running. If not, don't start the
185 # build.
186 status = subprocess.call([gomacc_path, 'port'],
187 stdout=subprocess.PIPE,
188 stderr=subprocess.PIPE,
189 shell=False)
190 if status == 1:
191 print(
192 'Goma is not running. Use "goma_ctl ensure_start" to start '
193 'it.',
194 file=sys.stderr)
195 if sys.platform.startswith('win'):
196 # Set an exit code of 1 in the batch file.
197 print('cmd "/c exit 1"')
198 else:
199 # Set an exit code of 1 by executing 'false' in the bash
200 # script.
201 print('false')
202 sys.exit(1)
203
204 # A large build (with or without goma) tends to hog all system resources.
205 # Launching the ninja process with 'nice' priorities improves this
206 # situation.
207 prefix_args = []
208 if (sys.platform.startswith('linux')
209 and os.environ.get('NINJA_BUILD_IN_BACKGROUND', '0') == '1'):
210 # nice -10 is process priority 10 lower than default 0
211 # ionice -c 3 is IO priority IDLE
212 prefix_args = ['nice'] + ['-10']
213
214 # Tell goma or reclient to do local compiles. On Windows these environment
215 # variables are set by the wrapper batch file.
216 offline_env = ['RBE_remote_disabled=1', 'GOMA_DISABLED=1'
217 ] if offline and not sys.platform.startswith('win') else []
218
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000219 # On macOS and most Linux distributions, the default limit of open file
220 # descriptors is too low (256 and 1024, respectively).
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000221 # This causes a large j value to result in 'Too many open files' errors.
222 # Check whether the limit can be raised to a large enough value. If yes,
223 # use `ulimit -n .... &&` as a prefix to increase the limit when running
224 # ninja.
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000225 prepend_command = []
226 if sys.platform in ['darwin', 'linux']:
227 # Increase the number of allowed open file descriptors to the maximum.
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000228 fileno_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000229 if fileno_limit < hard_limit:
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000230 try:
231 resource.setrlimit(resource.RLIMIT_NOFILE,
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000232 (hard_limit, hard_limit))
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000233 except Exception as _:
234 pass
235 fileno_limit, hard_limit = resource.getrlimit(
236 resource.RLIMIT_NOFILE)
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000237 if fileno_limit == hard_limit:
238 prepend_command = ['ulimit', '-n', f'{fileno_limit}', '&&']
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000239
240 # Call ninja.py so that it can find ninja binary installed by DEPS or one in
241 # PATH.
242 ninja_path = os.path.join(SCRIPT_DIR, 'ninja.py')
243 # If using remoteexec, use ninja_reclient.py which wraps ninja.py with
244 # starting and stopping reproxy.
245 if use_remoteexec:
246 ninja_path = os.path.join(SCRIPT_DIR, 'ninja_reclient.py')
247
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000248 args = prepend_command + offline_env + prefix_args + [
249 sys.executable, ninja_path
250 ] + input_args[1:]
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000251
252 num_cores = multiprocessing.cpu_count()
253 if not j_specified and not t_specified:
254 if not offline and (use_goma or use_remoteexec or use_rbe):
255 args.append('-j')
256 default_core_multiplier = 80
257 if platform.machine() in ('x86_64', 'AMD64'):
258 # Assume simultaneous multithreading and therefore half as many
259 # cores as logical processors.
260 num_cores //= 2
261
262 core_multiplier = int(
263 os.environ.get('NINJA_CORE_MULTIPLIER',
264 default_core_multiplier))
265 j_value = num_cores * core_multiplier
266
267 core_limit = int(os.environ.get('NINJA_CORE_LIMIT', j_value))
268 j_value = min(j_value, core_limit)
269
Henrique Ferreiro8bde1642023-09-14 11:41:19 +0000270 # On Windows, a -j higher than 1000 doesn't improve build times.
271 # On POSIX, ninja is limited to at most FD_SETSIZE (1024) open file
272 # descriptors.
273 j_value = min(j_value, 1000)
274 if sys.platform in ['darwin', 'linux']:
275 # Use a j value that reliably works with the open file
276 # descriptors limit.
277 j_value = min(j_value, int(fileno_limit * 0.8))
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000278
279 args.append('%d' % j_value)
280 else:
281 j_value = num_cores
282 # Ninja defaults to |num_cores + 2|
283 j_value += int(os.environ.get('NINJA_CORE_ADDITION', '2'))
284 args.append('-j')
285 args.append('%d' % j_value)
286
287 # On Windows, fully quote the path so that the command processor doesn't
288 # think the whole output is the command. On Linux and Mac, if people put
289 # depot_tools in directories with ' ', shell would misunderstand ' ' as a
290 # path separation. TODO(yyanagisawa): provide proper quoting for Windows.
291 # see https://cs.chromium.org/chromium/src/tools/mb/mb.py
292 for i in range(len(args)):
293 if (i == 0 and sys.platform.startswith('win')) or ' ' in args[i]:
294 args[i] = '"%s"' % args[i].replace('"', '\\"')
295
296 if os.environ.get('NINJA_SUMMARIZE_BUILD', '0') == '1':
297 args += ['-d', 'stats']
298
299 return ' '.join(args)
Takuto Ikuta381db682022-04-27 23:54:02 +0000300
301
302if __name__ == '__main__':
Mike Frysinger124bb8e2023-09-06 05:48:55 +0000303 print(main(sys.argv))