blob: 99912a8034d9e884c0c382eda199faa52df66cbb [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700111 'libgl1-mesa-dev',
112 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000113 'libgtest-dev',
114 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700115 'liblz4-tool',
116 'libncurses5',
117 'libnss3-dev',
118 'libprotobuf-dev',
119 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000120 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700121 'libssl-dev',
122 'libtinyxml2-dev',
123 'libx11-dev',
124 'libxml2-utils',
125 'ninja-build',
126 'openssl',
127 'protobuf-compiler',
128 'unzip',
129 'x11proto-core-dev',
130 'xsltproc',
131 'zip',
132 'zlib1g-dev',
133]
134
135# List of cargo packages required for linux build
136REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
137
138APT_PKG_LIST = ['apt', '-qq', 'list']
139CARGO_PKG_LIST = ['cargo', 'install', '--list']
140
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000141
142class UseFlags():
143
144 def __init__(self, use_flags):
145 """ Construct the use flags.
146
147 Args:
148 use_flags: List of use flags parsed from the command.
149 """
150 self.flags = {}
151
152 # Import use flags required by common-mk
153 for use in COMMON_MK_USES:
154 self.set_flag(use, False)
155
156 # Set our defaults
157 for use, value in USE_DEFAULTS.items():
158 self.set_flag(use, value)
159
160 # Set use flags - value is set to True unless the use starts with -
161 # All given use flags always override the defaults
162 for use in use_flags:
163 value = not use.startswith('-')
164 self.set_flag(use, value)
165
166 def set_flag(self, key, value=True):
167 setattr(self, key, value)
168 self.flags[key] = value
169
170
171class HostBuild():
172
173 def __init__(self, args):
174 """ Construct the builder.
175
176 Args:
177 args: Parsed arguments from ArgumentParser
178 """
179 self.args = args
180
181 # Set jobs to number of cpus unless explicitly set
182 self.jobs = self.args.jobs
183 if not self.jobs:
184 self.jobs = multiprocessing.cpu_count()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700185 sys.stderr.write("Number of jobs = {}\n".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000186
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700187 # Normalize bootstrap dir and make sure it exists
188 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
189 os.makedirs(self.bootstrap_dir, exist_ok=True)
190
191 # Output and platform directories are based on bootstrap
192 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
193 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Michael Sun4940f2b2022-09-15 16:08:24 -0700194 self.bt_dir = os.path.join(self.platform_dir, 'bt')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
Michael Sun4940f2b2022-09-15 16:08:24 -0700199 assert os.path.samefile(self.bt_dir,
200 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!"
201
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000202 # If default target isn't set, build everything
203 self.target = 'all'
204 if hasattr(self.args, 'target') and self.args.target:
205 self.target = self.args.target
206
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000207 target_use = self.args.use if self.args.use else []
208
209 # Unless set, always build test code
210 if not self.args.notest:
211 target_use.append('test')
212
213 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000214
215 # Validate platform directory
216 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
217 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
218
219 # Make sure output directory exists (or create it)
220 os.makedirs(self.output_dir, exist_ok=True)
221
222 # Set some default attributes
223 self.libbase_ver = None
224
225 self.configure_environ()
226
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000227 def _generate_rustflags(self):
228 """ Rustflags to include for the build.
229 """
230 rust_flags = [
231 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700232 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000233 '-C',
234 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700235 # exclude uninteresting warnings
236 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000237 ]
238
239 return ' '.join(rust_flags)
240
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000241 def configure_environ(self):
242 """ Configure environment variables for GN and Cargo.
243 """
244 self.env = os.environ.copy()
245
246 # Make sure cargo home dir exists and has a bin directory
247 cargo_home = os.path.join(self.output_dir, 'cargo_home')
248 os.makedirs(cargo_home, exist_ok=True)
249 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
250
251 # Configure Rust env variables
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700252 self.custom_env = {}
253 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
254 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
255 self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
256 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
257 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
258 self.custom_env['CXX_OUTDIR'] = self._gn_default_output()
259 self.env.update(self.custom_env)
260
261 def print_env(self):
262 """ Print the custom environment variables that are used in build.
263
264 Useful so that external tools can mimic the environment to be the same
265 as build.py, e.g. rust-analyzer.
266 """
267 for k, v in self.custom_env.items():
268 print("export {}='{}'".format(k, v))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000269
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000270 def run_command(self, target, args, cwd=None, env=None):
271 """ Run command and stream the output.
272 """
273 # Set some defaults
274 if not cwd:
275 cwd = self.platform_dir
276 if not env:
277 env = self.env
278
279 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
280 with open(log_file, 'wb') as lf:
281 rc = 0
282 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
283 while True:
284 line = process.stdout.readline()
285 print(line.decode('utf-8'), end="")
286 lf.write(line)
287 if not line:
288 rc = process.poll()
289 if rc is not None:
290 break
291
292 time.sleep(0.1)
293
294 if rc != 0:
295 raise Exception("Return code is {}".format(rc))
296
297 def _get_basever(self):
298 if self.libbase_ver:
299 return self.libbase_ver
300
301 self.libbase_ver = os.environ.get('BASE_VER', '')
302 if not self.libbase_ver:
303 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
304 try:
305 with open(base_file, 'r') as f:
306 self.libbase_ver = f.read().strip('\n')
307 except:
308 self.libbase_ver = 'NOT-INSTALLED'
309
310 return self.libbase_ver
311
312 def _gn_default_output(self):
313 return os.path.join(self.output_dir, 'out/Default')
314
315 def _gn_configure(self):
316 """ Configure all required parameters for platform2.
317
318 Mostly copied from //common-mk/platform2.py
319 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700320 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000321
322 def to_gn_string(s):
323 return '"%s"' % s.replace('"', '\\"')
324
325 def to_gn_list(strs):
326 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
327
328 def to_gn_args_args(gn_args):
329 for k, v in gn_args.items():
330 if isinstance(v, bool):
331 v = str(v).lower()
332 elif isinstance(v, list):
333 v = to_gn_list(v)
334 elif isinstance(v, six.string_types):
335 v = to_gn_string(v)
336 else:
337 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
338 yield '%s=%s' % (k.replace('-', '_'), v)
339
340 gn_args = {
341 'platform_subdir': 'bt',
342 'cc': 'clang' if clang else 'gcc',
343 'cxx': 'clang++' if clang else 'g++',
344 'ar': 'llvm-ar' if clang else 'ar',
345 'pkg-config': 'pkg-config',
346 'clang_cc': clang,
347 'clang_cxx': clang,
348 'OS': 'linux',
349 'sysroot': self.sysroot,
350 'libdir': os.path.join(self.sysroot, self.libdir),
351 'build_root': self.output_dir,
352 'platform2_root': self.platform_dir,
353 'libbase_ver': self._get_basever(),
354 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
355 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800356 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000357 'enable_werror': False,
358 }
359
360 if clang:
361 # Make sure to mark the clang use flag as true
362 self.use.set_flag('clang', True)
363 gn_args['external_cxxflags'] += ['-I/usr/include/']
364
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000365 gn_args_args = list(to_gn_args_args(gn_args))
366 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
367 gn_args_args += ['use={%s}' % (' '.join(use_args))]
368
369 gn_args = [
370 'gn',
371 'gen',
372 ]
373
374 if self.args.verbose:
375 gn_args.append('-v')
376
377 gn_args += [
378 '--root=%s' % self.platform_dir,
379 '--args=%s' % ' '.join(gn_args_args),
380 self._gn_default_output(),
381 ]
382
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700383 if 'PKG_CONFIG_PATH' in self.env:
384 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000385
386 self.run_command('configure', gn_args)
387
388 def _gn_build(self, target):
389 """ Generate the ninja command for the target and run it.
390 """
391 args = ['%s:%s' % ('bt', target)]
392 ninja_args = ['ninja', '-C', self._gn_default_output()]
393 if self.jobs:
394 ninja_args += ['-j', str(self.jobs)]
395 ninja_args += args
396
397 if self.args.verbose:
398 ninja_args.append('-v')
399
400 self.run_command('build', ninja_args)
401
402 def _rust_configure(self):
403 """ Generate config file at cargo_home so we use vendored crates.
404 """
405 template = """
406 [source.systembt]
407 directory = "{}/external/rust/vendor"
408
409 [source.crates-io]
410 replace-with = "systembt"
411 local-registry = "/nonexistent"
412 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700413
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700414 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700415 contents = template.format(self.platform_dir)
416 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
417 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000418
419 def _rust_build(self):
420 """ Run `cargo build` from platform2/bt directory.
421 """
Abhishek Pandit-Subedi30e57302022-10-17 13:41:09 -0700422 cmd = ['cargo', 'build']
423 if not self.args.rust_debug:
424 cmd.append('--release')
425
426 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000427
428 def _target_prepare(self):
429 """ Target to prepare the output directory for building.
430
431 This runs gn gen to generate all rquired files and set up the Rust
432 config properly. This will be run
433 """
434 self._gn_configure()
435 self._rust_configure()
436
437 def _target_tools(self):
438 """ Build the tools target in an already prepared environment.
439 """
440 self._gn_build('tools')
441
442 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
443 shutil.copy(
444 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
445
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800446 def _target_docs(self):
447 """Build the Rust docs."""
448 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
449
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000450 def _target_rust(self):
451 """ Build rust artifacts in an already prepared environment.
452 """
453 self._rust_build()
454
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100455 def _target_rootcanal(self):
456 """ Build rust artifacts for RootCanal in an already prepared environment.
457 """
Abhishek Pandit-Subedi30e57302022-10-17 13:41:09 -0700458 cmd = ['cargo', 'build']
459 if not self.args.rust_debug:
460 cmd.append('--release')
461
462 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100463
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000464 def _target_main(self):
465 """ Build the main GN artifacts in an already prepared environment.
466 """
467 self._gn_build('all')
468
469 def _target_test(self):
470 """ Runs the host tests.
471 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000472 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800473 rust_test_cmd = ['cargo', 'test']
Abhishek Pandit-Subedi30e57302022-10-17 13:41:09 -0700474 if not self.args.rust_debug:
475 rust_test_cmd.append('--release')
476
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800477 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700478 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800479
480 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100481 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000482
483 # Host tests second based on host test list
484 for t in HOST_TESTS:
485 self.run_command(
486 'test', [os.path.join(self.output_dir, 'out/Default', t)],
487 cwd=os.path.join(self.output_dir),
488 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000489
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800490 def _target_install(self):
491 """ Installs files required to run Floss to install directory.
492 """
493 # First make the install directory
494 prefix = self.install_dir
495 os.makedirs(prefix, exist_ok=True)
496
497 # Next save the cwd and change to install directory
498 last_cwd = os.getcwd()
499 os.chdir(prefix)
500
501 bindir = os.path.join(self.output_dir, 'debug')
502 srcdir = os.path.dirname(__file__)
503
504 install_map = [
505 {
506 'src': os.path.join(bindir, 'btadapterd'),
507 'dst': 'usr/libexec/bluetooth/btadapterd',
508 'strip': True
509 },
510 {
511 'src': os.path.join(bindir, 'btmanagerd'),
512 'dst': 'usr/libexec/bluetooth/btmanagerd',
513 'strip': True
514 },
515 {
516 'src': os.path.join(bindir, 'btclient'),
517 'dst': 'usr/local/bin/btclient',
518 'strip': True
519 },
520 ]
521
522 for v in install_map:
523 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
524 dst = os.path.join(prefix, partial_dst)
525
526 # Create dst directory first and copy file there
527 os.makedirs(os.path.dirname(dst), exist_ok=True)
528 print('Installing {}'.format(dst))
529 shutil.copy(src, dst)
530
531 # Binary should be marked for strip and no-strip option shouldn't be
532 # set. No-strip is useful while debugging.
533 if strip and not self.args.no_strip:
534 self.run_command('install', ['llvm-strip', dst])
535
536 # Put all files into a tar.gz for easier installation
537 tar_location = os.path.join(prefix, 'floss.tar.gz')
538 with tarfile.open(tar_location, 'w:gz') as tar:
539 for v in install_map:
540 tar.add(v['dst'])
541
542 print('Tarball created at {}'.format(tar_location))
543
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000544 def _target_clean(self):
545 """ Delete the output directory entirely.
546 """
547 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800548
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700549 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedi16617282022-09-28 10:11:10 -0700550 cargo_lock_files = [
551 os.path.join(self.platform_dir, 'bt', 'Cargo.lock'),
552 os.path.join(self.platform_dir, 'bt', 'tools', 'rootcanal', 'Cargo.lock'),
553 ]
554 for lock_file in cargo_lock_files:
555 try:
556 os.remove(lock_file)
557 print('Removed {}'.format(lock_file))
558 except FileNotFoundError:
559 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000560
561 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800562 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000563 """
564 self._target_prepare()
565 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000566 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700567 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000568
569 def build(self):
570 """ Builds according to self.target
571 """
572 print('Building target ', self.target)
573
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800574 # Validate that the target is valid
575 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800576 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800577 return
578
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000579 if self.target == 'prepare':
580 self._target_prepare()
581 elif self.target == 'tools':
582 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100583 elif self.target == 'rootcanal':
584 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000585 elif self.target == 'rust':
586 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800587 elif self.target == 'docs':
588 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000589 elif self.target == 'main':
590 self._target_main()
591 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000592 self._target_test()
593 elif self.target == 'clean':
594 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800595 elif self.target == 'install':
596 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000597 elif self.target == 'all':
598 self._target_all()
599
600
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700601class Bootstrap():
602
603 def __init__(self, base_dir, bt_dir):
604 """ Construct bootstrapper.
605
606 Args:
607 base_dir: Where to stage everything.
608 bt_dir: Where bluetooth source is kept (will be symlinked)
609 """
610 self.base_dir = os.path.abspath(base_dir)
611 self.bt_dir = os.path.abspath(bt_dir)
612
613 # Create base directory if it doesn't already exist
614 os.makedirs(self.base_dir, exist_ok=True)
615
616 if not os.path.isdir(self.bt_dir):
617 raise Exception('{} is not a valid directory'.format(self.bt_dir))
618
619 self.git_dir = os.path.join(self.base_dir, 'repos')
620 self.staging_dir = os.path.join(self.base_dir, 'staging')
621 self.output_dir = os.path.join(self.base_dir, 'output')
622 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
623
624 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
625
626 def _update_platform2(self):
627 """Updates repositories used for build."""
628 for repo in BOOTSTRAP_GIT_REPOS.keys():
629 cwd = os.path.join(self.git_dir, repo)
630 subprocess.check_call(['git', 'pull'], cwd=cwd)
631
632 def _setup_platform2(self):
633 """ Set up platform2.
634
635 This will check out all the git repos and symlink everything correctly.
636 """
637
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800638 # Create all directories we will need to use
639 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
640 os.makedirs(dirpath, exist_ok=True)
641
642 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700643 if os.path.isfile(self.dir_setup_complete):
644 print('{} already set-up. Updating instead.'.format(self.base_dir))
645 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800646 else:
647 # Check out all repos in git directory
648 for repo in BOOTSTRAP_GIT_REPOS.values():
649 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700650
651 # Symlink things
652 symlinks = [
653 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700654 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700655 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
656 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
657 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
658 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
659 ]
660
661 # Create symlinks
662 for pairs in symlinks:
663 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000664 try:
665 os.unlink(dst)
666 except Exception as e:
667 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700668 os.symlink(src, dst)
669
670 # Write to setup complete file so we don't repeat this step
671 with open(self.dir_setup_complete, 'w') as f:
672 f.write('Setup complete.')
673
674 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
675 """ Pretty print an install command.
676
677 Args:
678 install_cmd: Prefixed install command.
679 packages: Enumerate packages and append them to install command.
680 line_limit: Number of characters per line.
681
682 Return:
683 Array of lines to join and print.
684 """
685 install = [install_cmd]
686 line = ' '
687 # Remainder needed = space + len(pkg) + space + \
688 # Assuming 80 character lines, that's 80 - 3 = 77
689 line_limit = line_limit - 3
690 for pkg in packages:
691 if len(line) + len(pkg) < line_limit:
692 line = '{}{} '.format(line, pkg)
693 else:
694 install.append(line)
695 line = ' {} '.format(pkg)
696
697 if len(line) > 0:
698 install.append(line)
699
700 return install
701
702 def _check_package_installed(self, package, cmd, predicate):
703 """Check that the given package is installed.
704
705 Args:
706 package: Check that this package is installed.
707 cmd: Command prefix to check if installed (package appended to end)
708 predicate: Function/lambda to check if package is installed based
709 on output. Takes string output and returns boolean.
710
711 Return:
712 True if package is installed.
713 """
714 try:
715 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
716 is_installed = predicate(output.decode('utf-8'))
717 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
718
719 return is_installed
720 except Exception as e:
721 print(e)
722 return False
723
724 def _get_command_output(self, cmd):
725 """Runs the command and gets the output.
726
727 Args:
728 cmd: Command to run.
729
730 Return:
731 Tuple (Success, Output). Success represents if the command ran ok.
732 """
733 try:
734 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
735 return (True, output.decode('utf-8').split('\n'))
736 except Exception as e:
737 print(e)
738 return (False, "")
739
740 def _print_missing_packages(self):
741 """Print any missing packages found via apt.
742
743 This will find any missing packages necessary for build using apt and
744 print it out as an apt-get install printf.
745 """
746 print('Checking for any missing packages...')
747
748 (success, output) = self._get_command_output(APT_PKG_LIST)
749 if not success:
750 raise Exception("Could not query apt for packages.")
751
752 packages_installed = {}
753 for line in output:
754 if 'installed' in line:
755 split = line.split('/', 2)
756 packages_installed[split[0]] = True
757
758 need_packages = []
759 for pkg in REQUIRED_APT_PACKAGES:
760 if pkg not in packages_installed:
761 need_packages.append(pkg)
762
763 # No packages need to be installed
764 if len(need_packages) == 0:
765 print('+ All required packages are installed')
766 return
767
768 install = self._pretty_print_install('sudo apt-get install', need_packages)
769
770 # Print all lines so they can be run in cmdline
771 print('Missing system packages. Run the following command: ')
772 print(' \\\n'.join(install))
773
774 def _print_missing_rust_packages(self):
775 """Print any missing packages found via cargo.
776
777 This will find any missing packages necessary for build using cargo and
778 print it out as a cargo-install printf.
779 """
780 print('Checking for any missing cargo packages...')
781
782 (success, output) = self._get_command_output(CARGO_PKG_LIST)
783 if not success:
784 raise Exception("Could not query cargo for packages.")
785
786 packages_installed = {}
787 for line in output:
788 # Cargo installed packages have this format
789 # <crate name> <version>:
790 # <binary name>
791 # We only care about the crates themselves
792 if ':' not in line:
793 continue
794
795 split = line.split(' ', 2)
796 packages_installed[split[0]] = True
797
798 need_packages = []
799 for pkg in REQUIRED_CARGO_PACKAGES:
800 if pkg not in packages_installed:
801 need_packages.append(pkg)
802
803 # No packages to be installed
804 if len(need_packages) == 0:
805 print('+ All required cargo packages are installed')
806 return
807
808 install = self._pretty_print_install('cargo install', need_packages)
809 print('Missing cargo packages. Run the following command: ')
810 print(' \\\n'.join(install))
811
812 def bootstrap(self):
813 """ Bootstrap the Linux build."""
814 self._setup_platform2()
815 self._print_missing_packages()
816 self._print_missing_rust_packages()
817
818
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000819if __name__ == '__main__':
820 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700821 parser.add_argument(
822 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
823 parser.add_argument(
824 '--run-bootstrap',
825 help='Run bootstrap code to verify build env is ok to build.',
826 default=False,
827 action='store_true')
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700828 parser.add_argument(
829 '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000830 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800831 parser.add_argument(
832 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000833 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800834 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
835 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000836 parser.add_argument('--target', help='Run specific build target')
837 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700838 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000839 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700840 parser.add_argument(
841 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000842 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedi30e57302022-10-17 13:41:09 -0700843 parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000844 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700845
846 # Make sure we get absolute path + expanded path for bootstrap directory
847 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
848
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000849 # Possible values for machine() come from 'uname -m'
850 # Since this script only runs on Linux, x86_64 machines must have this value
851 if platform.machine() != 'x86_64':
852 raise Exception("Only x86_64 machines are currently supported by this build script.")
853
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700854 if args.run_bootstrap:
855 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
856 bootstrap.bootstrap()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700857 elif args.print_env:
858 build = HostBuild(args)
859 build.print_env()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700860 else:
861 build = HostBuild(args)
862 build.build()