blob: 9de30f9523118df9afbb324e928df3fd0abdfaa3 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700111 'libgl1-mesa-dev',
112 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000113 'libgtest-dev',
114 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700115 'liblz4-tool',
116 'libncurses5',
117 'libnss3-dev',
118 'libprotobuf-dev',
119 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000120 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700121 'libssl-dev',
122 'libtinyxml2-dev',
123 'libx11-dev',
124 'libxml2-utils',
125 'ninja-build',
126 'openssl',
127 'protobuf-compiler',
128 'unzip',
129 'x11proto-core-dev',
130 'xsltproc',
131 'zip',
132 'zlib1g-dev',
133]
134
135# List of cargo packages required for linux build
136REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
137
138APT_PKG_LIST = ['apt', '-qq', 'list']
139CARGO_PKG_LIST = ['cargo', 'install', '--list']
140
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000141
142class UseFlags():
143
144 def __init__(self, use_flags):
145 """ Construct the use flags.
146
147 Args:
148 use_flags: List of use flags parsed from the command.
149 """
150 self.flags = {}
151
152 # Import use flags required by common-mk
153 for use in COMMON_MK_USES:
154 self.set_flag(use, False)
155
156 # Set our defaults
157 for use, value in USE_DEFAULTS.items():
158 self.set_flag(use, value)
159
160 # Set use flags - value is set to True unless the use starts with -
161 # All given use flags always override the defaults
162 for use in use_flags:
163 value = not use.startswith('-')
164 self.set_flag(use, value)
165
166 def set_flag(self, key, value=True):
167 setattr(self, key, value)
168 self.flags[key] = value
169
170
171class HostBuild():
172
173 def __init__(self, args):
174 """ Construct the builder.
175
176 Args:
177 args: Parsed arguments from ArgumentParser
178 """
179 self.args = args
180
181 # Set jobs to number of cpus unless explicitly set
182 self.jobs = self.args.jobs
183 if not self.jobs:
184 self.jobs = multiprocessing.cpu_count()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700185 sys.stderr.write("Number of jobs = {}\n".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000186
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700187 # Normalize bootstrap dir and make sure it exists
188 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
189 os.makedirs(self.bootstrap_dir, exist_ok=True)
190
191 # Output and platform directories are based on bootstrap
192 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
193 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Michael Sun4940f2b2022-09-15 16:08:24 -0700194 self.bt_dir = os.path.join(self.platform_dir, 'bt')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
Michael Sun4940f2b2022-09-15 16:08:24 -0700199 assert os.path.samefile(self.bt_dir,
200 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!"
201
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000202 # If default target isn't set, build everything
203 self.target = 'all'
204 if hasattr(self.args, 'target') and self.args.target:
205 self.target = self.args.target
206
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000207 target_use = self.args.use if self.args.use else []
208
209 # Unless set, always build test code
210 if not self.args.notest:
211 target_use.append('test')
212
213 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000214
215 # Validate platform directory
216 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
217 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
218
219 # Make sure output directory exists (or create it)
220 os.makedirs(self.output_dir, exist_ok=True)
221
222 # Set some default attributes
223 self.libbase_ver = None
224
225 self.configure_environ()
226
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000227 def _generate_rustflags(self):
228 """ Rustflags to include for the build.
229 """
230 rust_flags = [
231 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700232 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000233 '-C',
234 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700235 # exclude uninteresting warnings
236 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000237 ]
238
239 return ' '.join(rust_flags)
240
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000241 def configure_environ(self):
242 """ Configure environment variables for GN and Cargo.
243 """
244 self.env = os.environ.copy()
245
246 # Make sure cargo home dir exists and has a bin directory
247 cargo_home = os.path.join(self.output_dir, 'cargo_home')
248 os.makedirs(cargo_home, exist_ok=True)
249 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
250
251 # Configure Rust env variables
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700252 self.custom_env = {}
253 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
254 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
255 self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
256 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
257 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
258 self.custom_env['CXX_OUTDIR'] = self._gn_default_output()
259 self.env.update(self.custom_env)
260
261 def print_env(self):
262 """ Print the custom environment variables that are used in build.
263
264 Useful so that external tools can mimic the environment to be the same
265 as build.py, e.g. rust-analyzer.
266 """
267 for k, v in self.custom_env.items():
268 print("export {}='{}'".format(k, v))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000269
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000270 def run_command(self, target, args, cwd=None, env=None):
271 """ Run command and stream the output.
272 """
273 # Set some defaults
274 if not cwd:
275 cwd = self.platform_dir
276 if not env:
277 env = self.env
278
279 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
280 with open(log_file, 'wb') as lf:
281 rc = 0
282 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
283 while True:
284 line = process.stdout.readline()
285 print(line.decode('utf-8'), end="")
286 lf.write(line)
287 if not line:
288 rc = process.poll()
289 if rc is not None:
290 break
291
292 time.sleep(0.1)
293
294 if rc != 0:
295 raise Exception("Return code is {}".format(rc))
296
297 def _get_basever(self):
298 if self.libbase_ver:
299 return self.libbase_ver
300
301 self.libbase_ver = os.environ.get('BASE_VER', '')
302 if not self.libbase_ver:
303 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
304 try:
305 with open(base_file, 'r') as f:
306 self.libbase_ver = f.read().strip('\n')
307 except:
308 self.libbase_ver = 'NOT-INSTALLED'
309
310 return self.libbase_ver
311
312 def _gn_default_output(self):
313 return os.path.join(self.output_dir, 'out/Default')
314
315 def _gn_configure(self):
316 """ Configure all required parameters for platform2.
317
318 Mostly copied from //common-mk/platform2.py
319 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700320 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000321
322 def to_gn_string(s):
323 return '"%s"' % s.replace('"', '\\"')
324
325 def to_gn_list(strs):
326 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
327
328 def to_gn_args_args(gn_args):
329 for k, v in gn_args.items():
330 if isinstance(v, bool):
331 v = str(v).lower()
332 elif isinstance(v, list):
333 v = to_gn_list(v)
334 elif isinstance(v, six.string_types):
335 v = to_gn_string(v)
336 else:
337 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
338 yield '%s=%s' % (k.replace('-', '_'), v)
339
340 gn_args = {
341 'platform_subdir': 'bt',
342 'cc': 'clang' if clang else 'gcc',
343 'cxx': 'clang++' if clang else 'g++',
344 'ar': 'llvm-ar' if clang else 'ar',
345 'pkg-config': 'pkg-config',
346 'clang_cc': clang,
347 'clang_cxx': clang,
348 'OS': 'linux',
349 'sysroot': self.sysroot,
350 'libdir': os.path.join(self.sysroot, self.libdir),
351 'build_root': self.output_dir,
352 'platform2_root': self.platform_dir,
353 'libbase_ver': self._get_basever(),
354 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
355 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800356 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000357 'enable_werror': False,
358 }
359
360 if clang:
361 # Make sure to mark the clang use flag as true
362 self.use.set_flag('clang', True)
363 gn_args['external_cxxflags'] += ['-I/usr/include/']
364
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000365 gn_args_args = list(to_gn_args_args(gn_args))
366 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
367 gn_args_args += ['use={%s}' % (' '.join(use_args))]
368
369 gn_args = [
370 'gn',
371 'gen',
372 ]
373
374 if self.args.verbose:
375 gn_args.append('-v')
376
377 gn_args += [
378 '--root=%s' % self.platform_dir,
379 '--args=%s' % ' '.join(gn_args_args),
380 self._gn_default_output(),
381 ]
382
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700383 if 'PKG_CONFIG_PATH' in self.env:
384 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000385
386 self.run_command('configure', gn_args)
387
388 def _gn_build(self, target):
389 """ Generate the ninja command for the target and run it.
390 """
391 args = ['%s:%s' % ('bt', target)]
392 ninja_args = ['ninja', '-C', self._gn_default_output()]
393 if self.jobs:
394 ninja_args += ['-j', str(self.jobs)]
395 ninja_args += args
396
397 if self.args.verbose:
398 ninja_args.append('-v')
399
400 self.run_command('build', ninja_args)
401
402 def _rust_configure(self):
403 """ Generate config file at cargo_home so we use vendored crates.
404 """
405 template = """
406 [source.systembt]
407 directory = "{}/external/rust/vendor"
408
409 [source.crates-io]
410 replace-with = "systembt"
411 local-registry = "/nonexistent"
412 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700413
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700414 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700415 contents = template.format(self.platform_dir)
416 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
417 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000418
419 def _rust_build(self):
420 """ Run `cargo build` from platform2/bt directory.
421 """
422 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
423
424 def _target_prepare(self):
425 """ Target to prepare the output directory for building.
426
427 This runs gn gen to generate all rquired files and set up the Rust
428 config properly. This will be run
429 """
430 self._gn_configure()
431 self._rust_configure()
432
433 def _target_tools(self):
434 """ Build the tools target in an already prepared environment.
435 """
436 self._gn_build('tools')
437
438 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
439 shutil.copy(
440 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
441
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800442 def _target_docs(self):
443 """Build the Rust docs."""
444 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
445
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000446 def _target_rust(self):
447 """ Build rust artifacts in an already prepared environment.
448 """
449 self._rust_build()
450
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100451 def _target_rootcanal(self):
452 """ Build rust artifacts for RootCanal in an already prepared environment.
453 """
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700454 self.run_command(
455 'rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100456
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000457 def _target_main(self):
458 """ Build the main GN artifacts in an already prepared environment.
459 """
460 self._gn_build('all')
461
462 def _target_test(self):
463 """ Runs the host tests.
464 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000465 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800466 rust_test_cmd = ['cargo', 'test']
467 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700468 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800469
470 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100471 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000472
473 # Host tests second based on host test list
474 for t in HOST_TESTS:
475 self.run_command(
476 'test', [os.path.join(self.output_dir, 'out/Default', t)],
477 cwd=os.path.join(self.output_dir),
478 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000479
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800480 def _target_install(self):
481 """ Installs files required to run Floss to install directory.
482 """
483 # First make the install directory
484 prefix = self.install_dir
485 os.makedirs(prefix, exist_ok=True)
486
487 # Next save the cwd and change to install directory
488 last_cwd = os.getcwd()
489 os.chdir(prefix)
490
491 bindir = os.path.join(self.output_dir, 'debug')
492 srcdir = os.path.dirname(__file__)
493
494 install_map = [
495 {
496 'src': os.path.join(bindir, 'btadapterd'),
497 'dst': 'usr/libexec/bluetooth/btadapterd',
498 'strip': True
499 },
500 {
501 'src': os.path.join(bindir, 'btmanagerd'),
502 'dst': 'usr/libexec/bluetooth/btmanagerd',
503 'strip': True
504 },
505 {
506 'src': os.path.join(bindir, 'btclient'),
507 'dst': 'usr/local/bin/btclient',
508 'strip': True
509 },
510 ]
511
512 for v in install_map:
513 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
514 dst = os.path.join(prefix, partial_dst)
515
516 # Create dst directory first and copy file there
517 os.makedirs(os.path.dirname(dst), exist_ok=True)
518 print('Installing {}'.format(dst))
519 shutil.copy(src, dst)
520
521 # Binary should be marked for strip and no-strip option shouldn't be
522 # set. No-strip is useful while debugging.
523 if strip and not self.args.no_strip:
524 self.run_command('install', ['llvm-strip', dst])
525
526 # Put all files into a tar.gz for easier installation
527 tar_location = os.path.join(prefix, 'floss.tar.gz')
528 with tarfile.open(tar_location, 'w:gz') as tar:
529 for v in install_map:
530 tar.add(v['dst'])
531
532 print('Tarball created at {}'.format(tar_location))
533
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000534 def _target_clean(self):
535 """ Delete the output directory entirely.
536 """
537 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800538
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700539 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800540 try:
541 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
542 except FileNotFoundError:
543 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000544
545 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800546 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000547 """
548 self._target_prepare()
549 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000550 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700551 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000552
553 def build(self):
554 """ Builds according to self.target
555 """
556 print('Building target ', self.target)
557
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800558 # Validate that the target is valid
559 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800560 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800561 return
562
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000563 if self.target == 'prepare':
564 self._target_prepare()
565 elif self.target == 'tools':
566 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100567 elif self.target == 'rootcanal':
568 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000569 elif self.target == 'rust':
570 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800571 elif self.target == 'docs':
572 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000573 elif self.target == 'main':
574 self._target_main()
575 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000576 self._target_test()
577 elif self.target == 'clean':
578 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800579 elif self.target == 'install':
580 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000581 elif self.target == 'all':
582 self._target_all()
583
584
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700585class Bootstrap():
586
587 def __init__(self, base_dir, bt_dir):
588 """ Construct bootstrapper.
589
590 Args:
591 base_dir: Where to stage everything.
592 bt_dir: Where bluetooth source is kept (will be symlinked)
593 """
594 self.base_dir = os.path.abspath(base_dir)
595 self.bt_dir = os.path.abspath(bt_dir)
596
597 # Create base directory if it doesn't already exist
598 os.makedirs(self.base_dir, exist_ok=True)
599
600 if not os.path.isdir(self.bt_dir):
601 raise Exception('{} is not a valid directory'.format(self.bt_dir))
602
603 self.git_dir = os.path.join(self.base_dir, 'repos')
604 self.staging_dir = os.path.join(self.base_dir, 'staging')
605 self.output_dir = os.path.join(self.base_dir, 'output')
606 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
607
608 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
609
610 def _update_platform2(self):
611 """Updates repositories used for build."""
612 for repo in BOOTSTRAP_GIT_REPOS.keys():
613 cwd = os.path.join(self.git_dir, repo)
614 subprocess.check_call(['git', 'pull'], cwd=cwd)
615
616 def _setup_platform2(self):
617 """ Set up platform2.
618
619 This will check out all the git repos and symlink everything correctly.
620 """
621
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800622 # Create all directories we will need to use
623 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
624 os.makedirs(dirpath, exist_ok=True)
625
626 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700627 if os.path.isfile(self.dir_setup_complete):
628 print('{} already set-up. Updating instead.'.format(self.base_dir))
629 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800630 else:
631 # Check out all repos in git directory
632 for repo in BOOTSTRAP_GIT_REPOS.values():
633 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700634
635 # Symlink things
636 symlinks = [
637 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700638 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700639 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
640 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
641 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
642 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
643 ]
644
645 # Create symlinks
646 for pairs in symlinks:
647 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000648 try:
649 os.unlink(dst)
650 except Exception as e:
651 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700652 os.symlink(src, dst)
653
654 # Write to setup complete file so we don't repeat this step
655 with open(self.dir_setup_complete, 'w') as f:
656 f.write('Setup complete.')
657
658 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
659 """ Pretty print an install command.
660
661 Args:
662 install_cmd: Prefixed install command.
663 packages: Enumerate packages and append them to install command.
664 line_limit: Number of characters per line.
665
666 Return:
667 Array of lines to join and print.
668 """
669 install = [install_cmd]
670 line = ' '
671 # Remainder needed = space + len(pkg) + space + \
672 # Assuming 80 character lines, that's 80 - 3 = 77
673 line_limit = line_limit - 3
674 for pkg in packages:
675 if len(line) + len(pkg) < line_limit:
676 line = '{}{} '.format(line, pkg)
677 else:
678 install.append(line)
679 line = ' {} '.format(pkg)
680
681 if len(line) > 0:
682 install.append(line)
683
684 return install
685
686 def _check_package_installed(self, package, cmd, predicate):
687 """Check that the given package is installed.
688
689 Args:
690 package: Check that this package is installed.
691 cmd: Command prefix to check if installed (package appended to end)
692 predicate: Function/lambda to check if package is installed based
693 on output. Takes string output and returns boolean.
694
695 Return:
696 True if package is installed.
697 """
698 try:
699 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
700 is_installed = predicate(output.decode('utf-8'))
701 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
702
703 return is_installed
704 except Exception as e:
705 print(e)
706 return False
707
708 def _get_command_output(self, cmd):
709 """Runs the command and gets the output.
710
711 Args:
712 cmd: Command to run.
713
714 Return:
715 Tuple (Success, Output). Success represents if the command ran ok.
716 """
717 try:
718 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
719 return (True, output.decode('utf-8').split('\n'))
720 except Exception as e:
721 print(e)
722 return (False, "")
723
724 def _print_missing_packages(self):
725 """Print any missing packages found via apt.
726
727 This will find any missing packages necessary for build using apt and
728 print it out as an apt-get install printf.
729 """
730 print('Checking for any missing packages...')
731
732 (success, output) = self._get_command_output(APT_PKG_LIST)
733 if not success:
734 raise Exception("Could not query apt for packages.")
735
736 packages_installed = {}
737 for line in output:
738 if 'installed' in line:
739 split = line.split('/', 2)
740 packages_installed[split[0]] = True
741
742 need_packages = []
743 for pkg in REQUIRED_APT_PACKAGES:
744 if pkg not in packages_installed:
745 need_packages.append(pkg)
746
747 # No packages need to be installed
748 if len(need_packages) == 0:
749 print('+ All required packages are installed')
750 return
751
752 install = self._pretty_print_install('sudo apt-get install', need_packages)
753
754 # Print all lines so they can be run in cmdline
755 print('Missing system packages. Run the following command: ')
756 print(' \\\n'.join(install))
757
758 def _print_missing_rust_packages(self):
759 """Print any missing packages found via cargo.
760
761 This will find any missing packages necessary for build using cargo and
762 print it out as a cargo-install printf.
763 """
764 print('Checking for any missing cargo packages...')
765
766 (success, output) = self._get_command_output(CARGO_PKG_LIST)
767 if not success:
768 raise Exception("Could not query cargo for packages.")
769
770 packages_installed = {}
771 for line in output:
772 # Cargo installed packages have this format
773 # <crate name> <version>:
774 # <binary name>
775 # We only care about the crates themselves
776 if ':' not in line:
777 continue
778
779 split = line.split(' ', 2)
780 packages_installed[split[0]] = True
781
782 need_packages = []
783 for pkg in REQUIRED_CARGO_PACKAGES:
784 if pkg not in packages_installed:
785 need_packages.append(pkg)
786
787 # No packages to be installed
788 if len(need_packages) == 0:
789 print('+ All required cargo packages are installed')
790 return
791
792 install = self._pretty_print_install('cargo install', need_packages)
793 print('Missing cargo packages. Run the following command: ')
794 print(' \\\n'.join(install))
795
796 def bootstrap(self):
797 """ Bootstrap the Linux build."""
798 self._setup_platform2()
799 self._print_missing_packages()
800 self._print_missing_rust_packages()
801
802
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000803if __name__ == '__main__':
804 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700805 parser.add_argument(
806 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
807 parser.add_argument(
808 '--run-bootstrap',
809 help='Run bootstrap code to verify build env is ok to build.',
810 default=False,
811 action='store_true')
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700812 parser.add_argument(
813 '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000814 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800815 parser.add_argument(
816 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000817 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800818 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
819 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000820 parser.add_argument('--target', help='Run specific build target')
821 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700822 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000823 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700824 parser.add_argument(
825 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000826 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000827 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700828
829 # Make sure we get absolute path + expanded path for bootstrap directory
830 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
831
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000832 # Possible values for machine() come from 'uname -m'
833 # Since this script only runs on Linux, x86_64 machines must have this value
834 if platform.machine() != 'x86_64':
835 raise Exception("Only x86_64 machines are currently supported by this build script.")
836
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700837 if args.run_bootstrap:
838 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
839 bootstrap.bootstrap()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700840 elif args.print_env:
841 build = HostBuild(args)
842 build.print_env()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700843 else:
844 build = HostBuild(args)
845 build.build()