blob: 049189aba4e45306d594363e21de7d7a38ff6f2e [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700111 'libgl1-mesa-dev',
112 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000113 'libgtest-dev',
114 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700115 'liblz4-tool',
116 'libncurses5',
117 'libnss3-dev',
118 'libprotobuf-dev',
119 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000120 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700121 'libssl-dev',
122 'libtinyxml2-dev',
123 'libx11-dev',
124 'libxml2-utils',
125 'ninja-build',
126 'openssl',
127 'protobuf-compiler',
128 'unzip',
129 'x11proto-core-dev',
130 'xsltproc',
131 'zip',
132 'zlib1g-dev',
133]
134
135# List of cargo packages required for linux build
136REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
137
138APT_PKG_LIST = ['apt', '-qq', 'list']
139CARGO_PKG_LIST = ['cargo', 'install', '--list']
140
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000141
142class UseFlags():
143
144 def __init__(self, use_flags):
145 """ Construct the use flags.
146
147 Args:
148 use_flags: List of use flags parsed from the command.
149 """
150 self.flags = {}
151
152 # Import use flags required by common-mk
153 for use in COMMON_MK_USES:
154 self.set_flag(use, False)
155
156 # Set our defaults
157 for use, value in USE_DEFAULTS.items():
158 self.set_flag(use, value)
159
160 # Set use flags - value is set to True unless the use starts with -
161 # All given use flags always override the defaults
162 for use in use_flags:
163 value = not use.startswith('-')
164 self.set_flag(use, value)
165
166 def set_flag(self, key, value=True):
167 setattr(self, key, value)
168 self.flags[key] = value
169
170
171class HostBuild():
172
173 def __init__(self, args):
174 """ Construct the builder.
175
176 Args:
177 args: Parsed arguments from ArgumentParser
178 """
179 self.args = args
180
181 # Set jobs to number of cpus unless explicitly set
182 self.jobs = self.args.jobs
183 if not self.jobs:
184 self.jobs = multiprocessing.cpu_count()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700185 sys.stderr.write("Number of jobs = {}\n".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000186
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700187 # Normalize bootstrap dir and make sure it exists
188 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
189 os.makedirs(self.bootstrap_dir, exist_ok=True)
190
191 # Output and platform directories are based on bootstrap
192 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
193 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Michael Sun4940f2b2022-09-15 16:08:24 -0700194 self.bt_dir = os.path.join(self.platform_dir, 'bt')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
Michael Sun4940f2b2022-09-15 16:08:24 -0700199 assert os.path.samefile(self.bt_dir,
200 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!"
201
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000202 # If default target isn't set, build everything
203 self.target = 'all'
204 if hasattr(self.args, 'target') and self.args.target:
205 self.target = self.args.target
206
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000207 target_use = self.args.use if self.args.use else []
208
209 # Unless set, always build test code
210 if not self.args.notest:
211 target_use.append('test')
212
213 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000214
215 # Validate platform directory
216 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
217 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
218
219 # Make sure output directory exists (or create it)
220 os.makedirs(self.output_dir, exist_ok=True)
221
222 # Set some default attributes
223 self.libbase_ver = None
224
225 self.configure_environ()
226
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000227 def _generate_rustflags(self):
228 """ Rustflags to include for the build.
229 """
230 rust_flags = [
231 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700232 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000233 '-C',
234 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700235 # exclude uninteresting warnings
236 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000237 ]
238
239 return ' '.join(rust_flags)
240
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000241 def configure_environ(self):
242 """ Configure environment variables for GN and Cargo.
243 """
244 self.env = os.environ.copy()
245
246 # Make sure cargo home dir exists and has a bin directory
247 cargo_home = os.path.join(self.output_dir, 'cargo_home')
248 os.makedirs(cargo_home, exist_ok=True)
249 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
250
251 # Configure Rust env variables
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700252 self.custom_env = {}
253 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
254 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
255 self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
256 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
257 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
258 self.custom_env['CXX_OUTDIR'] = self._gn_default_output()
259 self.env.update(self.custom_env)
260
261 def print_env(self):
262 """ Print the custom environment variables that are used in build.
263
264 Useful so that external tools can mimic the environment to be the same
265 as build.py, e.g. rust-analyzer.
266 """
267 for k, v in self.custom_env.items():
268 print("export {}='{}'".format(k, v))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000269
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000270 def run_command(self, target, args, cwd=None, env=None):
271 """ Run command and stream the output.
272 """
273 # Set some defaults
274 if not cwd:
275 cwd = self.platform_dir
276 if not env:
277 env = self.env
278
279 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
280 with open(log_file, 'wb') as lf:
281 rc = 0
282 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
283 while True:
284 line = process.stdout.readline()
285 print(line.decode('utf-8'), end="")
286 lf.write(line)
287 if not line:
288 rc = process.poll()
289 if rc is not None:
290 break
291
292 time.sleep(0.1)
293
294 if rc != 0:
295 raise Exception("Return code is {}".format(rc))
296
297 def _get_basever(self):
298 if self.libbase_ver:
299 return self.libbase_ver
300
301 self.libbase_ver = os.environ.get('BASE_VER', '')
302 if not self.libbase_ver:
303 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
304 try:
305 with open(base_file, 'r') as f:
306 self.libbase_ver = f.read().strip('\n')
307 except:
308 self.libbase_ver = 'NOT-INSTALLED'
309
310 return self.libbase_ver
311
312 def _gn_default_output(self):
313 return os.path.join(self.output_dir, 'out/Default')
314
315 def _gn_configure(self):
316 """ Configure all required parameters for platform2.
317
318 Mostly copied from //common-mk/platform2.py
319 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700320 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000321
322 def to_gn_string(s):
323 return '"%s"' % s.replace('"', '\\"')
324
325 def to_gn_list(strs):
326 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
327
328 def to_gn_args_args(gn_args):
329 for k, v in gn_args.items():
330 if isinstance(v, bool):
331 v = str(v).lower()
332 elif isinstance(v, list):
333 v = to_gn_list(v)
334 elif isinstance(v, six.string_types):
335 v = to_gn_string(v)
336 else:
337 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
338 yield '%s=%s' % (k.replace('-', '_'), v)
339
340 gn_args = {
341 'platform_subdir': 'bt',
342 'cc': 'clang' if clang else 'gcc',
343 'cxx': 'clang++' if clang else 'g++',
344 'ar': 'llvm-ar' if clang else 'ar',
345 'pkg-config': 'pkg-config',
346 'clang_cc': clang,
347 'clang_cxx': clang,
348 'OS': 'linux',
349 'sysroot': self.sysroot,
350 'libdir': os.path.join(self.sysroot, self.libdir),
351 'build_root': self.output_dir,
352 'platform2_root': self.platform_dir,
353 'libbase_ver': self._get_basever(),
354 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
355 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800356 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000357 'enable_werror': False,
358 }
359
360 if clang:
361 # Make sure to mark the clang use flag as true
362 self.use.set_flag('clang', True)
363 gn_args['external_cxxflags'] += ['-I/usr/include/']
364
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000365 gn_args_args = list(to_gn_args_args(gn_args))
366 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
367 gn_args_args += ['use={%s}' % (' '.join(use_args))]
368
369 gn_args = [
370 'gn',
371 'gen',
372 ]
373
374 if self.args.verbose:
375 gn_args.append('-v')
376
377 gn_args += [
378 '--root=%s' % self.platform_dir,
379 '--args=%s' % ' '.join(gn_args_args),
380 self._gn_default_output(),
381 ]
382
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700383 if 'PKG_CONFIG_PATH' in self.env:
384 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000385
386 self.run_command('configure', gn_args)
387
388 def _gn_build(self, target):
389 """ Generate the ninja command for the target and run it.
390 """
391 args = ['%s:%s' % ('bt', target)]
392 ninja_args = ['ninja', '-C', self._gn_default_output()]
393 if self.jobs:
394 ninja_args += ['-j', str(self.jobs)]
395 ninja_args += args
396
397 if self.args.verbose:
398 ninja_args.append('-v')
399
400 self.run_command('build', ninja_args)
401
402 def _rust_configure(self):
403 """ Generate config file at cargo_home so we use vendored crates.
404 """
405 template = """
406 [source.systembt]
407 directory = "{}/external/rust/vendor"
408
409 [source.crates-io]
410 replace-with = "systembt"
411 local-registry = "/nonexistent"
412 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700413
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700414 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700415 contents = template.format(self.platform_dir)
416 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
417 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000418
419 def _rust_build(self):
420 """ Run `cargo build` from platform2/bt directory.
421 """
422 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
423
424 def _target_prepare(self):
425 """ Target to prepare the output directory for building.
426
427 This runs gn gen to generate all rquired files and set up the Rust
428 config properly. This will be run
429 """
430 self._gn_configure()
431 self._rust_configure()
432
433 def _target_tools(self):
434 """ Build the tools target in an already prepared environment.
435 """
436 self._gn_build('tools')
437
438 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
439 shutil.copy(
440 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
441
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800442 def _target_docs(self):
443 """Build the Rust docs."""
444 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
445
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000446 def _target_rust(self):
447 """ Build rust artifacts in an already prepared environment.
448 """
449 self._rust_build()
450
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100451 def _target_rootcanal(self):
452 """ Build rust artifacts for RootCanal in an already prepared environment.
453 """
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700454 self.run_command(
455 'rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100456
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000457 def _target_main(self):
458 """ Build the main GN artifacts in an already prepared environment.
459 """
460 self._gn_build('all')
461
462 def _target_test(self):
463 """ Runs the host tests.
464 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000465 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800466 rust_test_cmd = ['cargo', 'test']
467 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700468 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800469
470 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100471 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000472
473 # Host tests second based on host test list
474 for t in HOST_TESTS:
475 self.run_command(
476 'test', [os.path.join(self.output_dir, 'out/Default', t)],
477 cwd=os.path.join(self.output_dir),
478 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000479
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800480 def _target_install(self):
481 """ Installs files required to run Floss to install directory.
482 """
483 # First make the install directory
484 prefix = self.install_dir
485 os.makedirs(prefix, exist_ok=True)
486
487 # Next save the cwd and change to install directory
488 last_cwd = os.getcwd()
489 os.chdir(prefix)
490
491 bindir = os.path.join(self.output_dir, 'debug')
492 srcdir = os.path.dirname(__file__)
493
494 install_map = [
495 {
496 'src': os.path.join(bindir, 'btadapterd'),
497 'dst': 'usr/libexec/bluetooth/btadapterd',
498 'strip': True
499 },
500 {
501 'src': os.path.join(bindir, 'btmanagerd'),
502 'dst': 'usr/libexec/bluetooth/btmanagerd',
503 'strip': True
504 },
505 {
506 'src': os.path.join(bindir, 'btclient'),
507 'dst': 'usr/local/bin/btclient',
508 'strip': True
509 },
510 ]
511
512 for v in install_map:
513 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
514 dst = os.path.join(prefix, partial_dst)
515
516 # Create dst directory first and copy file there
517 os.makedirs(os.path.dirname(dst), exist_ok=True)
518 print('Installing {}'.format(dst))
519 shutil.copy(src, dst)
520
521 # Binary should be marked for strip and no-strip option shouldn't be
522 # set. No-strip is useful while debugging.
523 if strip and not self.args.no_strip:
524 self.run_command('install', ['llvm-strip', dst])
525
526 # Put all files into a tar.gz for easier installation
527 tar_location = os.path.join(prefix, 'floss.tar.gz')
528 with tarfile.open(tar_location, 'w:gz') as tar:
529 for v in install_map:
530 tar.add(v['dst'])
531
532 print('Tarball created at {}'.format(tar_location))
533
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000534 def _target_clean(self):
535 """ Delete the output directory entirely.
536 """
537 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800538
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700539 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedi16617282022-09-28 10:11:10 -0700540 cargo_lock_files = [
541 os.path.join(self.platform_dir, 'bt', 'Cargo.lock'),
542 os.path.join(self.platform_dir, 'bt', 'tools', 'rootcanal', 'Cargo.lock'),
543 ]
544 for lock_file in cargo_lock_files:
545 try:
546 os.remove(lock_file)
547 print('Removed {}'.format(lock_file))
548 except FileNotFoundError:
549 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000550
551 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800552 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000553 """
554 self._target_prepare()
555 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000556 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700557 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000558
559 def build(self):
560 """ Builds according to self.target
561 """
562 print('Building target ', self.target)
563
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800564 # Validate that the target is valid
565 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800566 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800567 return
568
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000569 if self.target == 'prepare':
570 self._target_prepare()
571 elif self.target == 'tools':
572 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100573 elif self.target == 'rootcanal':
574 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000575 elif self.target == 'rust':
576 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800577 elif self.target == 'docs':
578 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000579 elif self.target == 'main':
580 self._target_main()
581 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000582 self._target_test()
583 elif self.target == 'clean':
584 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800585 elif self.target == 'install':
586 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000587 elif self.target == 'all':
588 self._target_all()
589
590
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700591class Bootstrap():
592
593 def __init__(self, base_dir, bt_dir):
594 """ Construct bootstrapper.
595
596 Args:
597 base_dir: Where to stage everything.
598 bt_dir: Where bluetooth source is kept (will be symlinked)
599 """
600 self.base_dir = os.path.abspath(base_dir)
601 self.bt_dir = os.path.abspath(bt_dir)
602
603 # Create base directory if it doesn't already exist
604 os.makedirs(self.base_dir, exist_ok=True)
605
606 if not os.path.isdir(self.bt_dir):
607 raise Exception('{} is not a valid directory'.format(self.bt_dir))
608
609 self.git_dir = os.path.join(self.base_dir, 'repos')
610 self.staging_dir = os.path.join(self.base_dir, 'staging')
611 self.output_dir = os.path.join(self.base_dir, 'output')
612 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
613
614 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
615
616 def _update_platform2(self):
617 """Updates repositories used for build."""
618 for repo in BOOTSTRAP_GIT_REPOS.keys():
619 cwd = os.path.join(self.git_dir, repo)
620 subprocess.check_call(['git', 'pull'], cwd=cwd)
621
622 def _setup_platform2(self):
623 """ Set up platform2.
624
625 This will check out all the git repos and symlink everything correctly.
626 """
627
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800628 # Create all directories we will need to use
629 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
630 os.makedirs(dirpath, exist_ok=True)
631
632 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700633 if os.path.isfile(self.dir_setup_complete):
634 print('{} already set-up. Updating instead.'.format(self.base_dir))
635 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800636 else:
637 # Check out all repos in git directory
638 for repo in BOOTSTRAP_GIT_REPOS.values():
639 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700640
641 # Symlink things
642 symlinks = [
643 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700644 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700645 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
646 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
647 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
648 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
649 ]
650
651 # Create symlinks
652 for pairs in symlinks:
653 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000654 try:
655 os.unlink(dst)
656 except Exception as e:
657 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700658 os.symlink(src, dst)
659
660 # Write to setup complete file so we don't repeat this step
661 with open(self.dir_setup_complete, 'w') as f:
662 f.write('Setup complete.')
663
664 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
665 """ Pretty print an install command.
666
667 Args:
668 install_cmd: Prefixed install command.
669 packages: Enumerate packages and append them to install command.
670 line_limit: Number of characters per line.
671
672 Return:
673 Array of lines to join and print.
674 """
675 install = [install_cmd]
676 line = ' '
677 # Remainder needed = space + len(pkg) + space + \
678 # Assuming 80 character lines, that's 80 - 3 = 77
679 line_limit = line_limit - 3
680 for pkg in packages:
681 if len(line) + len(pkg) < line_limit:
682 line = '{}{} '.format(line, pkg)
683 else:
684 install.append(line)
685 line = ' {} '.format(pkg)
686
687 if len(line) > 0:
688 install.append(line)
689
690 return install
691
692 def _check_package_installed(self, package, cmd, predicate):
693 """Check that the given package is installed.
694
695 Args:
696 package: Check that this package is installed.
697 cmd: Command prefix to check if installed (package appended to end)
698 predicate: Function/lambda to check if package is installed based
699 on output. Takes string output and returns boolean.
700
701 Return:
702 True if package is installed.
703 """
704 try:
705 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
706 is_installed = predicate(output.decode('utf-8'))
707 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
708
709 return is_installed
710 except Exception as e:
711 print(e)
712 return False
713
714 def _get_command_output(self, cmd):
715 """Runs the command and gets the output.
716
717 Args:
718 cmd: Command to run.
719
720 Return:
721 Tuple (Success, Output). Success represents if the command ran ok.
722 """
723 try:
724 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
725 return (True, output.decode('utf-8').split('\n'))
726 except Exception as e:
727 print(e)
728 return (False, "")
729
730 def _print_missing_packages(self):
731 """Print any missing packages found via apt.
732
733 This will find any missing packages necessary for build using apt and
734 print it out as an apt-get install printf.
735 """
736 print('Checking for any missing packages...')
737
738 (success, output) = self._get_command_output(APT_PKG_LIST)
739 if not success:
740 raise Exception("Could not query apt for packages.")
741
742 packages_installed = {}
743 for line in output:
744 if 'installed' in line:
745 split = line.split('/', 2)
746 packages_installed[split[0]] = True
747
748 need_packages = []
749 for pkg in REQUIRED_APT_PACKAGES:
750 if pkg not in packages_installed:
751 need_packages.append(pkg)
752
753 # No packages need to be installed
754 if len(need_packages) == 0:
755 print('+ All required packages are installed')
756 return
757
758 install = self._pretty_print_install('sudo apt-get install', need_packages)
759
760 # Print all lines so they can be run in cmdline
761 print('Missing system packages. Run the following command: ')
762 print(' \\\n'.join(install))
763
764 def _print_missing_rust_packages(self):
765 """Print any missing packages found via cargo.
766
767 This will find any missing packages necessary for build using cargo and
768 print it out as a cargo-install printf.
769 """
770 print('Checking for any missing cargo packages...')
771
772 (success, output) = self._get_command_output(CARGO_PKG_LIST)
773 if not success:
774 raise Exception("Could not query cargo for packages.")
775
776 packages_installed = {}
777 for line in output:
778 # Cargo installed packages have this format
779 # <crate name> <version>:
780 # <binary name>
781 # We only care about the crates themselves
782 if ':' not in line:
783 continue
784
785 split = line.split(' ', 2)
786 packages_installed[split[0]] = True
787
788 need_packages = []
789 for pkg in REQUIRED_CARGO_PACKAGES:
790 if pkg not in packages_installed:
791 need_packages.append(pkg)
792
793 # No packages to be installed
794 if len(need_packages) == 0:
795 print('+ All required cargo packages are installed')
796 return
797
798 install = self._pretty_print_install('cargo install', need_packages)
799 print('Missing cargo packages. Run the following command: ')
800 print(' \\\n'.join(install))
801
802 def bootstrap(self):
803 """ Bootstrap the Linux build."""
804 self._setup_platform2()
805 self._print_missing_packages()
806 self._print_missing_rust_packages()
807
808
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000809if __name__ == '__main__':
810 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700811 parser.add_argument(
812 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
813 parser.add_argument(
814 '--run-bootstrap',
815 help='Run bootstrap code to verify build env is ok to build.',
816 default=False,
817 action='store_true')
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700818 parser.add_argument(
819 '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000820 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800821 parser.add_argument(
822 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000823 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800824 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
825 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000826 parser.add_argument('--target', help='Run specific build target')
827 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700828 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000829 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700830 parser.add_argument(
831 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000832 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000833 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700834
835 # Make sure we get absolute path + expanded path for bootstrap directory
836 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
837
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000838 # Possible values for machine() come from 'uname -m'
839 # Since this script only runs on Linux, x86_64 machines must have this value
840 if platform.machine() != 'x86_64':
841 raise Exception("Only x86_64 machines are currently supported by this build script.")
842
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700843 if args.run_bootstrap:
844 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
845 bootstrap.bootstrap()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700846 elif args.print_env:
847 build = HostBuild(args)
848 build.print_env()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700849 else:
850 build = HostBuild(args)
851 build.build()