blob: f423de55be72b108e6b89c9f7a8653518b3d8dc9 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
111 'libflatbuffers1',
112 'libgl1-mesa-dev',
113 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000114 'libgtest-dev',
115 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700116 'liblz4-tool',
117 'libncurses5',
118 'libnss3-dev',
119 'libprotobuf-dev',
120 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000121 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700122 'libssl-dev',
123 'libtinyxml2-dev',
124 'libx11-dev',
125 'libxml2-utils',
126 'ninja-build',
127 'openssl',
128 'protobuf-compiler',
129 'unzip',
130 'x11proto-core-dev',
131 'xsltproc',
132 'zip',
133 'zlib1g-dev',
134]
135
136# List of cargo packages required for linux build
137REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
138
139APT_PKG_LIST = ['apt', '-qq', 'list']
140CARGO_PKG_LIST = ['cargo', 'install', '--list']
141
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000142
143class UseFlags():
144
145 def __init__(self, use_flags):
146 """ Construct the use flags.
147
148 Args:
149 use_flags: List of use flags parsed from the command.
150 """
151 self.flags = {}
152
153 # Import use flags required by common-mk
154 for use in COMMON_MK_USES:
155 self.set_flag(use, False)
156
157 # Set our defaults
158 for use, value in USE_DEFAULTS.items():
159 self.set_flag(use, value)
160
161 # Set use flags - value is set to True unless the use starts with -
162 # All given use flags always override the defaults
163 for use in use_flags:
164 value = not use.startswith('-')
165 self.set_flag(use, value)
166
167 def set_flag(self, key, value=True):
168 setattr(self, key, value)
169 self.flags[key] = value
170
171
172class HostBuild():
173
174 def __init__(self, args):
175 """ Construct the builder.
176
177 Args:
178 args: Parsed arguments from ArgumentParser
179 """
180 self.args = args
181
182 # Set jobs to number of cpus unless explicitly set
183 self.jobs = self.args.jobs
184 if not self.jobs:
185 self.jobs = multiprocessing.cpu_count()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700186 sys.stderr.write("Number of jobs = {}\n".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000187
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700188 # Normalize bootstrap dir and make sure it exists
189 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
190 os.makedirs(self.bootstrap_dir, exist_ok=True)
191
192 # Output and platform directories are based on bootstrap
193 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
194 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
199 # If default target isn't set, build everything
200 self.target = 'all'
201 if hasattr(self.args, 'target') and self.args.target:
202 self.target = self.args.target
203
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000204 target_use = self.args.use if self.args.use else []
205
206 # Unless set, always build test code
207 if not self.args.notest:
208 target_use.append('test')
209
210 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000211
212 # Validate platform directory
213 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
214 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
215
216 # Make sure output directory exists (or create it)
217 os.makedirs(self.output_dir, exist_ok=True)
218
219 # Set some default attributes
220 self.libbase_ver = None
221
222 self.configure_environ()
223
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000224 def _generate_rustflags(self):
225 """ Rustflags to include for the build.
226 """
227 rust_flags = [
228 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700229 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000230 '-C',
231 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700232 # exclude uninteresting warnings
233 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000234 ]
235
236 return ' '.join(rust_flags)
237
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000238 def configure_environ(self):
239 """ Configure environment variables for GN and Cargo.
240 """
241 self.env = os.environ.copy()
242
243 # Make sure cargo home dir exists and has a bin directory
244 cargo_home = os.path.join(self.output_dir, 'cargo_home')
245 os.makedirs(cargo_home, exist_ok=True)
246 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
247
248 # Configure Rust env variables
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700249 self.custom_env = {}
250 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir
251 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
252 self.custom_env['RUSTFLAGS'] = self._generate_rustflags()
253 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
254 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
255 self.custom_env['CXX_OUTDIR'] = self._gn_default_output()
256 self.env.update(self.custom_env)
257
258 def print_env(self):
259 """ Print the custom environment variables that are used in build.
260
261 Useful so that external tools can mimic the environment to be the same
262 as build.py, e.g. rust-analyzer.
263 """
264 for k, v in self.custom_env.items():
265 print("export {}='{}'".format(k, v))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000266
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000267 def run_command(self, target, args, cwd=None, env=None):
268 """ Run command and stream the output.
269 """
270 # Set some defaults
271 if not cwd:
272 cwd = self.platform_dir
273 if not env:
274 env = self.env
275
276 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
277 with open(log_file, 'wb') as lf:
278 rc = 0
279 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
280 while True:
281 line = process.stdout.readline()
282 print(line.decode('utf-8'), end="")
283 lf.write(line)
284 if not line:
285 rc = process.poll()
286 if rc is not None:
287 break
288
289 time.sleep(0.1)
290
291 if rc != 0:
292 raise Exception("Return code is {}".format(rc))
293
294 def _get_basever(self):
295 if self.libbase_ver:
296 return self.libbase_ver
297
298 self.libbase_ver = os.environ.get('BASE_VER', '')
299 if not self.libbase_ver:
300 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
301 try:
302 with open(base_file, 'r') as f:
303 self.libbase_ver = f.read().strip('\n')
304 except:
305 self.libbase_ver = 'NOT-INSTALLED'
306
307 return self.libbase_ver
308
309 def _gn_default_output(self):
310 return os.path.join(self.output_dir, 'out/Default')
311
312 def _gn_configure(self):
313 """ Configure all required parameters for platform2.
314
315 Mostly copied from //common-mk/platform2.py
316 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700317 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000318
319 def to_gn_string(s):
320 return '"%s"' % s.replace('"', '\\"')
321
322 def to_gn_list(strs):
323 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
324
325 def to_gn_args_args(gn_args):
326 for k, v in gn_args.items():
327 if isinstance(v, bool):
328 v = str(v).lower()
329 elif isinstance(v, list):
330 v = to_gn_list(v)
331 elif isinstance(v, six.string_types):
332 v = to_gn_string(v)
333 else:
334 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
335 yield '%s=%s' % (k.replace('-', '_'), v)
336
337 gn_args = {
338 'platform_subdir': 'bt',
339 'cc': 'clang' if clang else 'gcc',
340 'cxx': 'clang++' if clang else 'g++',
341 'ar': 'llvm-ar' if clang else 'ar',
342 'pkg-config': 'pkg-config',
343 'clang_cc': clang,
344 'clang_cxx': clang,
345 'OS': 'linux',
346 'sysroot': self.sysroot,
347 'libdir': os.path.join(self.sysroot, self.libdir),
348 'build_root': self.output_dir,
349 'platform2_root': self.platform_dir,
350 'libbase_ver': self._get_basever(),
351 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
352 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800353 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000354 'enable_werror': False,
355 }
356
357 if clang:
358 # Make sure to mark the clang use flag as true
359 self.use.set_flag('clang', True)
360 gn_args['external_cxxflags'] += ['-I/usr/include/']
361
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000362 gn_args_args = list(to_gn_args_args(gn_args))
363 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
364 gn_args_args += ['use={%s}' % (' '.join(use_args))]
365
366 gn_args = [
367 'gn',
368 'gen',
369 ]
370
371 if self.args.verbose:
372 gn_args.append('-v')
373
374 gn_args += [
375 '--root=%s' % self.platform_dir,
376 '--args=%s' % ' '.join(gn_args_args),
377 self._gn_default_output(),
378 ]
379
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700380 if 'PKG_CONFIG_PATH' in self.env:
381 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000382
383 self.run_command('configure', gn_args)
384
385 def _gn_build(self, target):
386 """ Generate the ninja command for the target and run it.
387 """
388 args = ['%s:%s' % ('bt', target)]
389 ninja_args = ['ninja', '-C', self._gn_default_output()]
390 if self.jobs:
391 ninja_args += ['-j', str(self.jobs)]
392 ninja_args += args
393
394 if self.args.verbose:
395 ninja_args.append('-v')
396
397 self.run_command('build', ninja_args)
398
399 def _rust_configure(self):
400 """ Generate config file at cargo_home so we use vendored crates.
401 """
402 template = """
403 [source.systembt]
404 directory = "{}/external/rust/vendor"
405
406 [source.crates-io]
407 replace-with = "systembt"
408 local-registry = "/nonexistent"
409 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700410
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700411 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700412 contents = template.format(self.platform_dir)
413 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
414 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000415
416 def _rust_build(self):
417 """ Run `cargo build` from platform2/bt directory.
418 """
419 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
420
421 def _target_prepare(self):
422 """ Target to prepare the output directory for building.
423
424 This runs gn gen to generate all rquired files and set up the Rust
425 config properly. This will be run
426 """
427 self._gn_configure()
428 self._rust_configure()
429
430 def _target_tools(self):
431 """ Build the tools target in an already prepared environment.
432 """
433 self._gn_build('tools')
434
435 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
436 shutil.copy(
437 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
438
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800439 def _target_docs(self):
440 """Build the Rust docs."""
441 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
442
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000443 def _target_rust(self):
444 """ Build rust artifacts in an already prepared environment.
445 """
446 self._rust_build()
447
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100448 def _target_rootcanal(self):
449 """ Build rust artifacts for RootCanal in an already prepared environment.
450 """
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700451 self.run_command(
452 'rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100453
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000454 def _target_main(self):
455 """ Build the main GN artifacts in an already prepared environment.
456 """
457 self._gn_build('all')
458
459 def _target_test(self):
460 """ Runs the host tests.
461 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000462 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800463 rust_test_cmd = ['cargo', 'test']
464 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700465 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800466
467 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100468 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000469
470 # Host tests second based on host test list
471 for t in HOST_TESTS:
472 self.run_command(
473 'test', [os.path.join(self.output_dir, 'out/Default', t)],
474 cwd=os.path.join(self.output_dir),
475 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000476
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800477 def _target_install(self):
478 """ Installs files required to run Floss to install directory.
479 """
480 # First make the install directory
481 prefix = self.install_dir
482 os.makedirs(prefix, exist_ok=True)
483
484 # Next save the cwd and change to install directory
485 last_cwd = os.getcwd()
486 os.chdir(prefix)
487
488 bindir = os.path.join(self.output_dir, 'debug')
489 srcdir = os.path.dirname(__file__)
490
491 install_map = [
492 {
493 'src': os.path.join(bindir, 'btadapterd'),
494 'dst': 'usr/libexec/bluetooth/btadapterd',
495 'strip': True
496 },
497 {
498 'src': os.path.join(bindir, 'btmanagerd'),
499 'dst': 'usr/libexec/bluetooth/btmanagerd',
500 'strip': True
501 },
502 {
503 'src': os.path.join(bindir, 'btclient'),
504 'dst': 'usr/local/bin/btclient',
505 'strip': True
506 },
507 ]
508
509 for v in install_map:
510 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
511 dst = os.path.join(prefix, partial_dst)
512
513 # Create dst directory first and copy file there
514 os.makedirs(os.path.dirname(dst), exist_ok=True)
515 print('Installing {}'.format(dst))
516 shutil.copy(src, dst)
517
518 # Binary should be marked for strip and no-strip option shouldn't be
519 # set. No-strip is useful while debugging.
520 if strip and not self.args.no_strip:
521 self.run_command('install', ['llvm-strip', dst])
522
523 # Put all files into a tar.gz for easier installation
524 tar_location = os.path.join(prefix, 'floss.tar.gz')
525 with tarfile.open(tar_location, 'w:gz') as tar:
526 for v in install_map:
527 tar.add(v['dst'])
528
529 print('Tarball created at {}'.format(tar_location))
530
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000531 def _target_clean(self):
532 """ Delete the output directory entirely.
533 """
534 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800535
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700536 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800537 try:
538 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
539 except FileNotFoundError:
540 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000541
542 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800543 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000544 """
545 self._target_prepare()
546 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000547 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700548 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000549
550 def build(self):
551 """ Builds according to self.target
552 """
553 print('Building target ', self.target)
554
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800555 # Validate that the target is valid
556 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800557 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800558 return
559
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000560 if self.target == 'prepare':
561 self._target_prepare()
562 elif self.target == 'tools':
563 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100564 elif self.target == 'rootcanal':
565 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000566 elif self.target == 'rust':
567 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800568 elif self.target == 'docs':
569 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000570 elif self.target == 'main':
571 self._target_main()
572 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000573 self._target_test()
574 elif self.target == 'clean':
575 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800576 elif self.target == 'install':
577 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000578 elif self.target == 'all':
579 self._target_all()
580
581
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700582class Bootstrap():
583
584 def __init__(self, base_dir, bt_dir):
585 """ Construct bootstrapper.
586
587 Args:
588 base_dir: Where to stage everything.
589 bt_dir: Where bluetooth source is kept (will be symlinked)
590 """
591 self.base_dir = os.path.abspath(base_dir)
592 self.bt_dir = os.path.abspath(bt_dir)
593
594 # Create base directory if it doesn't already exist
595 os.makedirs(self.base_dir, exist_ok=True)
596
597 if not os.path.isdir(self.bt_dir):
598 raise Exception('{} is not a valid directory'.format(self.bt_dir))
599
600 self.git_dir = os.path.join(self.base_dir, 'repos')
601 self.staging_dir = os.path.join(self.base_dir, 'staging')
602 self.output_dir = os.path.join(self.base_dir, 'output')
603 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
604
605 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
606
607 def _update_platform2(self):
608 """Updates repositories used for build."""
609 for repo in BOOTSTRAP_GIT_REPOS.keys():
610 cwd = os.path.join(self.git_dir, repo)
611 subprocess.check_call(['git', 'pull'], cwd=cwd)
612
613 def _setup_platform2(self):
614 """ Set up platform2.
615
616 This will check out all the git repos and symlink everything correctly.
617 """
618
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800619 # Create all directories we will need to use
620 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
621 os.makedirs(dirpath, exist_ok=True)
622
623 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700624 if os.path.isfile(self.dir_setup_complete):
625 print('{} already set-up. Updating instead.'.format(self.base_dir))
626 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800627 else:
628 # Check out all repos in git directory
629 for repo in BOOTSTRAP_GIT_REPOS.values():
630 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700631
632 # Symlink things
633 symlinks = [
634 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700635 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700636 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
637 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
638 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
639 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
640 ]
641
642 # Create symlinks
643 for pairs in symlinks:
644 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000645 try:
646 os.unlink(dst)
647 except Exception as e:
648 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700649 os.symlink(src, dst)
650
651 # Write to setup complete file so we don't repeat this step
652 with open(self.dir_setup_complete, 'w') as f:
653 f.write('Setup complete.')
654
655 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
656 """ Pretty print an install command.
657
658 Args:
659 install_cmd: Prefixed install command.
660 packages: Enumerate packages and append them to install command.
661 line_limit: Number of characters per line.
662
663 Return:
664 Array of lines to join and print.
665 """
666 install = [install_cmd]
667 line = ' '
668 # Remainder needed = space + len(pkg) + space + \
669 # Assuming 80 character lines, that's 80 - 3 = 77
670 line_limit = line_limit - 3
671 for pkg in packages:
672 if len(line) + len(pkg) < line_limit:
673 line = '{}{} '.format(line, pkg)
674 else:
675 install.append(line)
676 line = ' {} '.format(pkg)
677
678 if len(line) > 0:
679 install.append(line)
680
681 return install
682
683 def _check_package_installed(self, package, cmd, predicate):
684 """Check that the given package is installed.
685
686 Args:
687 package: Check that this package is installed.
688 cmd: Command prefix to check if installed (package appended to end)
689 predicate: Function/lambda to check if package is installed based
690 on output. Takes string output and returns boolean.
691
692 Return:
693 True if package is installed.
694 """
695 try:
696 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
697 is_installed = predicate(output.decode('utf-8'))
698 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
699
700 return is_installed
701 except Exception as e:
702 print(e)
703 return False
704
705 def _get_command_output(self, cmd):
706 """Runs the command and gets the output.
707
708 Args:
709 cmd: Command to run.
710
711 Return:
712 Tuple (Success, Output). Success represents if the command ran ok.
713 """
714 try:
715 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
716 return (True, output.decode('utf-8').split('\n'))
717 except Exception as e:
718 print(e)
719 return (False, "")
720
721 def _print_missing_packages(self):
722 """Print any missing packages found via apt.
723
724 This will find any missing packages necessary for build using apt and
725 print it out as an apt-get install printf.
726 """
727 print('Checking for any missing packages...')
728
729 (success, output) = self._get_command_output(APT_PKG_LIST)
730 if not success:
731 raise Exception("Could not query apt for packages.")
732
733 packages_installed = {}
734 for line in output:
735 if 'installed' in line:
736 split = line.split('/', 2)
737 packages_installed[split[0]] = True
738
739 need_packages = []
740 for pkg in REQUIRED_APT_PACKAGES:
741 if pkg not in packages_installed:
742 need_packages.append(pkg)
743
744 # No packages need to be installed
745 if len(need_packages) == 0:
746 print('+ All required packages are installed')
747 return
748
749 install = self._pretty_print_install('sudo apt-get install', need_packages)
750
751 # Print all lines so they can be run in cmdline
752 print('Missing system packages. Run the following command: ')
753 print(' \\\n'.join(install))
754
755 def _print_missing_rust_packages(self):
756 """Print any missing packages found via cargo.
757
758 This will find any missing packages necessary for build using cargo and
759 print it out as a cargo-install printf.
760 """
761 print('Checking for any missing cargo packages...')
762
763 (success, output) = self._get_command_output(CARGO_PKG_LIST)
764 if not success:
765 raise Exception("Could not query cargo for packages.")
766
767 packages_installed = {}
768 for line in output:
769 # Cargo installed packages have this format
770 # <crate name> <version>:
771 # <binary name>
772 # We only care about the crates themselves
773 if ':' not in line:
774 continue
775
776 split = line.split(' ', 2)
777 packages_installed[split[0]] = True
778
779 need_packages = []
780 for pkg in REQUIRED_CARGO_PACKAGES:
781 if pkg not in packages_installed:
782 need_packages.append(pkg)
783
784 # No packages to be installed
785 if len(need_packages) == 0:
786 print('+ All required cargo packages are installed')
787 return
788
789 install = self._pretty_print_install('cargo install', need_packages)
790 print('Missing cargo packages. Run the following command: ')
791 print(' \\\n'.join(install))
792
793 def bootstrap(self):
794 """ Bootstrap the Linux build."""
795 self._setup_platform2()
796 self._print_missing_packages()
797 self._print_missing_rust_packages()
798
799
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000800if __name__ == '__main__':
801 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700802 parser.add_argument(
803 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
804 parser.add_argument(
805 '--run-bootstrap',
806 help='Run bootstrap code to verify build env is ok to build.',
807 default=False,
808 action='store_true')
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700809 parser.add_argument(
810 '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000811 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800812 parser.add_argument(
813 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000814 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800815 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
816 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000817 parser.add_argument('--target', help='Run specific build target')
818 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700819 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000820 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700821 parser.add_argument(
822 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000823 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000824 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700825
826 # Make sure we get absolute path + expanded path for bootstrap directory
827 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
828
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000829 # Possible values for machine() come from 'uname -m'
830 # Since this script only runs on Linux, x86_64 machines must have this value
831 if platform.machine() != 'x86_64':
832 raise Exception("Only x86_64 machines are currently supported by this build script.")
833
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700834 if args.run_bootstrap:
835 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
836 bootstrap.bootstrap()
Sonny Sasaka5b7e6f22022-09-06 11:31:53 -0700837 elif args.print_env:
838 build = HostBuild(args)
839 build.print_env()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700840 else:
841 build = HostBuild(args)
842 build.build()