blob: 65fcc0a5fa36cb1a8f28706c5fd29c437b1b92d7 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
111 'libflatbuffers1',
112 'libgl1-mesa-dev',
113 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000114 'libgtest-dev',
115 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700116 'liblz4-tool',
117 'libncurses5',
118 'libnss3-dev',
119 'libprotobuf-dev',
120 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000121 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700122 'libssl-dev',
123 'libtinyxml2-dev',
124 'libx11-dev',
125 'libxml2-utils',
126 'ninja-build',
127 'openssl',
128 'protobuf-compiler',
129 'unzip',
130 'x11proto-core-dev',
131 'xsltproc',
132 'zip',
133 'zlib1g-dev',
134]
135
136# List of cargo packages required for linux build
137REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
138
139APT_PKG_LIST = ['apt', '-qq', 'list']
140CARGO_PKG_LIST = ['cargo', 'install', '--list']
141
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000142
143class UseFlags():
144
145 def __init__(self, use_flags):
146 """ Construct the use flags.
147
148 Args:
149 use_flags: List of use flags parsed from the command.
150 """
151 self.flags = {}
152
153 # Import use flags required by common-mk
154 for use in COMMON_MK_USES:
155 self.set_flag(use, False)
156
157 # Set our defaults
158 for use, value in USE_DEFAULTS.items():
159 self.set_flag(use, value)
160
161 # Set use flags - value is set to True unless the use starts with -
162 # All given use flags always override the defaults
163 for use in use_flags:
164 value = not use.startswith('-')
165 self.set_flag(use, value)
166
167 def set_flag(self, key, value=True):
168 setattr(self, key, value)
169 self.flags[key] = value
170
171
172class HostBuild():
173
174 def __init__(self, args):
175 """ Construct the builder.
176
177 Args:
178 args: Parsed arguments from ArgumentParser
179 """
180 self.args = args
181
182 # Set jobs to number of cpus unless explicitly set
183 self.jobs = self.args.jobs
184 if not self.jobs:
185 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000186 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000187
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700188 # Normalize bootstrap dir and make sure it exists
189 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
190 os.makedirs(self.bootstrap_dir, exist_ok=True)
191
192 # Output and platform directories are based on bootstrap
193 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
194 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
199 # If default target isn't set, build everything
200 self.target = 'all'
201 if hasattr(self.args, 'target') and self.args.target:
202 self.target = self.args.target
203
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000204 target_use = self.args.use if self.args.use else []
205
206 # Unless set, always build test code
207 if not self.args.notest:
208 target_use.append('test')
209
210 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000211
212 # Validate platform directory
213 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
214 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
215
216 # Make sure output directory exists (or create it)
217 os.makedirs(self.output_dir, exist_ok=True)
218
219 # Set some default attributes
220 self.libbase_ver = None
221
222 self.configure_environ()
223
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000224 def _generate_rustflags(self):
225 """ Rustflags to include for the build.
226 """
227 rust_flags = [
228 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700229 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000230 '-C',
231 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700232 # exclude uninteresting warnings
233 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000234 ]
235
236 return ' '.join(rust_flags)
237
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000238 def configure_environ(self):
239 """ Configure environment variables for GN and Cargo.
240 """
241 self.env = os.environ.copy()
242
243 # Make sure cargo home dir exists and has a bin directory
244 cargo_home = os.path.join(self.output_dir, 'cargo_home')
245 os.makedirs(cargo_home, exist_ok=True)
246 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
247
248 # Configure Rust env variables
249 self.env['CARGO_TARGET_DIR'] = self.output_dir
250 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000251 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700252 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700253 self.env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
Sonny Sasakac1bc8e12022-06-14 17:52:50 -0700254 self.env['CXX_OUTDIR'] = self._gn_default_output()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000255
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000256 def run_command(self, target, args, cwd=None, env=None):
257 """ Run command and stream the output.
258 """
259 # Set some defaults
260 if not cwd:
261 cwd = self.platform_dir
262 if not env:
263 env = self.env
264
265 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
266 with open(log_file, 'wb') as lf:
267 rc = 0
268 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
269 while True:
270 line = process.stdout.readline()
271 print(line.decode('utf-8'), end="")
272 lf.write(line)
273 if not line:
274 rc = process.poll()
275 if rc is not None:
276 break
277
278 time.sleep(0.1)
279
280 if rc != 0:
281 raise Exception("Return code is {}".format(rc))
282
283 def _get_basever(self):
284 if self.libbase_ver:
285 return self.libbase_ver
286
287 self.libbase_ver = os.environ.get('BASE_VER', '')
288 if not self.libbase_ver:
289 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
290 try:
291 with open(base_file, 'r') as f:
292 self.libbase_ver = f.read().strip('\n')
293 except:
294 self.libbase_ver = 'NOT-INSTALLED'
295
296 return self.libbase_ver
297
298 def _gn_default_output(self):
299 return os.path.join(self.output_dir, 'out/Default')
300
301 def _gn_configure(self):
302 """ Configure all required parameters for platform2.
303
304 Mostly copied from //common-mk/platform2.py
305 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700306 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000307
308 def to_gn_string(s):
309 return '"%s"' % s.replace('"', '\\"')
310
311 def to_gn_list(strs):
312 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
313
314 def to_gn_args_args(gn_args):
315 for k, v in gn_args.items():
316 if isinstance(v, bool):
317 v = str(v).lower()
318 elif isinstance(v, list):
319 v = to_gn_list(v)
320 elif isinstance(v, six.string_types):
321 v = to_gn_string(v)
322 else:
323 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
324 yield '%s=%s' % (k.replace('-', '_'), v)
325
326 gn_args = {
327 'platform_subdir': 'bt',
328 'cc': 'clang' if clang else 'gcc',
329 'cxx': 'clang++' if clang else 'g++',
330 'ar': 'llvm-ar' if clang else 'ar',
331 'pkg-config': 'pkg-config',
332 'clang_cc': clang,
333 'clang_cxx': clang,
334 'OS': 'linux',
335 'sysroot': self.sysroot,
336 'libdir': os.path.join(self.sysroot, self.libdir),
337 'build_root': self.output_dir,
338 'platform2_root': self.platform_dir,
339 'libbase_ver': self._get_basever(),
340 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
341 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800342 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000343 'enable_werror': False,
344 }
345
346 if clang:
347 # Make sure to mark the clang use flag as true
348 self.use.set_flag('clang', True)
349 gn_args['external_cxxflags'] += ['-I/usr/include/']
350
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000351 gn_args_args = list(to_gn_args_args(gn_args))
352 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
353 gn_args_args += ['use={%s}' % (' '.join(use_args))]
354
355 gn_args = [
356 'gn',
357 'gen',
358 ]
359
360 if self.args.verbose:
361 gn_args.append('-v')
362
363 gn_args += [
364 '--root=%s' % self.platform_dir,
365 '--args=%s' % ' '.join(gn_args_args),
366 self._gn_default_output(),
367 ]
368
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700369 if 'PKG_CONFIG_PATH' in self.env:
370 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000371
372 self.run_command('configure', gn_args)
373
374 def _gn_build(self, target):
375 """ Generate the ninja command for the target and run it.
376 """
377 args = ['%s:%s' % ('bt', target)]
378 ninja_args = ['ninja', '-C', self._gn_default_output()]
379 if self.jobs:
380 ninja_args += ['-j', str(self.jobs)]
381 ninja_args += args
382
383 if self.args.verbose:
384 ninja_args.append('-v')
385
386 self.run_command('build', ninja_args)
387
388 def _rust_configure(self):
389 """ Generate config file at cargo_home so we use vendored crates.
390 """
391 template = """
392 [source.systembt]
393 directory = "{}/external/rust/vendor"
394
395 [source.crates-io]
396 replace-with = "systembt"
397 local-registry = "/nonexistent"
398 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700399
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700400 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700401 contents = template.format(self.platform_dir)
402 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
403 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000404
405 def _rust_build(self):
406 """ Run `cargo build` from platform2/bt directory.
407 """
408 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
409
410 def _target_prepare(self):
411 """ Target to prepare the output directory for building.
412
413 This runs gn gen to generate all rquired files and set up the Rust
414 config properly. This will be run
415 """
416 self._gn_configure()
417 self._rust_configure()
418
419 def _target_tools(self):
420 """ Build the tools target in an already prepared environment.
421 """
422 self._gn_build('tools')
423
424 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
425 shutil.copy(
426 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
427
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800428 def _target_docs(self):
429 """Build the Rust docs."""
430 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
431
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000432 def _target_rust(self):
433 """ Build rust artifacts in an already prepared environment.
434 """
435 self._rust_build()
436
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100437 def _target_rootcanal(self):
438 """ Build rust artifacts for RootCanal in an already prepared environment.
439 """
440 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
441
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000442 def _target_main(self):
443 """ Build the main GN artifacts in an already prepared environment.
444 """
445 self._gn_build('all')
446
447 def _target_test(self):
448 """ Runs the host tests.
449 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000450 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800451 rust_test_cmd = ['cargo', 'test']
452 if self.args.test_name:
453 rust_test_cmd = rust_test_cmd + [self.args.test_name]
454
455 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100456 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000457
458 # Host tests second based on host test list
459 for t in HOST_TESTS:
460 self.run_command(
461 'test', [os.path.join(self.output_dir, 'out/Default', t)],
462 cwd=os.path.join(self.output_dir),
463 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000464
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800465 def _target_install(self):
466 """ Installs files required to run Floss to install directory.
467 """
468 # First make the install directory
469 prefix = self.install_dir
470 os.makedirs(prefix, exist_ok=True)
471
472 # Next save the cwd and change to install directory
473 last_cwd = os.getcwd()
474 os.chdir(prefix)
475
476 bindir = os.path.join(self.output_dir, 'debug')
477 srcdir = os.path.dirname(__file__)
478
479 install_map = [
480 {
481 'src': os.path.join(bindir, 'btadapterd'),
482 'dst': 'usr/libexec/bluetooth/btadapterd',
483 'strip': True
484 },
485 {
486 'src': os.path.join(bindir, 'btmanagerd'),
487 'dst': 'usr/libexec/bluetooth/btmanagerd',
488 'strip': True
489 },
490 {
491 'src': os.path.join(bindir, 'btclient'),
492 'dst': 'usr/local/bin/btclient',
493 'strip': True
494 },
495 ]
496
497 for v in install_map:
498 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
499 dst = os.path.join(prefix, partial_dst)
500
501 # Create dst directory first and copy file there
502 os.makedirs(os.path.dirname(dst), exist_ok=True)
503 print('Installing {}'.format(dst))
504 shutil.copy(src, dst)
505
506 # Binary should be marked for strip and no-strip option shouldn't be
507 # set. No-strip is useful while debugging.
508 if strip and not self.args.no_strip:
509 self.run_command('install', ['llvm-strip', dst])
510
511 # Put all files into a tar.gz for easier installation
512 tar_location = os.path.join(prefix, 'floss.tar.gz')
513 with tarfile.open(tar_location, 'w:gz') as tar:
514 for v in install_map:
515 tar.add(v['dst'])
516
517 print('Tarball created at {}'.format(tar_location))
518
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000519 def _target_clean(self):
520 """ Delete the output directory entirely.
521 """
522 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800523
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700524 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800525 try:
526 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
527 except FileNotFoundError:
528 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000529
530 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800531 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000532 """
533 self._target_prepare()
534 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000535 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700536 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000537
538 def build(self):
539 """ Builds according to self.target
540 """
541 print('Building target ', self.target)
542
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800543 # Validate that the target is valid
544 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800545 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800546 return
547
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000548 if self.target == 'prepare':
549 self._target_prepare()
550 elif self.target == 'tools':
551 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100552 elif self.target == 'rootcanal':
553 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000554 elif self.target == 'rust':
555 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800556 elif self.target == 'docs':
557 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000558 elif self.target == 'main':
559 self._target_main()
560 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000561 self._target_test()
562 elif self.target == 'clean':
563 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800564 elif self.target == 'install':
565 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000566 elif self.target == 'all':
567 self._target_all()
568
569
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700570class Bootstrap():
571
572 def __init__(self, base_dir, bt_dir):
573 """ Construct bootstrapper.
574
575 Args:
576 base_dir: Where to stage everything.
577 bt_dir: Where bluetooth source is kept (will be symlinked)
578 """
579 self.base_dir = os.path.abspath(base_dir)
580 self.bt_dir = os.path.abspath(bt_dir)
581
582 # Create base directory if it doesn't already exist
583 os.makedirs(self.base_dir, exist_ok=True)
584
585 if not os.path.isdir(self.bt_dir):
586 raise Exception('{} is not a valid directory'.format(self.bt_dir))
587
588 self.git_dir = os.path.join(self.base_dir, 'repos')
589 self.staging_dir = os.path.join(self.base_dir, 'staging')
590 self.output_dir = os.path.join(self.base_dir, 'output')
591 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
592
593 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
594
595 def _update_platform2(self):
596 """Updates repositories used for build."""
597 for repo in BOOTSTRAP_GIT_REPOS.keys():
598 cwd = os.path.join(self.git_dir, repo)
599 subprocess.check_call(['git', 'pull'], cwd=cwd)
600
601 def _setup_platform2(self):
602 """ Set up platform2.
603
604 This will check out all the git repos and symlink everything correctly.
605 """
606
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800607 # Create all directories we will need to use
608 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
609 os.makedirs(dirpath, exist_ok=True)
610
611 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700612 if os.path.isfile(self.dir_setup_complete):
613 print('{} already set-up. Updating instead.'.format(self.base_dir))
614 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800615 else:
616 # Check out all repos in git directory
617 for repo in BOOTSTRAP_GIT_REPOS.values():
618 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700619
620 # Symlink things
621 symlinks = [
622 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700623 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700624 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
625 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
626 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
627 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
628 ]
629
630 # Create symlinks
631 for pairs in symlinks:
632 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000633 try:
634 os.unlink(dst)
635 except Exception as e:
636 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700637 os.symlink(src, dst)
638
639 # Write to setup complete file so we don't repeat this step
640 with open(self.dir_setup_complete, 'w') as f:
641 f.write('Setup complete.')
642
643 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
644 """ Pretty print an install command.
645
646 Args:
647 install_cmd: Prefixed install command.
648 packages: Enumerate packages and append them to install command.
649 line_limit: Number of characters per line.
650
651 Return:
652 Array of lines to join and print.
653 """
654 install = [install_cmd]
655 line = ' '
656 # Remainder needed = space + len(pkg) + space + \
657 # Assuming 80 character lines, that's 80 - 3 = 77
658 line_limit = line_limit - 3
659 for pkg in packages:
660 if len(line) + len(pkg) < line_limit:
661 line = '{}{} '.format(line, pkg)
662 else:
663 install.append(line)
664 line = ' {} '.format(pkg)
665
666 if len(line) > 0:
667 install.append(line)
668
669 return install
670
671 def _check_package_installed(self, package, cmd, predicate):
672 """Check that the given package is installed.
673
674 Args:
675 package: Check that this package is installed.
676 cmd: Command prefix to check if installed (package appended to end)
677 predicate: Function/lambda to check if package is installed based
678 on output. Takes string output and returns boolean.
679
680 Return:
681 True if package is installed.
682 """
683 try:
684 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
685 is_installed = predicate(output.decode('utf-8'))
686 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
687
688 return is_installed
689 except Exception as e:
690 print(e)
691 return False
692
693 def _get_command_output(self, cmd):
694 """Runs the command and gets the output.
695
696 Args:
697 cmd: Command to run.
698
699 Return:
700 Tuple (Success, Output). Success represents if the command ran ok.
701 """
702 try:
703 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
704 return (True, output.decode('utf-8').split('\n'))
705 except Exception as e:
706 print(e)
707 return (False, "")
708
709 def _print_missing_packages(self):
710 """Print any missing packages found via apt.
711
712 This will find any missing packages necessary for build using apt and
713 print it out as an apt-get install printf.
714 """
715 print('Checking for any missing packages...')
716
717 (success, output) = self._get_command_output(APT_PKG_LIST)
718 if not success:
719 raise Exception("Could not query apt for packages.")
720
721 packages_installed = {}
722 for line in output:
723 if 'installed' in line:
724 split = line.split('/', 2)
725 packages_installed[split[0]] = True
726
727 need_packages = []
728 for pkg in REQUIRED_APT_PACKAGES:
729 if pkg not in packages_installed:
730 need_packages.append(pkg)
731
732 # No packages need to be installed
733 if len(need_packages) == 0:
734 print('+ All required packages are installed')
735 return
736
737 install = self._pretty_print_install('sudo apt-get install', need_packages)
738
739 # Print all lines so they can be run in cmdline
740 print('Missing system packages. Run the following command: ')
741 print(' \\\n'.join(install))
742
743 def _print_missing_rust_packages(self):
744 """Print any missing packages found via cargo.
745
746 This will find any missing packages necessary for build using cargo and
747 print it out as a cargo-install printf.
748 """
749 print('Checking for any missing cargo packages...')
750
751 (success, output) = self._get_command_output(CARGO_PKG_LIST)
752 if not success:
753 raise Exception("Could not query cargo for packages.")
754
755 packages_installed = {}
756 for line in output:
757 # Cargo installed packages have this format
758 # <crate name> <version>:
759 # <binary name>
760 # We only care about the crates themselves
761 if ':' not in line:
762 continue
763
764 split = line.split(' ', 2)
765 packages_installed[split[0]] = True
766
767 need_packages = []
768 for pkg in REQUIRED_CARGO_PACKAGES:
769 if pkg not in packages_installed:
770 need_packages.append(pkg)
771
772 # No packages to be installed
773 if len(need_packages) == 0:
774 print('+ All required cargo packages are installed')
775 return
776
777 install = self._pretty_print_install('cargo install', need_packages)
778 print('Missing cargo packages. Run the following command: ')
779 print(' \\\n'.join(install))
780
781 def bootstrap(self):
782 """ Bootstrap the Linux build."""
783 self._setup_platform2()
784 self._print_missing_packages()
785 self._print_missing_rust_packages()
786
787
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000788if __name__ == '__main__':
789 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700790 parser.add_argument(
791 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
792 parser.add_argument(
793 '--run-bootstrap',
794 help='Run bootstrap code to verify build env is ok to build.',
795 default=False,
796 action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000797 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800798 parser.add_argument(
799 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000800 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800801 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
802 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000803 parser.add_argument('--target', help='Run specific build target')
804 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700805 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000806 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700807 parser.add_argument(
808 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000809 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000810 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700811
812 # Make sure we get absolute path + expanded path for bootstrap directory
813 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
814
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000815 # Possible values for machine() come from 'uname -m'
816 # Since this script only runs on Linux, x86_64 machines must have this value
817 if platform.machine() != 'x86_64':
818 raise Exception("Only x86_64 machines are currently supported by this build script.")
819
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700820 if args.run_bootstrap:
821 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
822 bootstrap.bootstrap()
823 else:
824 build = HostBuild(args)
825 build.build()