blob: 485607d98ca3bdefead9944d9e1aa638ae13189f [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
Andre Bragaaa11e7d2022-08-10 21:46:44 +000033import platform
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000034import shutil
35import six
36import subprocess
37import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080038import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070039import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000040
41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
42COMMON_MK_USES = [
43 'asan',
44 'coverage',
45 'cros_host',
46 'fuzzer',
47 'fuzzer',
48 'msan',
49 'profiling',
50 'tcmalloc',
51 'test',
52 'ubsan',
53]
54
55# Default use flags.
56USE_DEFAULTS = {
57 'android': False,
58 'bt_nonstandard_codecs': False,
59 'test': False,
60}
61
62VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000063 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080064 'clean', # Clean up output directory
65 'docs', # Build Rust docs
66 'main', # Build the main C++ codebase
67 'prepare', # Prepare the output directory (gn gen + rust setup)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +010068 'rootcanal', # Build Rust targets for RootCanal
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080069 'rust', # Build only the rust components + copy artifacts to output dir
70 'test', # Run the unit tests
71 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000072]
73
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000075HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000076 # 'bluetooth_test_common',
77 # 'bluetoothtbd_test',
78 # 'net_test_avrcp',
79 # 'net_test_btcore',
80 # 'net_test_types',
81 # 'net_test_btm_iso',
82 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000083]
84
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070085BOOTSTRAP_GIT_REPOS = {
86 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
87 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
88 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
89}
90
91# List of packages required for linux build
92REQUIRED_APT_PACKAGES = [
93 'bison',
94 'build-essential',
95 'curl',
96 'debmake',
97 'flatbuffers-compiler',
98 'flex',
99 'g++-multilib',
100 'gcc-multilib',
101 'generate-ninja',
102 'gnupg',
103 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000104 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700105 'libc++-dev',
106 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000107 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700108 'libevent-dev',
109 'libevent-dev',
110 'libflatbuffers-dev',
111 'libflatbuffers1',
112 'libgl1-mesa-dev',
113 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000114 'libgtest-dev',
115 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700116 'liblz4-tool',
117 'libncurses5',
118 'libnss3-dev',
119 'libprotobuf-dev',
120 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000121 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700122 'libssl-dev',
123 'libtinyxml2-dev',
124 'libx11-dev',
125 'libxml2-utils',
126 'ninja-build',
127 'openssl',
128 'protobuf-compiler',
129 'unzip',
130 'x11proto-core-dev',
131 'xsltproc',
132 'zip',
133 'zlib1g-dev',
134]
135
136# List of cargo packages required for linux build
137REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
138
139APT_PKG_LIST = ['apt', '-qq', 'list']
140CARGO_PKG_LIST = ['cargo', 'install', '--list']
141
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000142
143class UseFlags():
144
145 def __init__(self, use_flags):
146 """ Construct the use flags.
147
148 Args:
149 use_flags: List of use flags parsed from the command.
150 """
151 self.flags = {}
152
153 # Import use flags required by common-mk
154 for use in COMMON_MK_USES:
155 self.set_flag(use, False)
156
157 # Set our defaults
158 for use, value in USE_DEFAULTS.items():
159 self.set_flag(use, value)
160
161 # Set use flags - value is set to True unless the use starts with -
162 # All given use flags always override the defaults
163 for use in use_flags:
164 value = not use.startswith('-')
165 self.set_flag(use, value)
166
167 def set_flag(self, key, value=True):
168 setattr(self, key, value)
169 self.flags[key] = value
170
171
172class HostBuild():
173
174 def __init__(self, args):
175 """ Construct the builder.
176
177 Args:
178 args: Parsed arguments from ArgumentParser
179 """
180 self.args = args
181
182 # Set jobs to number of cpus unless explicitly set
183 self.jobs = self.args.jobs
184 if not self.jobs:
185 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000186 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000187
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700188 # Normalize bootstrap dir and make sure it exists
189 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
190 os.makedirs(self.bootstrap_dir, exist_ok=True)
191
192 # Output and platform directories are based on bootstrap
193 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
194 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000195 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800197 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000198
199 # If default target isn't set, build everything
200 self.target = 'all'
201 if hasattr(self.args, 'target') and self.args.target:
202 self.target = self.args.target
203
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000204 target_use = self.args.use if self.args.use else []
205
206 # Unless set, always build test code
207 if not self.args.notest:
208 target_use.append('test')
209
210 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000211
212 # Validate platform directory
213 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
214 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
215
216 # Make sure output directory exists (or create it)
217 os.makedirs(self.output_dir, exist_ok=True)
218
219 # Set some default attributes
220 self.libbase_ver = None
221
222 self.configure_environ()
223
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000224 def _generate_rustflags(self):
225 """ Rustflags to include for the build.
226 """
227 rust_flags = [
228 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700229 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000230 '-C',
231 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700232 # exclude uninteresting warnings
233 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000234 ]
235
236 return ' '.join(rust_flags)
237
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000238 def configure_environ(self):
239 """ Configure environment variables for GN and Cargo.
240 """
241 self.env = os.environ.copy()
242
243 # Make sure cargo home dir exists and has a bin directory
244 cargo_home = os.path.join(self.output_dir, 'cargo_home')
245 os.makedirs(cargo_home, exist_ok=True)
246 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
247
248 # Configure Rust env variables
249 self.env['CARGO_TARGET_DIR'] = self.output_dir
250 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000251 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700252 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700253 self.env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
Sonny Sasakac1bc8e12022-06-14 17:52:50 -0700254 self.env['CXX_OUTDIR'] = self._gn_default_output()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000255
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000256 def run_command(self, target, args, cwd=None, env=None):
257 """ Run command and stream the output.
258 """
259 # Set some defaults
260 if not cwd:
261 cwd = self.platform_dir
262 if not env:
263 env = self.env
264
265 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
266 with open(log_file, 'wb') as lf:
267 rc = 0
268 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
269 while True:
270 line = process.stdout.readline()
271 print(line.decode('utf-8'), end="")
272 lf.write(line)
273 if not line:
274 rc = process.poll()
275 if rc is not None:
276 break
277
278 time.sleep(0.1)
279
280 if rc != 0:
281 raise Exception("Return code is {}".format(rc))
282
283 def _get_basever(self):
284 if self.libbase_ver:
285 return self.libbase_ver
286
287 self.libbase_ver = os.environ.get('BASE_VER', '')
288 if not self.libbase_ver:
289 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
290 try:
291 with open(base_file, 'r') as f:
292 self.libbase_ver = f.read().strip('\n')
293 except:
294 self.libbase_ver = 'NOT-INSTALLED'
295
296 return self.libbase_ver
297
298 def _gn_default_output(self):
299 return os.path.join(self.output_dir, 'out/Default')
300
301 def _gn_configure(self):
302 """ Configure all required parameters for platform2.
303
304 Mostly copied from //common-mk/platform2.py
305 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700306 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000307
308 def to_gn_string(s):
309 return '"%s"' % s.replace('"', '\\"')
310
311 def to_gn_list(strs):
312 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
313
314 def to_gn_args_args(gn_args):
315 for k, v in gn_args.items():
316 if isinstance(v, bool):
317 v = str(v).lower()
318 elif isinstance(v, list):
319 v = to_gn_list(v)
320 elif isinstance(v, six.string_types):
321 v = to_gn_string(v)
322 else:
323 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
324 yield '%s=%s' % (k.replace('-', '_'), v)
325
326 gn_args = {
327 'platform_subdir': 'bt',
328 'cc': 'clang' if clang else 'gcc',
329 'cxx': 'clang++' if clang else 'g++',
330 'ar': 'llvm-ar' if clang else 'ar',
331 'pkg-config': 'pkg-config',
332 'clang_cc': clang,
333 'clang_cxx': clang,
334 'OS': 'linux',
335 'sysroot': self.sysroot,
336 'libdir': os.path.join(self.sysroot, self.libdir),
337 'build_root': self.output_dir,
338 'platform2_root': self.platform_dir,
339 'libbase_ver': self._get_basever(),
340 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
341 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800342 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000343 'enable_werror': False,
344 }
345
346 if clang:
347 # Make sure to mark the clang use flag as true
348 self.use.set_flag('clang', True)
349 gn_args['external_cxxflags'] += ['-I/usr/include/']
350
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000351 gn_args_args = list(to_gn_args_args(gn_args))
352 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
353 gn_args_args += ['use={%s}' % (' '.join(use_args))]
354
355 gn_args = [
356 'gn',
357 'gen',
358 ]
359
360 if self.args.verbose:
361 gn_args.append('-v')
362
363 gn_args += [
364 '--root=%s' % self.platform_dir,
365 '--args=%s' % ' '.join(gn_args_args),
366 self._gn_default_output(),
367 ]
368
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700369 if 'PKG_CONFIG_PATH' in self.env:
370 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000371
372 self.run_command('configure', gn_args)
373
374 def _gn_build(self, target):
375 """ Generate the ninja command for the target and run it.
376 """
377 args = ['%s:%s' % ('bt', target)]
378 ninja_args = ['ninja', '-C', self._gn_default_output()]
379 if self.jobs:
380 ninja_args += ['-j', str(self.jobs)]
381 ninja_args += args
382
383 if self.args.verbose:
384 ninja_args.append('-v')
385
386 self.run_command('build', ninja_args)
387
388 def _rust_configure(self):
389 """ Generate config file at cargo_home so we use vendored crates.
390 """
391 template = """
392 [source.systembt]
393 directory = "{}/external/rust/vendor"
394
395 [source.crates-io]
396 replace-with = "systembt"
397 local-registry = "/nonexistent"
398 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700399
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700400 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700401 contents = template.format(self.platform_dir)
402 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
403 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000404
405 def _rust_build(self):
406 """ Run `cargo build` from platform2/bt directory.
407 """
408 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
409
410 def _target_prepare(self):
411 """ Target to prepare the output directory for building.
412
413 This runs gn gen to generate all rquired files and set up the Rust
414 config properly. This will be run
415 """
416 self._gn_configure()
417 self._rust_configure()
418
419 def _target_tools(self):
420 """ Build the tools target in an already prepared environment.
421 """
422 self._gn_build('tools')
423
424 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
425 shutil.copy(
426 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
427
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800428 def _target_docs(self):
429 """Build the Rust docs."""
430 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
431
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000432 def _target_rust(self):
433 """ Build rust artifacts in an already prepared environment.
434 """
435 self._rust_build()
436
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100437 def _target_rootcanal(self):
438 """ Build rust artifacts for RootCanal in an already prepared environment.
439 """
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700440 self.run_command(
441 'rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100442
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000443 def _target_main(self):
444 """ Build the main GN artifacts in an already prepared environment.
445 """
446 self._gn_build('all')
447
448 def _target_test(self):
449 """ Runs the host tests.
450 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000451 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800452 rust_test_cmd = ['cargo', 'test']
453 if self.args.test_name:
Abhishek Pandit-Subedib04e6a92022-09-08 19:19:40 -0700454 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"]
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800455
456 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100457 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt/tools/rootcanal'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000458
459 # Host tests second based on host test list
460 for t in HOST_TESTS:
461 self.run_command(
462 'test', [os.path.join(self.output_dir, 'out/Default', t)],
463 cwd=os.path.join(self.output_dir),
464 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000465
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800466 def _target_install(self):
467 """ Installs files required to run Floss to install directory.
468 """
469 # First make the install directory
470 prefix = self.install_dir
471 os.makedirs(prefix, exist_ok=True)
472
473 # Next save the cwd and change to install directory
474 last_cwd = os.getcwd()
475 os.chdir(prefix)
476
477 bindir = os.path.join(self.output_dir, 'debug')
478 srcdir = os.path.dirname(__file__)
479
480 install_map = [
481 {
482 'src': os.path.join(bindir, 'btadapterd'),
483 'dst': 'usr/libexec/bluetooth/btadapterd',
484 'strip': True
485 },
486 {
487 'src': os.path.join(bindir, 'btmanagerd'),
488 'dst': 'usr/libexec/bluetooth/btmanagerd',
489 'strip': True
490 },
491 {
492 'src': os.path.join(bindir, 'btclient'),
493 'dst': 'usr/local/bin/btclient',
494 'strip': True
495 },
496 ]
497
498 for v in install_map:
499 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
500 dst = os.path.join(prefix, partial_dst)
501
502 # Create dst directory first and copy file there
503 os.makedirs(os.path.dirname(dst), exist_ok=True)
504 print('Installing {}'.format(dst))
505 shutil.copy(src, dst)
506
507 # Binary should be marked for strip and no-strip option shouldn't be
508 # set. No-strip is useful while debugging.
509 if strip and not self.args.no_strip:
510 self.run_command('install', ['llvm-strip', dst])
511
512 # Put all files into a tar.gz for easier installation
513 tar_location = os.path.join(prefix, 'floss.tar.gz')
514 with tarfile.open(tar_location, 'w:gz') as tar:
515 for v in install_map:
516 tar.add(v['dst'])
517
518 print('Tarball created at {}'.format(tar_location))
519
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000520 def _target_clean(self):
521 """ Delete the output directory entirely.
522 """
523 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800524
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700525 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800526 try:
527 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
528 except FileNotFoundError:
529 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000530
531 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800532 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000533 """
534 self._target_prepare()
535 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000536 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700537 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000538
539 def build(self):
540 """ Builds according to self.target
541 """
542 print('Building target ', self.target)
543
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800544 # Validate that the target is valid
545 if self.target not in VALID_TARGETS:
howardchung63187b62022-08-16 17:06:17 +0800546 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS))
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800547 return
548
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000549 if self.target == 'prepare':
550 self._target_prepare()
551 elif self.target == 'tools':
552 self._target_tools()
Ivan Podogovd7b6b6d2022-08-10 11:07:12 +0100553 elif self.target == 'rootcanal':
554 self._target_rootcanal()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000555 elif self.target == 'rust':
556 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800557 elif self.target == 'docs':
558 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000559 elif self.target == 'main':
560 self._target_main()
561 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000562 self._target_test()
563 elif self.target == 'clean':
564 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800565 elif self.target == 'install':
566 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000567 elif self.target == 'all':
568 self._target_all()
569
570
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700571class Bootstrap():
572
573 def __init__(self, base_dir, bt_dir):
574 """ Construct bootstrapper.
575
576 Args:
577 base_dir: Where to stage everything.
578 bt_dir: Where bluetooth source is kept (will be symlinked)
579 """
580 self.base_dir = os.path.abspath(base_dir)
581 self.bt_dir = os.path.abspath(bt_dir)
582
583 # Create base directory if it doesn't already exist
584 os.makedirs(self.base_dir, exist_ok=True)
585
586 if not os.path.isdir(self.bt_dir):
587 raise Exception('{} is not a valid directory'.format(self.bt_dir))
588
589 self.git_dir = os.path.join(self.base_dir, 'repos')
590 self.staging_dir = os.path.join(self.base_dir, 'staging')
591 self.output_dir = os.path.join(self.base_dir, 'output')
592 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
593
594 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
595
596 def _update_platform2(self):
597 """Updates repositories used for build."""
598 for repo in BOOTSTRAP_GIT_REPOS.keys():
599 cwd = os.path.join(self.git_dir, repo)
600 subprocess.check_call(['git', 'pull'], cwd=cwd)
601
602 def _setup_platform2(self):
603 """ Set up platform2.
604
605 This will check out all the git repos and symlink everything correctly.
606 """
607
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800608 # Create all directories we will need to use
609 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
610 os.makedirs(dirpath, exist_ok=True)
611
612 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700613 if os.path.isfile(self.dir_setup_complete):
614 print('{} already set-up. Updating instead.'.format(self.base_dir))
615 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800616 else:
617 # Check out all repos in git directory
618 for repo in BOOTSTRAP_GIT_REPOS.values():
619 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700620
621 # Symlink things
622 symlinks = [
623 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700624 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700625 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
626 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
627 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
628 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
629 ]
630
631 # Create symlinks
632 for pairs in symlinks:
633 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000634 try:
635 os.unlink(dst)
636 except Exception as e:
637 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700638 os.symlink(src, dst)
639
640 # Write to setup complete file so we don't repeat this step
641 with open(self.dir_setup_complete, 'w') as f:
642 f.write('Setup complete.')
643
644 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
645 """ Pretty print an install command.
646
647 Args:
648 install_cmd: Prefixed install command.
649 packages: Enumerate packages and append them to install command.
650 line_limit: Number of characters per line.
651
652 Return:
653 Array of lines to join and print.
654 """
655 install = [install_cmd]
656 line = ' '
657 # Remainder needed = space + len(pkg) + space + \
658 # Assuming 80 character lines, that's 80 - 3 = 77
659 line_limit = line_limit - 3
660 for pkg in packages:
661 if len(line) + len(pkg) < line_limit:
662 line = '{}{} '.format(line, pkg)
663 else:
664 install.append(line)
665 line = ' {} '.format(pkg)
666
667 if len(line) > 0:
668 install.append(line)
669
670 return install
671
672 def _check_package_installed(self, package, cmd, predicate):
673 """Check that the given package is installed.
674
675 Args:
676 package: Check that this package is installed.
677 cmd: Command prefix to check if installed (package appended to end)
678 predicate: Function/lambda to check if package is installed based
679 on output. Takes string output and returns boolean.
680
681 Return:
682 True if package is installed.
683 """
684 try:
685 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
686 is_installed = predicate(output.decode('utf-8'))
687 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
688
689 return is_installed
690 except Exception as e:
691 print(e)
692 return False
693
694 def _get_command_output(self, cmd):
695 """Runs the command and gets the output.
696
697 Args:
698 cmd: Command to run.
699
700 Return:
701 Tuple (Success, Output). Success represents if the command ran ok.
702 """
703 try:
704 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
705 return (True, output.decode('utf-8').split('\n'))
706 except Exception as e:
707 print(e)
708 return (False, "")
709
710 def _print_missing_packages(self):
711 """Print any missing packages found via apt.
712
713 This will find any missing packages necessary for build using apt and
714 print it out as an apt-get install printf.
715 """
716 print('Checking for any missing packages...')
717
718 (success, output) = self._get_command_output(APT_PKG_LIST)
719 if not success:
720 raise Exception("Could not query apt for packages.")
721
722 packages_installed = {}
723 for line in output:
724 if 'installed' in line:
725 split = line.split('/', 2)
726 packages_installed[split[0]] = True
727
728 need_packages = []
729 for pkg in REQUIRED_APT_PACKAGES:
730 if pkg not in packages_installed:
731 need_packages.append(pkg)
732
733 # No packages need to be installed
734 if len(need_packages) == 0:
735 print('+ All required packages are installed')
736 return
737
738 install = self._pretty_print_install('sudo apt-get install', need_packages)
739
740 # Print all lines so they can be run in cmdline
741 print('Missing system packages. Run the following command: ')
742 print(' \\\n'.join(install))
743
744 def _print_missing_rust_packages(self):
745 """Print any missing packages found via cargo.
746
747 This will find any missing packages necessary for build using cargo and
748 print it out as a cargo-install printf.
749 """
750 print('Checking for any missing cargo packages...')
751
752 (success, output) = self._get_command_output(CARGO_PKG_LIST)
753 if not success:
754 raise Exception("Could not query cargo for packages.")
755
756 packages_installed = {}
757 for line in output:
758 # Cargo installed packages have this format
759 # <crate name> <version>:
760 # <binary name>
761 # We only care about the crates themselves
762 if ':' not in line:
763 continue
764
765 split = line.split(' ', 2)
766 packages_installed[split[0]] = True
767
768 need_packages = []
769 for pkg in REQUIRED_CARGO_PACKAGES:
770 if pkg not in packages_installed:
771 need_packages.append(pkg)
772
773 # No packages to be installed
774 if len(need_packages) == 0:
775 print('+ All required cargo packages are installed')
776 return
777
778 install = self._pretty_print_install('cargo install', need_packages)
779 print('Missing cargo packages. Run the following command: ')
780 print(' \\\n'.join(install))
781
782 def bootstrap(self):
783 """ Bootstrap the Linux build."""
784 self._setup_platform2()
785 self._print_missing_packages()
786 self._print_missing_rust_packages()
787
788
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000789if __name__ == '__main__':
790 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700791 parser.add_argument(
792 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
793 parser.add_argument(
794 '--run-bootstrap',
795 help='Run bootstrap code to verify build env is ok to build.',
796 default=False,
797 action='store_true')
Andre Braga17ff7bc2022-06-24 22:43:18 +0000798 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800799 parser.add_argument(
800 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000801 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800802 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
803 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000804 parser.add_argument('--target', help='Run specific build target')
805 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700806 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000807 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700808 parser.add_argument(
809 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000810 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000811 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700812
813 # Make sure we get absolute path + expanded path for bootstrap directory
814 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
815
Andre Bragaaa11e7d2022-08-10 21:46:44 +0000816 # Possible values for machine() come from 'uname -m'
817 # Since this script only runs on Linux, x86_64 machines must have this value
818 if platform.machine() != 'x86_64':
819 raise Exception("Only x86_64 machines are currently supported by this build script.")
820
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700821 if args.run_bootstrap:
822 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
823 bootstrap.bootstrap()
824 else:
825 build = HostBuild(args)
826 build.build()