blob: 5d0e0d9cd874b3e7db67afeb9900fc8cb2b9d5a5 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
33import shutil
34import six
35import subprocess
36import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080037import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070038import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000039
40# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
41COMMON_MK_USES = [
42 'asan',
43 'coverage',
44 'cros_host',
45 'fuzzer',
46 'fuzzer',
47 'msan',
48 'profiling',
49 'tcmalloc',
50 'test',
51 'ubsan',
52]
53
54# Default use flags.
55USE_DEFAULTS = {
56 'android': False,
57 'bt_nonstandard_codecs': False,
58 'test': False,
59}
60
61VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000062 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080063 'clean', # Clean up output directory
64 'docs', # Build Rust docs
65 'main', # Build the main C++ codebase
66 'prepare', # Prepare the output directory (gn gen + rust setup)
67 'rust', # Build only the rust components + copy artifacts to output dir
68 'test', # Run the unit tests
69 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000070]
71
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000072# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000073HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074 # 'bluetooth_test_common',
75 # 'bluetoothtbd_test',
76 # 'net_test_avrcp',
77 # 'net_test_btcore',
78 # 'net_test_types',
79 # 'net_test_btm_iso',
80 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000081]
82
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070083BOOTSTRAP_GIT_REPOS = {
84 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
85 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
86 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
87}
88
89# List of packages required for linux build
90REQUIRED_APT_PACKAGES = [
91 'bison',
92 'build-essential',
93 'curl',
94 'debmake',
95 'flatbuffers-compiler',
96 'flex',
97 'g++-multilib',
98 'gcc-multilib',
99 'generate-ninja',
100 'gnupg',
101 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000102 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700103 'libc++-dev',
104 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000105 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700106 'libevent-dev',
107 'libevent-dev',
108 'libflatbuffers-dev',
109 'libflatbuffers1',
110 'libgl1-mesa-dev',
111 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000112 'libgtest-dev',
113 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700114 'liblz4-tool',
115 'libncurses5',
116 'libnss3-dev',
117 'libprotobuf-dev',
118 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000119 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700120 'libssl-dev',
121 'libtinyxml2-dev',
122 'libx11-dev',
123 'libxml2-utils',
124 'ninja-build',
125 'openssl',
126 'protobuf-compiler',
127 'unzip',
128 'x11proto-core-dev',
129 'xsltproc',
130 'zip',
131 'zlib1g-dev',
132]
133
134# List of cargo packages required for linux build
135REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
136
137APT_PKG_LIST = ['apt', '-qq', 'list']
138CARGO_PKG_LIST = ['cargo', 'install', '--list']
139
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000140
141class UseFlags():
142
143 def __init__(self, use_flags):
144 """ Construct the use flags.
145
146 Args:
147 use_flags: List of use flags parsed from the command.
148 """
149 self.flags = {}
150
151 # Import use flags required by common-mk
152 for use in COMMON_MK_USES:
153 self.set_flag(use, False)
154
155 # Set our defaults
156 for use, value in USE_DEFAULTS.items():
157 self.set_flag(use, value)
158
159 # Set use flags - value is set to True unless the use starts with -
160 # All given use flags always override the defaults
161 for use in use_flags:
162 value = not use.startswith('-')
163 self.set_flag(use, value)
164
165 def set_flag(self, key, value=True):
166 setattr(self, key, value)
167 self.flags[key] = value
168
169
170class HostBuild():
171
172 def __init__(self, args):
173 """ Construct the builder.
174
175 Args:
176 args: Parsed arguments from ArgumentParser
177 """
178 self.args = args
179
180 # Set jobs to number of cpus unless explicitly set
181 self.jobs = self.args.jobs
182 if not self.jobs:
183 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000184 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000185
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700186 # Normalize bootstrap dir and make sure it exists
187 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
188 os.makedirs(self.bootstrap_dir, exist_ok=True)
189
190 # Output and platform directories are based on bootstrap
191 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
192 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000193 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000194 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800195 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196
197 # If default target isn't set, build everything
198 self.target = 'all'
199 if hasattr(self.args, 'target') and self.args.target:
200 self.target = self.args.target
201
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000202 target_use = self.args.use if self.args.use else []
203
204 # Unless set, always build test code
205 if not self.args.notest:
206 target_use.append('test')
207
208 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000209
210 # Validate platform directory
211 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
212 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
213
214 # Make sure output directory exists (or create it)
215 os.makedirs(self.output_dir, exist_ok=True)
216
217 # Set some default attributes
218 self.libbase_ver = None
219
220 self.configure_environ()
221
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000222 def _generate_rustflags(self):
223 """ Rustflags to include for the build.
224 """
225 rust_flags = [
226 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700227 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000228 '-C',
229 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700230 # exclude uninteresting warnings
231 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000232 ]
233
234 return ' '.join(rust_flags)
235
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000236 def configure_environ(self):
237 """ Configure environment variables for GN and Cargo.
238 """
239 self.env = os.environ.copy()
240
241 # Make sure cargo home dir exists and has a bin directory
242 cargo_home = os.path.join(self.output_dir, 'cargo_home')
243 os.makedirs(cargo_home, exist_ok=True)
244 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
245
246 # Configure Rust env variables
247 self.env['CARGO_TARGET_DIR'] = self.output_dir
248 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000249 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700250 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700251 self.env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000252
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000253 def run_command(self, target, args, cwd=None, env=None):
254 """ Run command and stream the output.
255 """
256 # Set some defaults
257 if not cwd:
258 cwd = self.platform_dir
259 if not env:
260 env = self.env
261
262 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
263 with open(log_file, 'wb') as lf:
264 rc = 0
265 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
266 while True:
267 line = process.stdout.readline()
268 print(line.decode('utf-8'), end="")
269 lf.write(line)
270 if not line:
271 rc = process.poll()
272 if rc is not None:
273 break
274
275 time.sleep(0.1)
276
277 if rc != 0:
278 raise Exception("Return code is {}".format(rc))
279
280 def _get_basever(self):
281 if self.libbase_ver:
282 return self.libbase_ver
283
284 self.libbase_ver = os.environ.get('BASE_VER', '')
285 if not self.libbase_ver:
286 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
287 try:
288 with open(base_file, 'r') as f:
289 self.libbase_ver = f.read().strip('\n')
290 except:
291 self.libbase_ver = 'NOT-INSTALLED'
292
293 return self.libbase_ver
294
295 def _gn_default_output(self):
296 return os.path.join(self.output_dir, 'out/Default')
297
298 def _gn_configure(self):
299 """ Configure all required parameters for platform2.
300
301 Mostly copied from //common-mk/platform2.py
302 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700303 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000304
305 def to_gn_string(s):
306 return '"%s"' % s.replace('"', '\\"')
307
308 def to_gn_list(strs):
309 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
310
311 def to_gn_args_args(gn_args):
312 for k, v in gn_args.items():
313 if isinstance(v, bool):
314 v = str(v).lower()
315 elif isinstance(v, list):
316 v = to_gn_list(v)
317 elif isinstance(v, six.string_types):
318 v = to_gn_string(v)
319 else:
320 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
321 yield '%s=%s' % (k.replace('-', '_'), v)
322
323 gn_args = {
324 'platform_subdir': 'bt',
325 'cc': 'clang' if clang else 'gcc',
326 'cxx': 'clang++' if clang else 'g++',
327 'ar': 'llvm-ar' if clang else 'ar',
328 'pkg-config': 'pkg-config',
329 'clang_cc': clang,
330 'clang_cxx': clang,
331 'OS': 'linux',
332 'sysroot': self.sysroot,
333 'libdir': os.path.join(self.sysroot, self.libdir),
334 'build_root': self.output_dir,
335 'platform2_root': self.platform_dir,
336 'libbase_ver': self._get_basever(),
337 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
338 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800339 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000340 'enable_werror': False,
341 }
342
343 if clang:
344 # Make sure to mark the clang use flag as true
345 self.use.set_flag('clang', True)
346 gn_args['external_cxxflags'] += ['-I/usr/include/']
347
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000348 gn_args_args = list(to_gn_args_args(gn_args))
349 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
350 gn_args_args += ['use={%s}' % (' '.join(use_args))]
351
352 gn_args = [
353 'gn',
354 'gen',
355 ]
356
357 if self.args.verbose:
358 gn_args.append('-v')
359
360 gn_args += [
361 '--root=%s' % self.platform_dir,
362 '--args=%s' % ' '.join(gn_args_args),
363 self._gn_default_output(),
364 ]
365
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700366 if 'PKG_CONFIG_PATH' in self.env:
367 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000368
369 self.run_command('configure', gn_args)
370
371 def _gn_build(self, target):
372 """ Generate the ninja command for the target and run it.
373 """
374 args = ['%s:%s' % ('bt', target)]
375 ninja_args = ['ninja', '-C', self._gn_default_output()]
376 if self.jobs:
377 ninja_args += ['-j', str(self.jobs)]
378 ninja_args += args
379
380 if self.args.verbose:
381 ninja_args.append('-v')
382
383 self.run_command('build', ninja_args)
384
385 def _rust_configure(self):
386 """ Generate config file at cargo_home so we use vendored crates.
387 """
388 template = """
389 [source.systembt]
390 directory = "{}/external/rust/vendor"
391
392 [source.crates-io]
393 replace-with = "systembt"
394 local-registry = "/nonexistent"
395 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700396
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700397 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700398 contents = template.format(self.platform_dir)
399 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
400 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000401
402 def _rust_build(self):
403 """ Run `cargo build` from platform2/bt directory.
404 """
405 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
406
407 def _target_prepare(self):
408 """ Target to prepare the output directory for building.
409
410 This runs gn gen to generate all rquired files and set up the Rust
411 config properly. This will be run
412 """
413 self._gn_configure()
414 self._rust_configure()
415
416 def _target_tools(self):
417 """ Build the tools target in an already prepared environment.
418 """
419 self._gn_build('tools')
420
421 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
422 shutil.copy(
423 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
424
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800425 def _target_docs(self):
426 """Build the Rust docs."""
427 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
428
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000429 def _target_rust(self):
430 """ Build rust artifacts in an already prepared environment.
431 """
432 self._rust_build()
433
434 def _target_main(self):
435 """ Build the main GN artifacts in an already prepared environment.
436 """
437 self._gn_build('all')
438
439 def _target_test(self):
440 """ Runs the host tests.
441 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000442 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800443 rust_test_cmd = ['cargo', 'test']
444 if self.args.test_name:
445 rust_test_cmd = rust_test_cmd + [self.args.test_name]
446
447 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000448
449 # Host tests second based on host test list
450 for t in HOST_TESTS:
451 self.run_command(
452 'test', [os.path.join(self.output_dir, 'out/Default', t)],
453 cwd=os.path.join(self.output_dir),
454 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000455
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800456 def _target_install(self):
457 """ Installs files required to run Floss to install directory.
458 """
459 # First make the install directory
460 prefix = self.install_dir
461 os.makedirs(prefix, exist_ok=True)
462
463 # Next save the cwd and change to install directory
464 last_cwd = os.getcwd()
465 os.chdir(prefix)
466
467 bindir = os.path.join(self.output_dir, 'debug')
468 srcdir = os.path.dirname(__file__)
469
470 install_map = [
471 {
472 'src': os.path.join(bindir, 'btadapterd'),
473 'dst': 'usr/libexec/bluetooth/btadapterd',
474 'strip': True
475 },
476 {
477 'src': os.path.join(bindir, 'btmanagerd'),
478 'dst': 'usr/libexec/bluetooth/btmanagerd',
479 'strip': True
480 },
481 {
482 'src': os.path.join(bindir, 'btclient'),
483 'dst': 'usr/local/bin/btclient',
484 'strip': True
485 },
486 ]
487
488 for v in install_map:
489 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
490 dst = os.path.join(prefix, partial_dst)
491
492 # Create dst directory first and copy file there
493 os.makedirs(os.path.dirname(dst), exist_ok=True)
494 print('Installing {}'.format(dst))
495 shutil.copy(src, dst)
496
497 # Binary should be marked for strip and no-strip option shouldn't be
498 # set. No-strip is useful while debugging.
499 if strip and not self.args.no_strip:
500 self.run_command('install', ['llvm-strip', dst])
501
502 # Put all files into a tar.gz for easier installation
503 tar_location = os.path.join(prefix, 'floss.tar.gz')
504 with tarfile.open(tar_location, 'w:gz') as tar:
505 for v in install_map:
506 tar.add(v['dst'])
507
508 print('Tarball created at {}'.format(tar_location))
509
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000510 def _target_clean(self):
511 """ Delete the output directory entirely.
512 """
513 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800514
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700515 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800516 try:
517 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
518 except FileNotFoundError:
519 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000520
521 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800522 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000523 """
524 self._target_prepare()
525 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000526 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700527 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000528
529 def build(self):
530 """ Builds according to self.target
531 """
532 print('Building target ', self.target)
533
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800534 # Validate that the target is valid
535 if self.target not in VALID_TARGETS:
536 print('Target {} is not valid. Must be in {}', self.target, VALID_TARGETS)
537 return
538
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000539 if self.target == 'prepare':
540 self._target_prepare()
541 elif self.target == 'tools':
542 self._target_tools()
543 elif self.target == 'rust':
544 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800545 elif self.target == 'docs':
546 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000547 elif self.target == 'main':
548 self._target_main()
549 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000550 self._target_test()
551 elif self.target == 'clean':
552 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800553 elif self.target == 'install':
554 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000555 elif self.target == 'all':
556 self._target_all()
557
558
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700559class Bootstrap():
560
561 def __init__(self, base_dir, bt_dir):
562 """ Construct bootstrapper.
563
564 Args:
565 base_dir: Where to stage everything.
566 bt_dir: Where bluetooth source is kept (will be symlinked)
567 """
568 self.base_dir = os.path.abspath(base_dir)
569 self.bt_dir = os.path.abspath(bt_dir)
570
571 # Create base directory if it doesn't already exist
572 os.makedirs(self.base_dir, exist_ok=True)
573
574 if not os.path.isdir(self.bt_dir):
575 raise Exception('{} is not a valid directory'.format(self.bt_dir))
576
577 self.git_dir = os.path.join(self.base_dir, 'repos')
578 self.staging_dir = os.path.join(self.base_dir, 'staging')
579 self.output_dir = os.path.join(self.base_dir, 'output')
580 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
581
582 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
583
584 def _update_platform2(self):
585 """Updates repositories used for build."""
586 for repo in BOOTSTRAP_GIT_REPOS.keys():
587 cwd = os.path.join(self.git_dir, repo)
588 subprocess.check_call(['git', 'pull'], cwd=cwd)
589
590 def _setup_platform2(self):
591 """ Set up platform2.
592
593 This will check out all the git repos and symlink everything correctly.
594 """
595
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800596 # Create all directories we will need to use
597 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
598 os.makedirs(dirpath, exist_ok=True)
599
600 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700601 if os.path.isfile(self.dir_setup_complete):
602 print('{} already set-up. Updating instead.'.format(self.base_dir))
603 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800604 else:
605 # Check out all repos in git directory
606 for repo in BOOTSTRAP_GIT_REPOS.values():
607 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700608
609 # Symlink things
610 symlinks = [
611 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700612 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700613 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
614 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
615 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
616 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
617 ]
618
619 # Create symlinks
620 for pairs in symlinks:
621 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000622 try:
623 os.unlink(dst)
624 except Exception as e:
625 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700626 os.symlink(src, dst)
627
628 # Write to setup complete file so we don't repeat this step
629 with open(self.dir_setup_complete, 'w') as f:
630 f.write('Setup complete.')
631
632 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
633 """ Pretty print an install command.
634
635 Args:
636 install_cmd: Prefixed install command.
637 packages: Enumerate packages and append them to install command.
638 line_limit: Number of characters per line.
639
640 Return:
641 Array of lines to join and print.
642 """
643 install = [install_cmd]
644 line = ' '
645 # Remainder needed = space + len(pkg) + space + \
646 # Assuming 80 character lines, that's 80 - 3 = 77
647 line_limit = line_limit - 3
648 for pkg in packages:
649 if len(line) + len(pkg) < line_limit:
650 line = '{}{} '.format(line, pkg)
651 else:
652 install.append(line)
653 line = ' {} '.format(pkg)
654
655 if len(line) > 0:
656 install.append(line)
657
658 return install
659
660 def _check_package_installed(self, package, cmd, predicate):
661 """Check that the given package is installed.
662
663 Args:
664 package: Check that this package is installed.
665 cmd: Command prefix to check if installed (package appended to end)
666 predicate: Function/lambda to check if package is installed based
667 on output. Takes string output and returns boolean.
668
669 Return:
670 True if package is installed.
671 """
672 try:
673 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
674 is_installed = predicate(output.decode('utf-8'))
675 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
676
677 return is_installed
678 except Exception as e:
679 print(e)
680 return False
681
682 def _get_command_output(self, cmd):
683 """Runs the command and gets the output.
684
685 Args:
686 cmd: Command to run.
687
688 Return:
689 Tuple (Success, Output). Success represents if the command ran ok.
690 """
691 try:
692 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
693 return (True, output.decode('utf-8').split('\n'))
694 except Exception as e:
695 print(e)
696 return (False, "")
697
698 def _print_missing_packages(self):
699 """Print any missing packages found via apt.
700
701 This will find any missing packages necessary for build using apt and
702 print it out as an apt-get install printf.
703 """
704 print('Checking for any missing packages...')
705
706 (success, output) = self._get_command_output(APT_PKG_LIST)
707 if not success:
708 raise Exception("Could not query apt for packages.")
709
710 packages_installed = {}
711 for line in output:
712 if 'installed' in line:
713 split = line.split('/', 2)
714 packages_installed[split[0]] = True
715
716 need_packages = []
717 for pkg in REQUIRED_APT_PACKAGES:
718 if pkg not in packages_installed:
719 need_packages.append(pkg)
720
721 # No packages need to be installed
722 if len(need_packages) == 0:
723 print('+ All required packages are installed')
724 return
725
726 install = self._pretty_print_install('sudo apt-get install', need_packages)
727
728 # Print all lines so they can be run in cmdline
729 print('Missing system packages. Run the following command: ')
730 print(' \\\n'.join(install))
731
732 def _print_missing_rust_packages(self):
733 """Print any missing packages found via cargo.
734
735 This will find any missing packages necessary for build using cargo and
736 print it out as a cargo-install printf.
737 """
738 print('Checking for any missing cargo packages...')
739
740 (success, output) = self._get_command_output(CARGO_PKG_LIST)
741 if not success:
742 raise Exception("Could not query cargo for packages.")
743
744 packages_installed = {}
745 for line in output:
746 # Cargo installed packages have this format
747 # <crate name> <version>:
748 # <binary name>
749 # We only care about the crates themselves
750 if ':' not in line:
751 continue
752
753 split = line.split(' ', 2)
754 packages_installed[split[0]] = True
755
756 need_packages = []
757 for pkg in REQUIRED_CARGO_PACKAGES:
758 if pkg not in packages_installed:
759 need_packages.append(pkg)
760
761 # No packages to be installed
762 if len(need_packages) == 0:
763 print('+ All required cargo packages are installed')
764 return
765
766 install = self._pretty_print_install('cargo install', need_packages)
767 print('Missing cargo packages. Run the following command: ')
768 print(' \\\n'.join(install))
769
770 def bootstrap(self):
771 """ Bootstrap the Linux build."""
772 self._setup_platform2()
773 self._print_missing_packages()
774 self._print_missing_rust_packages()
775
776
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000777if __name__ == '__main__':
778 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700779 parser.add_argument(
780 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
781 parser.add_argument(
782 '--run-bootstrap',
783 help='Run bootstrap code to verify build env is ok to build.',
784 default=False,
785 action='store_true')
786 parser.add_argument('--no-clang', help='Use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800787 parser.add_argument(
788 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000789 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800790 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
791 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000792 parser.add_argument('--target', help='Run specific build target')
793 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700794 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000795 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700796 parser.add_argument(
797 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000798 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000799 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700800
801 # Make sure we get absolute path + expanded path for bootstrap directory
802 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
803
804 if args.run_bootstrap:
805 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
806 bootstrap.bootstrap()
807 else:
808 build = HostBuild(args)
809 build.build()