blob: 46841ac16640148e5d27330c032d5d778d412129 [file] [log] [blame]
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +00001#!/usr/bin/env python3
2
3# Copyright 2021 Google, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at:
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Build BT targets on the host system.
17
18For building, you will first have to stage a platform directory that has the
19following structure:
20|-common-mk
21|-bt
22|-external
23|-|-rust
24|-|-|-vendor
25
26The simplest way to do this is to check out platform2 to another directory (that
27is not a subdir of this bt directory), symlink bt there and symlink the rust
28vendor repository as well.
29"""
30import argparse
31import multiprocessing
32import os
33import shutil
34import six
35import subprocess
36import sys
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -080037import tarfile
Chris Mantone7ad6332021-09-30 22:55:39 -070038import time
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000039
40# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {})
41COMMON_MK_USES = [
42 'asan',
43 'coverage',
44 'cros_host',
45 'fuzzer',
46 'fuzzer',
47 'msan',
48 'profiling',
49 'tcmalloc',
50 'test',
51 'ubsan',
52]
53
54# Default use flags.
55USE_DEFAULTS = {
56 'android': False,
57 'bt_nonstandard_codecs': False,
58 'test': False,
59}
60
61VALID_TARGETS = [
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000062 'all', # All targets except test and clean
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -080063 'clean', # Clean up output directory
64 'docs', # Build Rust docs
65 'main', # Build the main C++ codebase
66 'prepare', # Prepare the output directory (gn gen + rust setup)
67 'rust', # Build only the rust components + copy artifacts to output dir
68 'test', # Run the unit tests
69 'tools', # Build the host tools (i.e. packetgen)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +000070]
71
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000072# TODO(b/190750167) - Host tests are disabled until we are full bazel build
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000073HOST_TESTS = [
Abhishek Pandit-Subedi70e43042021-06-10 21:16:52 +000074 # 'bluetooth_test_common',
75 # 'bluetoothtbd_test',
76 # 'net_test_avrcp',
77 # 'net_test_btcore',
78 # 'net_test_types',
79 # 'net_test_btm_iso',
80 # 'net_test_btpackets',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +000081]
82
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -070083BOOTSTRAP_GIT_REPOS = {
84 'platform2': 'https://chromium.googlesource.com/chromiumos/platform2',
85 'rust_crates': 'https://chromium.googlesource.com/chromiumos/third_party/rust_crates',
86 'proto_logging': 'https://android.googlesource.com/platform/frameworks/proto_logging'
87}
88
89# List of packages required for linux build
90REQUIRED_APT_PACKAGES = [
91 'bison',
92 'build-essential',
93 'curl',
94 'debmake',
95 'flatbuffers-compiler',
96 'flex',
97 'g++-multilib',
98 'gcc-multilib',
99 'generate-ninja',
100 'gnupg',
101 'gperf',
Martin Brabhamba22adf2022-02-04 19:51:21 +0000102 'libc++abi-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700103 'libc++-dev',
104 'libdbus-1-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000105 'libdouble-conversion-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700106 'libevent-dev',
107 'libevent-dev',
108 'libflatbuffers-dev',
109 'libflatbuffers1',
110 'libgl1-mesa-dev',
111 'libglib2.0-dev',
Martin Brabham996f1502022-02-14 17:39:23 +0000112 'libgtest-dev',
113 'libgmock-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700114 'liblz4-tool',
115 'libncurses5',
116 'libnss3-dev',
117 'libprotobuf-dev',
118 'libre2-9',
Martin Brabham996f1502022-02-14 17:39:23 +0000119 'libre2-dev',
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700120 'libssl-dev',
121 'libtinyxml2-dev',
122 'libx11-dev',
123 'libxml2-utils',
124 'ninja-build',
125 'openssl',
126 'protobuf-compiler',
127 'unzip',
128 'x11proto-core-dev',
129 'xsltproc',
130 'zip',
131 'zlib1g-dev',
132]
133
134# List of cargo packages required for linux build
135REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd']
136
137APT_PKG_LIST = ['apt', '-qq', 'list']
138CARGO_PKG_LIST = ['cargo', 'install', '--list']
139
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000140
141class UseFlags():
142
143 def __init__(self, use_flags):
144 """ Construct the use flags.
145
146 Args:
147 use_flags: List of use flags parsed from the command.
148 """
149 self.flags = {}
150
151 # Import use flags required by common-mk
152 for use in COMMON_MK_USES:
153 self.set_flag(use, False)
154
155 # Set our defaults
156 for use, value in USE_DEFAULTS.items():
157 self.set_flag(use, value)
158
159 # Set use flags - value is set to True unless the use starts with -
160 # All given use flags always override the defaults
161 for use in use_flags:
162 value = not use.startswith('-')
163 self.set_flag(use, value)
164
165 def set_flag(self, key, value=True):
166 setattr(self, key, value)
167 self.flags[key] = value
168
169
170class HostBuild():
171
172 def __init__(self, args):
173 """ Construct the builder.
174
175 Args:
176 args: Parsed arguments from ArgumentParser
177 """
178 self.args = args
179
180 # Set jobs to number of cpus unless explicitly set
181 self.jobs = self.args.jobs
182 if not self.jobs:
183 self.jobs = multiprocessing.cpu_count()
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000184 print("Number of jobs = {}".format(self.jobs))
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000185
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700186 # Normalize bootstrap dir and make sure it exists
187 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir)
188 os.makedirs(self.bootstrap_dir, exist_ok=True)
189
190 # Output and platform directories are based on bootstrap
191 self.output_dir = os.path.join(self.bootstrap_dir, 'output')
192 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000193 self.sysroot = self.args.sysroot
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000194 self.libdir = self.args.libdir
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800195 self.install_dir = os.path.join(self.output_dir, 'install')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000196
197 # If default target isn't set, build everything
198 self.target = 'all'
199 if hasattr(self.args, 'target') and self.args.target:
200 self.target = self.args.target
201
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000202 target_use = self.args.use if self.args.use else []
203
204 # Unless set, always build test code
205 if not self.args.notest:
206 target_use.append('test')
207
208 self.use = UseFlags(target_use)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000209
210 # Validate platform directory
211 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist'
212 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root'
213
214 # Make sure output directory exists (or create it)
215 os.makedirs(self.output_dir, exist_ok=True)
216
217 # Set some default attributes
218 self.libbase_ver = None
219
220 self.configure_environ()
221
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000222 def _generate_rustflags(self):
223 """ Rustflags to include for the build.
224 """
225 rust_flags = [
226 '-L',
Martin Brabham1c24fda2021-09-16 11:19:46 -0700227 '{}/out/Default'.format(self.output_dir),
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000228 '-C',
229 'link-arg=-Wl,--allow-multiple-definition',
Sonny Sasaka87bacb62022-04-29 10:34:29 -0700230 # exclude uninteresting warnings
231 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints',
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000232 ]
233
234 return ' '.join(rust_flags)
235
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000236 def configure_environ(self):
237 """ Configure environment variables for GN and Cargo.
238 """
239 self.env = os.environ.copy()
240
241 # Make sure cargo home dir exists and has a bin directory
242 cargo_home = os.path.join(self.output_dir, 'cargo_home')
243 os.makedirs(cargo_home, exist_ok=True)
244 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True)
245
246 # Configure Rust env variables
247 self.env['CARGO_TARGET_DIR'] = self.output_dir
248 self.env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home')
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000249 self.env['RUSTFLAGS'] = self._generate_rustflags()
Abhishek Pandit-Subedi1927afa2021-04-28 21:16:18 -0700250 self.env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt')
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700251 self.env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api')
Sonny Sasakac1bc8e12022-06-14 17:52:50 -0700252 self.env['CXX_OUTDIR'] = self._gn_default_output()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000253
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000254 def run_command(self, target, args, cwd=None, env=None):
255 """ Run command and stream the output.
256 """
257 # Set some defaults
258 if not cwd:
259 cwd = self.platform_dir
260 if not env:
261 env = self.env
262
263 log_file = os.path.join(self.output_dir, '{}.log'.format(target))
264 with open(log_file, 'wb') as lf:
265 rc = 0
266 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE)
267 while True:
268 line = process.stdout.readline()
269 print(line.decode('utf-8'), end="")
270 lf.write(line)
271 if not line:
272 rc = process.poll()
273 if rc is not None:
274 break
275
276 time.sleep(0.1)
277
278 if rc != 0:
279 raise Exception("Return code is {}".format(rc))
280
281 def _get_basever(self):
282 if self.libbase_ver:
283 return self.libbase_ver
284
285 self.libbase_ver = os.environ.get('BASE_VER', '')
286 if not self.libbase_ver:
287 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER')
288 try:
289 with open(base_file, 'r') as f:
290 self.libbase_ver = f.read().strip('\n')
291 except:
292 self.libbase_ver = 'NOT-INSTALLED'
293
294 return self.libbase_ver
295
296 def _gn_default_output(self):
297 return os.path.join(self.output_dir, 'out/Default')
298
299 def _gn_configure(self):
300 """ Configure all required parameters for platform2.
301
302 Mostly copied from //common-mk/platform2.py
303 """
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700304 clang = not self.args.no_clang
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000305
306 def to_gn_string(s):
307 return '"%s"' % s.replace('"', '\\"')
308
309 def to_gn_list(strs):
310 return '[%s]' % ','.join([to_gn_string(s) for s in strs])
311
312 def to_gn_args_args(gn_args):
313 for k, v in gn_args.items():
314 if isinstance(v, bool):
315 v = str(v).lower()
316 elif isinstance(v, list):
317 v = to_gn_list(v)
318 elif isinstance(v, six.string_types):
319 v = to_gn_string(v)
320 else:
321 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
322 yield '%s=%s' % (k.replace('-', '_'), v)
323
324 gn_args = {
325 'platform_subdir': 'bt',
326 'cc': 'clang' if clang else 'gcc',
327 'cxx': 'clang++' if clang else 'g++',
328 'ar': 'llvm-ar' if clang else 'ar',
329 'pkg-config': 'pkg-config',
330 'clang_cc': clang,
331 'clang_cxx': clang,
332 'OS': 'linux',
333 'sysroot': self.sysroot,
334 'libdir': os.path.join(self.sysroot, self.libdir),
335 'build_root': self.output_dir,
336 'platform2_root': self.platform_dir,
337 'libbase_ver': self._get_basever(),
338 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
339 'external_cflags': [],
Abhishek Pandit-Subedi852dc3a2022-02-14 15:12:36 -0800340 'external_cxxflags': ["-DNDEBUG"],
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000341 'enable_werror': False,
342 }
343
344 if clang:
345 # Make sure to mark the clang use flag as true
346 self.use.set_flag('clang', True)
347 gn_args['external_cxxflags'] += ['-I/usr/include/']
348
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000349 gn_args_args = list(to_gn_args_args(gn_args))
350 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()]
351 gn_args_args += ['use={%s}' % (' '.join(use_args))]
352
353 gn_args = [
354 'gn',
355 'gen',
356 ]
357
358 if self.args.verbose:
359 gn_args.append('-v')
360
361 gn_args += [
362 '--root=%s' % self.platform_dir,
363 '--args=%s' % ' '.join(gn_args_args),
364 self._gn_default_output(),
365 ]
366
Sonny Sasaka706ec3b2021-03-25 05:39:20 -0700367 if 'PKG_CONFIG_PATH' in self.env:
368 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH'])
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000369
370 self.run_command('configure', gn_args)
371
372 def _gn_build(self, target):
373 """ Generate the ninja command for the target and run it.
374 """
375 args = ['%s:%s' % ('bt', target)]
376 ninja_args = ['ninja', '-C', self._gn_default_output()]
377 if self.jobs:
378 ninja_args += ['-j', str(self.jobs)]
379 ninja_args += args
380
381 if self.args.verbose:
382 ninja_args.append('-v')
383
384 self.run_command('build', ninja_args)
385
386 def _rust_configure(self):
387 """ Generate config file at cargo_home so we use vendored crates.
388 """
389 template = """
390 [source.systembt]
391 directory = "{}/external/rust/vendor"
392
393 [source.crates-io]
394 replace-with = "systembt"
395 local-registry = "/nonexistent"
396 """
Sonny Sasakac1335a22021-03-25 07:10:47 -0700397
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700398 if not self.args.no_vendored_rust:
Sonny Sasakac1335a22021-03-25 07:10:47 -0700399 contents = template.format(self.platform_dir)
400 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f:
401 f.write(contents)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000402
403 def _rust_build(self):
404 """ Run `cargo build` from platform2/bt directory.
405 """
406 self.run_command('rust', ['cargo', 'build'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
407
408 def _target_prepare(self):
409 """ Target to prepare the output directory for building.
410
411 This runs gn gen to generate all rquired files and set up the Rust
412 config properly. This will be run
413 """
414 self._gn_configure()
415 self._rust_configure()
416
417 def _target_tools(self):
418 """ Build the tools target in an already prepared environment.
419 """
420 self._gn_build('tools')
421
422 # Also copy bluetooth_packetgen to CARGO_HOME so it's available
423 shutil.copy(
424 os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
425
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800426 def _target_docs(self):
427 """Build the Rust docs."""
428 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
429
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000430 def _target_rust(self):
431 """ Build rust artifacts in an already prepared environment.
432 """
433 self._rust_build()
434
435 def _target_main(self):
436 """ Build the main GN artifacts in an already prepared environment.
437 """
438 self._gn_build('all')
439
440 def _target_test(self):
441 """ Runs the host tests.
442 """
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000443 # Rust tests first
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800444 rust_test_cmd = ['cargo', 'test']
445 if self.args.test_name:
446 rust_test_cmd = rust_test_cmd + [self.args.test_name]
447
448 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env)
Abhishek Pandit-Subedid801b122021-04-10 00:41:07 +0000449
450 # Host tests second based on host test list
451 for t in HOST_TESTS:
452 self.run_command(
453 'test', [os.path.join(self.output_dir, 'out/Default', t)],
454 cwd=os.path.join(self.output_dir),
455 env=self.env)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000456
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800457 def _target_install(self):
458 """ Installs files required to run Floss to install directory.
459 """
460 # First make the install directory
461 prefix = self.install_dir
462 os.makedirs(prefix, exist_ok=True)
463
464 # Next save the cwd and change to install directory
465 last_cwd = os.getcwd()
466 os.chdir(prefix)
467
468 bindir = os.path.join(self.output_dir, 'debug')
469 srcdir = os.path.dirname(__file__)
470
471 install_map = [
472 {
473 'src': os.path.join(bindir, 'btadapterd'),
474 'dst': 'usr/libexec/bluetooth/btadapterd',
475 'strip': True
476 },
477 {
478 'src': os.path.join(bindir, 'btmanagerd'),
479 'dst': 'usr/libexec/bluetooth/btmanagerd',
480 'strip': True
481 },
482 {
483 'src': os.path.join(bindir, 'btclient'),
484 'dst': 'usr/local/bin/btclient',
485 'strip': True
486 },
487 ]
488
489 for v in install_map:
490 src, partial_dst, strip = (v['src'], v['dst'], v['strip'])
491 dst = os.path.join(prefix, partial_dst)
492
493 # Create dst directory first and copy file there
494 os.makedirs(os.path.dirname(dst), exist_ok=True)
495 print('Installing {}'.format(dst))
496 shutil.copy(src, dst)
497
498 # Binary should be marked for strip and no-strip option shouldn't be
499 # set. No-strip is useful while debugging.
500 if strip and not self.args.no_strip:
501 self.run_command('install', ['llvm-strip', dst])
502
503 # Put all files into a tar.gz for easier installation
504 tar_location = os.path.join(prefix, 'floss.tar.gz')
505 with tarfile.open(tar_location, 'w:gz') as tar:
506 for v in install_map:
507 tar.add(v['dst'])
508
509 print('Tarball created at {}'.format(tar_location))
510
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000511 def _target_clean(self):
512 """ Delete the output directory entirely.
513 """
514 shutil.rmtree(self.output_dir)
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800515
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700516 # Remove Cargo.lock that may have become generated
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800517 try:
518 os.remove(os.path.join(self.platform_dir, 'bt', 'Cargo.lock'))
519 except FileNotFoundError:
520 pass
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000521
522 def _target_all(self):
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800523 """ Build all common targets (skipping doc, test, and clean).
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000524 """
525 self._target_prepare()
526 self._target_tools()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000527 self._target_main()
Abhishek Pandit-Subedia7b57b72021-04-01 15:33:05 -0700528 self._target_rust()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000529
530 def build(self):
531 """ Builds according to self.target
532 """
533 print('Building target ', self.target)
534
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800535 # Validate that the target is valid
536 if self.target not in VALID_TARGETS:
537 print('Target {} is not valid. Must be in {}', self.target, VALID_TARGETS)
538 return
539
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000540 if self.target == 'prepare':
541 self._target_prepare()
542 elif self.target == 'tools':
543 self._target_tools()
544 elif self.target == 'rust':
545 self._target_rust()
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800546 elif self.target == 'docs':
547 self._target_docs()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000548 elif self.target == 'main':
549 self._target_main()
550 elif self.target == 'test':
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000551 self._target_test()
552 elif self.target == 'clean':
553 self._target_clean()
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800554 elif self.target == 'install':
555 self._target_install()
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000556 elif self.target == 'all':
557 self._target_all()
558
559
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700560class Bootstrap():
561
562 def __init__(self, base_dir, bt_dir):
563 """ Construct bootstrapper.
564
565 Args:
566 base_dir: Where to stage everything.
567 bt_dir: Where bluetooth source is kept (will be symlinked)
568 """
569 self.base_dir = os.path.abspath(base_dir)
570 self.bt_dir = os.path.abspath(bt_dir)
571
572 # Create base directory if it doesn't already exist
573 os.makedirs(self.base_dir, exist_ok=True)
574
575 if not os.path.isdir(self.bt_dir):
576 raise Exception('{} is not a valid directory'.format(self.bt_dir))
577
578 self.git_dir = os.path.join(self.base_dir, 'repos')
579 self.staging_dir = os.path.join(self.base_dir, 'staging')
580 self.output_dir = os.path.join(self.base_dir, 'output')
581 self.external_dir = os.path.join(self.base_dir, 'staging', 'external')
582
583 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete')
584
585 def _update_platform2(self):
586 """Updates repositories used for build."""
587 for repo in BOOTSTRAP_GIT_REPOS.keys():
588 cwd = os.path.join(self.git_dir, repo)
589 subprocess.check_call(['git', 'pull'], cwd=cwd)
590
591 def _setup_platform2(self):
592 """ Set up platform2.
593
594 This will check out all the git repos and symlink everything correctly.
595 """
596
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800597 # Create all directories we will need to use
598 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]:
599 os.makedirs(dirpath, exist_ok=True)
600
601 # If already set up, only update platform2
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700602 if os.path.isfile(self.dir_setup_complete):
603 print('{} already set-up. Updating instead.'.format(self.base_dir))
604 self._update_platform2()
Abhishek Pandit-Subedib987eb62021-11-17 17:27:31 -0800605 else:
606 # Check out all repos in git directory
607 for repo in BOOTSTRAP_GIT_REPOS.values():
608 subprocess.check_call(['git', 'clone', repo], cwd=self.git_dir)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700609
610 # Symlink things
611 symlinks = [
612 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')),
Sonny Sasakaae9f6522022-03-28 10:31:34 -0700613 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')),
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700614 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')),
615 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')),
616 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')),
617 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')),
618 ]
619
620 # Create symlinks
621 for pairs in symlinks:
622 (src, dst) = pairs
Martin Brabham247d80b2022-02-04 19:42:49 +0000623 try:
624 os.unlink(dst)
625 except Exception as e:
626 print(e)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700627 os.symlink(src, dst)
628
629 # Write to setup complete file so we don't repeat this step
630 with open(self.dir_setup_complete, 'w') as f:
631 f.write('Setup complete.')
632
633 def _pretty_print_install(self, install_cmd, packages, line_limit=80):
634 """ Pretty print an install command.
635
636 Args:
637 install_cmd: Prefixed install command.
638 packages: Enumerate packages and append them to install command.
639 line_limit: Number of characters per line.
640
641 Return:
642 Array of lines to join and print.
643 """
644 install = [install_cmd]
645 line = ' '
646 # Remainder needed = space + len(pkg) + space + \
647 # Assuming 80 character lines, that's 80 - 3 = 77
648 line_limit = line_limit - 3
649 for pkg in packages:
650 if len(line) + len(pkg) < line_limit:
651 line = '{}{} '.format(line, pkg)
652 else:
653 install.append(line)
654 line = ' {} '.format(pkg)
655
656 if len(line) > 0:
657 install.append(line)
658
659 return install
660
661 def _check_package_installed(self, package, cmd, predicate):
662 """Check that the given package is installed.
663
664 Args:
665 package: Check that this package is installed.
666 cmd: Command prefix to check if installed (package appended to end)
667 predicate: Function/lambda to check if package is installed based
668 on output. Takes string output and returns boolean.
669
670 Return:
671 True if package is installed.
672 """
673 try:
674 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT)
675 is_installed = predicate(output.decode('utf-8'))
676 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing'))
677
678 return is_installed
679 except Exception as e:
680 print(e)
681 return False
682
683 def _get_command_output(self, cmd):
684 """Runs the command and gets the output.
685
686 Args:
687 cmd: Command to run.
688
689 Return:
690 Tuple (Success, Output). Success represents if the command ran ok.
691 """
692 try:
693 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
694 return (True, output.decode('utf-8').split('\n'))
695 except Exception as e:
696 print(e)
697 return (False, "")
698
699 def _print_missing_packages(self):
700 """Print any missing packages found via apt.
701
702 This will find any missing packages necessary for build using apt and
703 print it out as an apt-get install printf.
704 """
705 print('Checking for any missing packages...')
706
707 (success, output) = self._get_command_output(APT_PKG_LIST)
708 if not success:
709 raise Exception("Could not query apt for packages.")
710
711 packages_installed = {}
712 for line in output:
713 if 'installed' in line:
714 split = line.split('/', 2)
715 packages_installed[split[0]] = True
716
717 need_packages = []
718 for pkg in REQUIRED_APT_PACKAGES:
719 if pkg not in packages_installed:
720 need_packages.append(pkg)
721
722 # No packages need to be installed
723 if len(need_packages) == 0:
724 print('+ All required packages are installed')
725 return
726
727 install = self._pretty_print_install('sudo apt-get install', need_packages)
728
729 # Print all lines so they can be run in cmdline
730 print('Missing system packages. Run the following command: ')
731 print(' \\\n'.join(install))
732
733 def _print_missing_rust_packages(self):
734 """Print any missing packages found via cargo.
735
736 This will find any missing packages necessary for build using cargo and
737 print it out as a cargo-install printf.
738 """
739 print('Checking for any missing cargo packages...')
740
741 (success, output) = self._get_command_output(CARGO_PKG_LIST)
742 if not success:
743 raise Exception("Could not query cargo for packages.")
744
745 packages_installed = {}
746 for line in output:
747 # Cargo installed packages have this format
748 # <crate name> <version>:
749 # <binary name>
750 # We only care about the crates themselves
751 if ':' not in line:
752 continue
753
754 split = line.split(' ', 2)
755 packages_installed[split[0]] = True
756
757 need_packages = []
758 for pkg in REQUIRED_CARGO_PACKAGES:
759 if pkg not in packages_installed:
760 need_packages.append(pkg)
761
762 # No packages to be installed
763 if len(need_packages) == 0:
764 print('+ All required cargo packages are installed')
765 return
766
767 install = self._pretty_print_install('cargo install', need_packages)
768 print('Missing cargo packages. Run the following command: ')
769 print(' \\\n'.join(install))
770
771 def bootstrap(self):
772 """ Bootstrap the Linux build."""
773 self._setup_platform2()
774 self._print_missing_packages()
775 self._print_missing_rust_packages()
776
777
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000778if __name__ == '__main__':
779 parser = argparse.ArgumentParser(description='Simple build for host.')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700780 parser.add_argument(
781 '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
782 parser.add_argument(
783 '--run-bootstrap',
784 help='Run bootstrap code to verify build env is ok to build.',
785 default=False,
786 action='store_true')
787 parser.add_argument('--no-clang', help='Use clang compiler.', default=False, action='store_true')
Abhishek Pandit-Subedia1a94d12021-11-18 13:14:04 -0800788 parser.add_argument(
789 '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000790 parser.add_argument('--use', help='Set a specific use flag.')
Abhishek Pandit-Subedi94764142022-02-07 17:18:00 -0800791 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
792 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000793 parser.add_argument('--target', help='Run specific build target')
794 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700795 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000796 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700797 parser.add_argument(
798 '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000799 parser.add_argument('--verbose', help='Verbose logs for build.')
Abhishek Pandit-Subedif52cf662021-03-02 22:33:25 +0000800 args = parser.parse_args()
Abhishek Pandit-Subedi0db77012021-09-20 17:54:26 -0700801
802 # Make sure we get absolute path + expanded path for bootstrap directory
803 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir))
804
805 if args.run_bootstrap:
806 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__))
807 bootstrap.bootstrap()
808 else:
809 build = HostBuild(args)
810 build.build()