blob: 8b63c3655e799a3aef2724190fc0c6a87ec702d7 [file] [log] [blame]
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
Paul Fagerburge868e832020-01-22 17:14:04 -07003"""Create a new variant of an existing reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -07004
5This program will call all of the scripts that create the various pieces
6of a new variant. For example to create a new variant of the hatch base
7board, the following scripts are called:
8
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07009* third_party/coreboot/util/mainboard/google/create_coreboot_variant.sh
Paul Fagerburg3b534f92019-11-07 15:05:22 -070010* platform/dev/contrib/variant/create_coreboot_config.sh
11* private-overlays/baseboard-hatch-private/sys-boot/
12 * coreboot-private-files-hatch/files/add_fitimage.sh
13 * coreboot-private-files-hatch/asset_generation/gen_fit_image.sh
14 * Outside the chroot, because it uses WINE to run the FIT tools
15* platform/dev/contrib/variant/create_initial_ec_image.sh
16* platform/dev/contrib/variant/add_variant_to_yaml.sh
17* private-overlays/overlay-hatch-private/chromeos-base/
18 * chromeos-config-bsp-hatch-private/add_variant.sh
19
20Once the scripts are done, the following repos have changes
21
22* third_party/coreboot
23* third_party/chromiumos-overlay
24* private-overlays/baseboard-hatch-private
25* platform/ec
26* private-overlays/overlay-hatch-private
27* overlays
28
Paul Fagerburge868e832020-01-22 17:14:04 -070029The program has support for multiple reference boards, so the repos, directories,
30and scripts above can change depending on what the reference board is.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070031
32Copyright 2020 The Chromium OS Authors. All rights reserved.
Paul Fagerburg3b534f92019-11-07 15:05:22 -070033Use of this source code is governed by a BSD-style license that can be
34found in the LICENSE file.
35"""
36
37from __future__ import print_function
38import argparse
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070039import importlib
Paul Fagerburg3b534f92019-11-07 15:05:22 -070040import logging
41import os
42import re
43import shutil
44import subprocess
45import sys
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070046import step_names
Paul Fagerburg3b534f92019-11-07 15:05:22 -070047import variant_status
48
49
50def main():
Paul Fagerburge868e832020-01-22 17:14:04 -070051 """Create a new variant of an existing reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -070052
53 This program automates the creation of a new variant of an existing
Paul Fagerburge868e832020-01-22 17:14:04 -070054 reference board by calling various scripts that copy the reference board,
55 modify files for the new variant, stage commits, and upload to gerrit.
Paul Fagerburg3b534f92019-11-07 15:05:22 -070056
57 Note that one of the following is required:
58 * --continue
59 * --board=BOARD --variant=VARIANT [--bug=BUG]
60 """
61 board, variant, bug, continue_flag = get_args()
62
63 if not check_flags(board, variant, bug, continue_flag):
64 return False
65
66 status = get_status(board, variant, bug, continue_flag)
67 if status is None:
68 return False
69
70 status.load()
71
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070072 while status.step is not None:
Paul Fagerburg3b534f92019-11-07 15:05:22 -070073 status.save()
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070074 if not perform_step(status):
75 logging.debug('perform_step returned False; exiting ...')
Paul Fagerburg3b534f92019-11-07 15:05:22 -070076 return False
77
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070078 move_to_next_step(status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -070079
80 return True
81
82
83def get_args():
84 """Parse the command-line arguments
85
86 There doesn't appear to be a way to specify that --continue is
87 mutually exclusive with --board, --variant, and --bug. As a result,
88 all arguments are optional, and another function will apply the logic
89 to check if there is an illegal combination of arguments.
90
91 Returns a list of:
Paul Fagerburge868e832020-01-22 17:14:04 -070092 board Name of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -070093 variant Name of the variant being created
94 bug Text for bug number, if any ('None' otherwise)
95 continue_flag Flag if --continue was specified
96 """
97 parser = argparse.ArgumentParser(
98 description=main.__doc__,
99 formatter_class=argparse.RawTextHelpFormatter)
Paul Fagerburge868e832020-01-22 17:14:04 -0700100 parser.add_argument('--board', type=str, help='Name of the reference board')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700101 parser.add_argument(
102 '--variant', type=str, help='Name of the new variant to create')
103 parser.add_argument(
104 '--bug', type=str, help='Bug number to reference in commits')
105 parser.add_argument(
106 '--continue', action='store_true',
107 dest='continue_flag', help='Continue the process from where it paused')
108 parser.add_argument(
109 '--verbose', action='store_true',
110 dest='verbose_flag', help='Enable verbose output of progress')
111 args = parser.parse_args()
112
113 if args.verbose_flag:
114 logging.basicConfig(level=logging.DEBUG)
115 else:
116 logging.basicConfig(level=logging.INFO)
117
118 board = args.board
119 if board is not None:
120 board = board.lower()
121
122 variant = args.variant
123 if variant is not None:
124 variant = variant.lower()
125
126 bug = args.bug or 'None'
127
128 return (board, variant, bug, args.continue_flag)
129
130
131def check_flags(board, variant, bug, continue_flag):
132 """Check the flags to ensure no invalid combinations
133
134 We allow any of the following:
135 --continue
136 --board=board_name --variant=variant_name
137 --board=board_name --variant=variant_name --bug=bug_text
138
139 The argument parser does have the functionality to represent the
140 combination of --board and --variant as a single mutually-exclusive
141 argument, so we have to use this function to do the checking.
142
143 Params:
Paul Fagerburge868e832020-01-22 17:14:04 -0700144 board Name of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700145 variant Name of the variant being created
146 bug Text for bug number, if any ('None' otherwise)
147 continue_flag Flag if --continue was specified
148
149 Returns:
150 True if the arguments are acceptable, False otherwise
151 """
152 if continue_flag:
153 if board is not None or variant is not None:
154 logging.error('--continue cannot have other options')
155 return False
156
157 if bug != 'None':
158 logging.error('--continue cannot have other options')
159 return False
160 else:
161 if board is None:
162 logging.error('--board must be specified')
163 return False
164
165 if variant is None:
166 logging.error('--variant must be specified')
167 return False
168
169 return True
170
171
172def file_exists(filepath, filename):
173 """Determine if a path and file exists
174
175 Params:
176 filepath Path where build outputs should be found, e.g.
177 /build/hatch/firmware
178 filename File that should exist in that path, e.g.
179 image-sushi.bin
180
181 Returns:
182 True if file exists in that path, False otherwise
183 """
184 fullname = os.path.join(filepath, filename)
185 return os.path.exists(fullname) and os.path.isfile(fullname)
186
187
188def get_status(board, variant, bug, continue_flag):
189 """Create the status file or get the previous status
190
191 This program can stop at several places as we have to wait for CLs
192 to work through CQ or be upstreamed into the chromiumos tree, so just
193 like a git cherry-pick, there is a --continue option to pick up where
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700194 you left off by reading a specially-named status file.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700195
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700196 If --continue is specified, the status file must exist.
197 If the status file exists, then --continue must be specified.
198 When --continue is specified, we read the status file and return
199 with the contents.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700200
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700201 If the status file does not exist, we will create the state file with
202 the board, variant, and (optional) bug details.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700203
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700204 To decouple the list of boards supported from this main program, we
Paul Fagerburge868e832020-01-22 17:14:04 -0700205 try to import a module with the same name as the reference board,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700206 so --board=hatch means that we import hatch.py. If we can't import
Paul Fagerburge868e832020-01-22 17:14:04 -0700207 the file, then we don't support that reference board.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700208
209 The board-specific module will set several variables, which we will
210 copy into the object that we return.
211
Paul Fagerburge868e832020-01-22 17:14:04 -0700212 * base - the name of the base board, such as Hatch, Volteer, or Zork.
213 This can be different from the reference board, e.g. the Trembyle
214 reference board in the Zork project.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700215 * step_list - list of steps (named in step_names.py) to run in sequence
Paul Fagerburge868e832020-01-22 17:14:04 -0700216 to create the new variant of the reference board
217 * fsp - package name for FSP. This may be None, depending on the
218 processor on the reference board
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700219 * fitimage_pkg - package name for the fitimage
220 * fitimage_dir - directory for fitimage; prepend '~/trunk/src/' in chroot,
221 prepend '~/chromiumos/src' outside the chroot
222 * workon_pkgs - list of packages to cros_workon
223 * emerge_cmd - the emerge command, e.g. 'emerge-hatch'
224 * emerge_pkgs - list of packages to emerge
225 * yaml_emerge_pkgs - list of packages to emerge just to build the yaml
226 * private_yaml_dir - directory for the private yaml file
227
228 Additionally, the following fields will be set:
229
Paul Fagerburge868e832020-01-22 17:14:04 -0700230 * board - the name of the reference board, e.g. 'hatch'
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700231 * variant - the name of the variant, e.g. 'sushi'
232 * bug - optional text for a bug ID, used in the git commit messages.
233 Could be 'None' (as text, not the python None), or something like
234 'b:12345' for buganizer, or 'chromium:12345'
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700235 * step - internal state tracking, what step of the variant creation
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700236 we are at.
237 * yaml_file - internal, just the name of the file where all this data
238 gets saved.
239
240 These data might come from the status file (because we read it), or
241 they might be the initial values after we created the file (because
242 it did not already exist).
243
244 Params:
Paul Fagerburge868e832020-01-22 17:14:04 -0700245 board Name of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700246 variant Name of the variant being created
247 bug Text for bug number, if any ('None' otherwise)
248 continue_flag Flag if --continue was specified
249
250 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700251 variant_status object with all the data mentioned above
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700252 """
253 status = variant_status.variant_status()
254 if continue_flag:
255 if not status.yaml_file_exists():
256 logging.error(
257 'new_variant is not in progress; nothing to --continue')
258 return None
259 else:
260 if status.yaml_file_exists():
261 logging.error(
262 'new_variant already in progress; did you forget --continue?')
263 return None
264
265 status.board = board
266 status.variant = variant
267 status.bug = bug
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700268
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700269 # We're just starting out, so load the appropriate module and copy
270 # all the data from it.
271 try:
272 module = importlib.import_module(board)
273 except ImportError:
Paul Fagerburge868e832020-01-22 17:14:04 -0700274 print('Unsupported board "' + board + '"')
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700275 sys.exit(1)
276
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700277 # pylint: disable=bad-whitespace
278 # Allow extra spaces around = so that we can line things up nicely
Paul Fagerburge868e832020-01-22 17:14:04 -0700279 status.base = module.base
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700280 status.emerge_cmd = module.emerge_cmd
281 status.emerge_pkgs = module.emerge_pkgs
282 status.fitimage_dir = module.fitimage_dir
283 status.fitimage_pkg = module.fitimage_pkg
284 status.fsp = module.fsp
285 status.private_yaml_dir = module.private_yaml_dir
286 status.step_list = module.step_list
287 status.workon_pkgs = module.workon_pkgs
288 status.yaml_emerge_pkgs = module.yaml_emerge_pkgs
289 # pylint: enable=bad-whitespace
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700290
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700291 # Start at the first entry in the step list
292 status.step = status.step_list[0]
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700293
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700294 status.save()
295
296 return status
297
298
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700299def perform_step(status):
300 """Call the appropriate function for the current step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700301
302 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700303 status variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700304
305 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700306 True if the step succeeded, False if it failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700307 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700308 # Function to call based on the step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700309 dispatch = {
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700310 step_names.CB_VARIANT: create_coreboot_variant,
311 step_names.CB_CONFIG: create_coreboot_config,
Paul Fagerburge868e832020-01-22 17:14:04 -0700312 step_names.COPY_CONFIG: copy_coreboot_config,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700313 step_names.ADD_FIT: add_fitimage,
314 step_names.GEN_FIT: gen_fit_image_outside_chroot,
315 step_names.COMMIT_FIT: commit_fitimage,
316 step_names.EC_IMAGE: create_initial_ec_image,
317 step_names.EC_BUILDALL: ec_buildall,
318 step_names.ADD_YAML: add_variant_to_yaml,
319 step_names.BUILD_YAML: build_yaml,
320 step_names.EMERGE: emerge_all,
321 step_names.PUSH: push_coreboot,
322 step_names.UPLOAD: upload_CLs,
323 step_names.FIND: find_coreboot_upstream,
324 step_names.CQ_DEPEND: add_cq_depends,
325 step_names.CLEAN_UP: clean_up,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700326 }
327
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700328 if status.step not in dispatch:
329 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700330 sys.exit(1)
331
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700332 return dispatch[status.step](status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700333
334
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700335def move_to_next_step(status):
336 """Move to the next step in the list
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700337
338 Params:
339 status variant_status object tracking our board, variant, etc.
340 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700341 if status.step not in status.step_list:
342 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700343 sys.exit(1)
344
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700345 idx = status.step_list.index(status.step)
346 if idx == len(status.step_list)-1:
347 status.step = None
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700348 else:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700349 status.step = status.step_list[idx+1]
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700350
351
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700352def run_process(args, cwd=None, env=None, capture_output=False):
353 """Run a process, log debug messages, return text output of process
354
355 The capture_output parameter allows us to capture the output when we
356 care about it (and not sending it to the screen), or ignoring it when
357 we don't care, and letting the user see the output so they know that
358 the build is still running, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700359
360 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700361 args List of the command and its params
362 cwd If not None, cd to this directory before running
363 env Environment to use for execution; if needed, get
364 os.environ.copy() and add variables. If None, just
365 use the current environment
366 capture_output True if we should capture the stdout, false
367 if we just care about success or not.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700368
369 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700370 If capture_output == True, we return the text output from running
371 the subprocess as a list of lines, or None if the process failed.
372 If capture_output == False, we return a True if it successed, or
373 None if the process failed.
374
375 The caller can evaluate as a bool, because bool(None) == False, and
376 bool() of a non-empty list is True, or the caller can use the returned
377 text for further processing.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700378 """
379 logging.debug('Run %s', str(args))
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700380 try:
381 if capture_output:
382 output = subprocess.check_output(args, cwd=cwd, env=env,
383 stderr=subprocess.STDOUT)
384 else:
385 subprocess.run(args, cwd=cwd, env=env, check=True)
386 # Just something to decode so we don't get an empty list
387 output = b'True'
388
389 logging.debug('process returns 0')
390 # Convert from byte string to ASCII
391 decoded = output.decode('utf-8')
392 # Split into array of individual lines
393 lines = decoded.split('\n')
394 return lines
395 except subprocess.CalledProcessError as err:
396 logging.debug('process returns %s', str(err.returncode))
397 return None
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700398
399
400def cros_workon(status, action):
401 """Call cros_workon for all the 9999 ebuilds we'll be touching
402
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700403 Params:
404 status variant_status object tracking our board, variant, etc.
405 action 'start' or 'stop'
406
407 Returns:
408 True if the call to cros_workon was successful, False if failed
409 """
410
411 # Build up the command from all the packages in the list
Paul Fagerburge868e832020-01-22 17:14:04 -0700412 workon_cmd = ['cros_workon', '--board=' + status.base, action] + status.workon_pkgs
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700413 return bool(run_process(workon_cmd))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700414
415
416def create_coreboot_variant(status):
Paul Fagerburge868e832020-01-22 17:14:04 -0700417 """Create source files for a new variant of the reference board in coreboot
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700418
419 This function calls create_coreboot_variant.sh to set up a new variant
Paul Fagerburge868e832020-01-22 17:14:04 -0700420 of the reference board.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700421
422 Params:
423 status variant_status object tracking our board, variant, etc.
424
425 Returns:
426 True if everything succeeded, False if something failed
427 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700428 logging.info('Running step create_coreboot_variant')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700429 create_coreboot_variant_sh = os.path.join(
430 os.path.expanduser('~/trunk/src/third_party/coreboot'),
Paul Fagerburg7b516d72019-12-20 13:13:41 -0700431 'util/mainboard/google/create_coreboot_variant.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700432 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700433 [create_coreboot_variant_sh,
Paul Fagerburge868e832020-01-22 17:14:04 -0700434 status.base,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700435 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700436 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700437
438
439def create_coreboot_config(status):
440 """Create a coreboot configuration for a new variant
441
442 This function calls create_coreboot_config.sh, which will make a copy
443 of coreboot.${BOARD} into coreboot.${VARIANT}.
444
445 Params:
446 status variant_status object tracking our board, variant, etc.
447
448 Returns:
449 True if the script and test build succeeded, False if something failed
450 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700451 logging.info('Running step create_coreboot_config')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700452 create_coreboot_config_sh = os.path.expanduser(
453 '~/trunk/src/platform/dev/contrib/variant/create_coreboot_config.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700454 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700455 [create_coreboot_config_sh,
Paul Fagerburge868e832020-01-22 17:14:04 -0700456 status.base,
457 status.board,
458 status.variant,
459 status.bug]))
460
461
462def copy_coreboot_config(status):
463 """Copy the coreboot configuration for a new variant
464
465 This is only necessary for the Zork baseboard right now.
466 This function calls copy_coreboot_config.sh, which will copy
467 coreboot.${VARIANT} from
468 third_party/chromiumos-overlay/sys-boot/coreboot/files/configs
469 to
470 src/overlays/overlay-${BASE}/sys-boot/coreboot-${BASE}/files/configs
471
472 Params:
473 status variant_status object tracking our board, variant, etc.
474
475 Returns:
476 True if the script and test build succeeded, False if something failed
477 """
478 logging.info('Running step copy_coreboot_config')
479 copy_coreboot_config_sh = os.path.expanduser(
480 '~/trunk/src/platform/dev/contrib/variant/copy_coreboot_config.sh')
481 return bool(run_process(
482 [copy_coreboot_config_sh,
483 status.base,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700484 status.board,
485 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700486 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700487
488
489def add_fitimage(status):
490 """Add the source files for a fitimage for the new variant
491
492 This function calls add_fitimage.sh to create a new XSL file for the
Paul Fagerburge868e832020-01-22 17:14:04 -0700493 variant's fitimage, which can override settings from the reference board's
494 XSL. When this is done, the user will have to build the fitimage by running
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700495 gen_fit_image.sh outside of the chroot (and outside of this program's
496 control) because gen_fit_image.sh uses WINE, which is not installed in
497 the chroot. (There is a linux version of FIT, but it requires Open GL,
498 which is also not installed in the chroot.)
499
500 Params:
501 status variant_status object tracking our board, variant, etc.
502
503 Returns:
504 True if the script succeeded, False otherwise
505 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700506 logging.info('Running step add_fitimage')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700507 add_fitimage_sh = os.path.expanduser(os.path.join(
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700508 '~/trunk/src', status.fitimage_dir, 'files/add_fitimage.sh'))
509 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700510 [add_fitimage_sh,
511 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700512 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700513
514
515def gen_fit_image_outside_chroot(status):
516 """Tell the user to run gen_fit_image.sh outside the chroot
517
518 As noted for add_Fitimage(), gen_fit_image.sh cannot run inside the
519 chroot. This function tells the user to run gen_fit_image.sh in
520 their normal environment, and then come back (--continue) when that
521 is done.
522
523 Params:
524 status variant_status object tracking our board, variant, etc.
525
526 Returns:
527 True
528 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700529 logging.info('Running step gen_fit_image_outside_chroot')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700530 fit_image_files = check_fit_image_files(status)
531 # If the list is empty, then `not` of the list is True, so the files
532 # we need are all present and we can continue.
533 if not fit_image_files:
534 return True
535
536 logging.info('The following files need to be generated:')
537 for filename in fit_image_files:
538 logging.info('* %s', filename)
539 logging.info('The fitimage sources are ready for gen_fit_image.sh to process.')
540 logging.info('gen_fit_image.sh cannot run inside the chroot. Please open a new terminal')
541 logging.info('window, change to the directory where gen_fit_image.sh is located, and run')
542 logging.info('./gen_fit_image.sh %s [location of FIT] -b', status.variant)
543 logging.info('Then re-start this program with --continue.')
544 logging.info('If your chroot is based in ~/chromiumos, then the folder you want is')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700545 logging.info('~/chromiumos/src/%s/asset_generation', status.fitimage_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700546 return False
547
548
549def check_fit_image_files(status):
550 """Check if the fitimage has been generated
551
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700552 This function is not called directly as a step, and so it doesn't need
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700553 to produce any error messages to the user (except with --verbose).
554 gen_fit_image_outside_chroot will call this function to see if the
555 fitimage files exist, and if not, then that function will print the
556 message about how the user needs to run gen_fit_image.sh outside the
557 chroot.
558
559 Params:
560 status variant_status object tracking our board, variant, etc.
561
562 Returns:
563 List of files that *DO NOT* exist and need to be created, [] if
564 all files are present.
565 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700566 outputs_dir = os.path.expanduser(os.path.join(
567 '~/trunk/src', status.fitimage_dir, 'asset_generation/outputs'))
568 logging.debug('outputs_dir = "%s"', outputs_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700569
570 files = []
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700571 if not file_exists(outputs_dir, 'fitimage-' + status.variant + '.bin'):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700572 files.append('fitimage-' + status.variant + '.bin')
573
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700574 if not file_exists(outputs_dir,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700575 'fitimage-' + status.variant + '-versions.txt'):
576 files.append('fitimage-' + status.variant + '-versions.txt')
577
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700578 if not file_exists(outputs_dir, 'fit.log'):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700579 files.append('fit.log')
580
581 return files
582
583
584def move_fitimage_file(fitimage_dir, filename):
585 """Move fitimage files from create-place to commit-place
586
587 commit_fitimage needs to move the fitimage files from the place where
588 they were created to a different directory in the tree. This utility
589 function handles joining paths and calling a file move function.
590
591 Params:
592 fitimage_dir Directory where the fitimage files are
593 filename Name of the file being moved
594
595 Returns:
596 True if the move succeeded, False if it failed
597 """
598 src_dir = os.path.join(fitimage_dir, 'asset_generation/outputs')
599 src = os.path.join(src_dir, filename)
600 dest_dir = os.path.join(fitimage_dir, 'files')
601 dest = os.path.join(dest_dir, filename)
602 # If src does not exist and dest does, the move is already done => success!
603 if not file_exists(src_dir, filename) and file_exists(dest_dir, filename):
604 logging.debug('move "%s", "%s" unnecessary because dest exists and'
605 ' src does not exist', src, dest)
606 return True
607
608 logging.debug('move "%s", "%s"', src, dest)
609 return shutil.move(src, dest)
610
611
612def commit_fitimage(status):
613 """Move the fitimage files and add them to a git commit
614
615 This function moves the fitimage binary and -versions files from
616 asset_generation/outputs to files/ and then adds those files and
617 fit.log to the existing git commit.
618
619 Params:
620 status variant_status object tracking our board, variant, etc.
621
622 Returns:
623 True if the copy, git add, and git commit --amend all succeeded.
624 False if something failed.
625 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700626 logging.info('Running step commit_fitimage')
627 fitimage_dir = os.path.expanduser(os.path.join('~/trunk/src', status.fitimage_dir))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700628 logging.debug('fitimage_dir = "%s"', fitimage_dir)
629
630 # The copy operation will check that the source file exists, so no
631 # need to check separately.
632 if not move_fitimage_file(fitimage_dir,
633 'fitimage-' + status.variant + '.bin'):
634 logging.error('Moving fitimage binary failed')
635 return False
636
637 if not move_fitimage_file(fitimage_dir,
638 'fitimage-' + status.variant + '-versions.txt'):
639 logging.error('Moving fitimage versions.txt failed')
640 return False
641
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700642 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700643 ['git', 'add',
644 'asset_generation/outputs/fit.log',
645 'files/fitimage-' + status.variant + '.bin',
646 'files/fitimage-' + status.variant + '-versions.txt'
647 ],
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700648 cwd=fitimage_dir)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700649 return False
650
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700651 return bool(run_process(['git', 'commit', '--amend', '--no-edit'],
652 cwd=fitimage_dir))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700653
654
655def create_initial_ec_image(status):
Paul Fagerburge868e832020-01-22 17:14:04 -0700656 """Create an EC image for the variant as a clone of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700657
658 This function calls create_initial_ec_image.sh, which will clone the
Paul Fagerburge868e832020-01-22 17:14:04 -0700659 reference board to create the variant. The shell script will build the
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700660 EC code for the variant, but the repo upload hook insists that we
661 have done a `make buildall` before it will allow an upload, so this
662 function does the buildall.
663
664 Params:
665 status variant_status object tracking our board, variant, etc.
666
667 Returns:
668 True if the script and test build succeeded, False if something failed
669 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700670 logging.info('Running step create_initial_ec_image')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700671 create_initial_ec_image_sh = os.path.expanduser(
672 '~/trunk/src/platform/dev/contrib/variant/create_initial_ec_image.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700673 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700674 [create_initial_ec_image_sh,
675 status.board,
676 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700677 status.bug])):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700678 return False
679
680 # create_initial_ec_image.sh will build the ec.bin for this variant
681 # if successful.
682 ec = os.path.expanduser('~/trunk/src/platform/ec')
683 logging.debug('ec = "%s"', ec)
684 ec_bin = 'build/' + status.variant + '/ec.bin'
685 logging.debug('ec.bin = "%s"', ec_bin)
686
687 return file_exists(ec, ec_bin)
688
689
690def ec_buildall(status):
691 """Do a make buildall -j for the EC, which is required for repo upload
692
693 The upload hook checks to ensure that the entire EC codebase builds
694 without error, so we have to run make buildall -j before uploading.
695
696 Params:
697 status variant_status object tracking our board, variant, etc.
698
699 Returns:
700 True if the script and test build succeeded, False if something failed
701 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700702 logging.info('Running step ec_buildall')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700703 del status # unused parameter
704 ec = os.path.expanduser('~/trunk/src/platform/ec')
705 logging.debug('ec = "%s"', ec)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700706 return bool(run_process(['make', 'buildall', '-j'], cwd=ec))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700707
708
709def add_variant_to_yaml(status):
710 """Add the new variant to the public and private model.yaml files
711
712 This function calls add_variant_to_yaml.sh (the public yaml) and
713 add_variant.sh (the private yaml) to add the new variant to
714 the yaml files.
715
716 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700717 status variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700718
719 Returns:
720 True if the scripts and build succeeded, False is something failed
721 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700722 logging.info('Running step add_variant_to_yaml')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700723 add_variant_to_yaml_sh = os.path.expanduser(
724 '~/trunk/src/platform/dev/contrib/variant/add_variant_to_yaml.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700725 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700726 [add_variant_to_yaml_sh,
Paul Fagerburge868e832020-01-22 17:14:04 -0700727 status.base,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700728 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700729 status.bug])):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700730 return False
731
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700732 add_variant_sh = os.path.expanduser(os.path.join(status.private_yaml_dir, 'add_variant.sh'))
733 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700734 [add_variant_sh,
735 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700736 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700737
738
739def build_yaml(status):
740 """Build config files from the yaml files
741
742 This function builds the yaml files into the JSON and C code that
743 mosys and other tools use, then verifies that the new variant's name
744 shows up in all of the output files.
745
746 Params:
747 status variant_status object tracking our board, variant, etc.
748
749 Returns:
750 True if the scripts and build succeeded, False is something failed
751 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700752 logging.info('Running step build_yaml')
753 if not bool(run_process([status.emerge_cmd] + status.yaml_emerge_pkgs)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700754 return False
755
756 # Check generated files for occurences of the variant name.
757 # Each file should have at least one occurence, so use `grep -c` to
758 # count the occurrences of the variant name in each file.
759 # The results will be something like this:
760 # config.json:10
761 # yaml/config.c:6
762 # yaml/config.yaml:27
763 # yaml/model.yaml:6
764 # yaml/private-model.yaml:10
765 # If the variant name doesn't show up in the file, then the count
766 # will be 0, so we would see, e.g.
767 # config.json:0
Paul Fagerburge868e832020-01-22 17:14:04 -0700768 # Note that we leave out yaml/model.yaml (the public one) because for
769 # some boards, there is nothing in the public yaml file.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700770 # We gather the output from grep, then look for any of the strings
771 # ending in :0. If none of them match, then we're good, but if even
772 # one of them ends with :0 then there was a problem with generating
773 # the files from the yaml.
Paul Fagerburge868e832020-01-22 17:14:04 -0700774 chromeos_config = '/build/' + status.base + '/usr/share/chromeos-config'
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700775 logging.debug('chromeos_config = "%s"', chromeos_config)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700776 grep = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700777 ['grep',
Paul Fagerburge868e832020-01-22 17:14:04 -0700778 '-ci',
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700779 status.variant,
780 'config.json',
781 'yaml/config.c',
782 'yaml/config.yaml',
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700783 'yaml/private-model.yaml'], cwd=chromeos_config, capture_output=True)
784
785 if grep is None:
786 return False
787
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700788 return not bool([s for s in grep if re.search(r':0$', s)])
789
790
791def emerge_all(status):
792 """Build the coreboot BIOS and EC code for the new variant
793
794 Params:
795 status variant_status object tracking our board, variant, etc.
796
797 Returns:
798 True if the build succeeded, False if something failed
799 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700800 logging.info('Running step emerge_all')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700801 cros_workon(status, 'start')
802 environ = os.environ.copy()
803 environ['FW_NAME'] = status.variant
804 # Build up the command for emerge from all the packages in the list
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700805 emerge_cmd_and_params = [status.emerge_cmd] + status.emerge_pkgs
806 if not bool(run_process(emerge_cmd_and_params, env=environ)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700807 return False
808
809 cros_workon(status, 'stop')
Paul Fagerburge868e832020-01-22 17:14:04 -0700810 build_path = '/build/' + status.base + '/firmware'
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700811 logging.debug('build_path = "%s"', build_path)
812 if not file_exists(build_path, 'image-' + status.variant + '.bin'):
813 logging.error('emerge failed because image-%s.bin does not exist',
814 status.variant)
815 return False
816
817 if not file_exists(build_path, 'image-' + status.variant + '.dev.bin'):
818 logging.error('emerge failed because image-%s.dev.bin does not exist',
819 status.variant)
820 return False
821
822 if not file_exists(build_path, 'image-' + status.variant + '.net.bin'):
823 logging.error('emerge failed because image-%s.net.bin does not exist',
824 status.variant)
825 return False
826
827 if not file_exists(build_path, 'image-' + status.variant + '.serial.bin'):
828 logging.error('emerge failed because image-%s.serial.bin does not exist',
829 status.variant)
830 return False
831
832 return True
833
834
835def push_coreboot(status):
836 """Push the coreboot CL to coreboot.org
837
838 Params:
839 status variant_status object tracking our board, variant, etc.
840
841 Returns:
842 True if the build succeeded, False if something failed
843 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700844 logging.info('Running step push_coreboot')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700845 del status # unused parameter
846 logging.error('TODO (pfagerburg): implement push_coreboot')
847 return True
848
849
850def upload_CLs(status):
851 """Upload all CLs to chromiumos
852
853 Params:
854 status variant_status object tracking our board, variant, etc.
855
856 Returns:
857 True if the build succeeded, False if something failed
858 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700859 logging.info('Running step upload_CLs')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700860 del status # unused parameter
861 logging.error('TODO (pfagerburg): implement upload_CLs')
862 return True
863
864
865def find_coreboot_upstream(status):
866 """Find the coreboot CL after it has been upstreamed to chromiumos
867
868 Params:
869 status variant_status object tracking our board, variant, etc.
870
871 Returns:
872 True if the build succeeded, False if something failed
873 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700874 logging.info('Running step find_coreboot_upstream')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700875 del status # unused parameter
876 logging.error('TODO (pfagerburg): implement find_coreboot_upstream')
877 return True
878
879
880def add_cq_depends(status):
881 """Add Cq-Depends to all of the CLs in chromiumos
882
883 The CL in coreboot needs to be pushed to coreboot.org, get merged,
884 and then get upstreamed into the chromiumos tree before the other
885 CLs can cq-depend on it and pass CQ.
886
887 Params:
888 status variant_status object tracking our board, variant, etc.
889
890 Returns:
891 True if the build succeeded, False if something failed
892 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700893 logging.info('Running step add_cq_depends')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700894 del status # unused parameter
895 logging.error('TODO (pfagerburg): implement add_cq_depends')
896 return True
897
898
899def clean_up(status):
900 """Final clean-up, including delete the status file
901
902 Params:
903 status variant_status object tracking our board, variant, etc.
904
905 Returns:
906 True
907 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700908 logging.info('Running step clean_up')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700909 status.rm()
910 return True
911
912
913if __name__ == '__main__':
914 sys.exit(not int(main()))