blob: 2945dac80f9e16cc186f1c69218a0fa714bb61f5 [file] [log] [blame]
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3"""Create a new variant of an existing base board
4
5This program will call all of the scripts that create the various pieces
6of a new variant. For example to create a new variant of the hatch base
7board, the following scripts are called:
8
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07009* third_party/coreboot/util/mainboard/google/create_coreboot_variant.sh
Paul Fagerburg3b534f92019-11-07 15:05:22 -070010* platform/dev/contrib/variant/create_coreboot_config.sh
11* private-overlays/baseboard-hatch-private/sys-boot/
12 * coreboot-private-files-hatch/files/add_fitimage.sh
13 * coreboot-private-files-hatch/asset_generation/gen_fit_image.sh
14 * Outside the chroot, because it uses WINE to run the FIT tools
15* platform/dev/contrib/variant/create_initial_ec_image.sh
16* platform/dev/contrib/variant/add_variant_to_yaml.sh
17* private-overlays/overlay-hatch-private/chromeos-base/
18 * chromeos-config-bsp-hatch-private/add_variant.sh
19
20Once the scripts are done, the following repos have changes
21
22* third_party/coreboot
23* third_party/chromiumos-overlay
24* private-overlays/baseboard-hatch-private
25* platform/ec
26* private-overlays/overlay-hatch-private
27* overlays
28
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070029The program has support for multiple baseboards, so the repos, directories,
30and scripts above can change depending on what the baseboard is.
31
32Copyright 2020 The Chromium OS Authors. All rights reserved.
Paul Fagerburg3b534f92019-11-07 15:05:22 -070033Use of this source code is governed by a BSD-style license that can be
34found in the LICENSE file.
35"""
36
37from __future__ import print_function
38import argparse
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070039import importlib
Paul Fagerburg3b534f92019-11-07 15:05:22 -070040import logging
41import os
42import re
43import shutil
44import subprocess
45import sys
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070046import step_names
Paul Fagerburg3b534f92019-11-07 15:05:22 -070047import variant_status
48
49
50def main():
51 """Create a new variant of an existing base board
52
53 This program automates the creation of a new variant of an existing
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070054 base board by calling various scripts that copy the base board, modify
Paul Fagerburg3b534f92019-11-07 15:05:22 -070055 files for the new variant, stage commits, and upload to gerrit.
56
57 Note that one of the following is required:
58 * --continue
59 * --board=BOARD --variant=VARIANT [--bug=BUG]
60 """
61 board, variant, bug, continue_flag = get_args()
62
63 if not check_flags(board, variant, bug, continue_flag):
64 return False
65
66 status = get_status(board, variant, bug, continue_flag)
67 if status is None:
68 return False
69
70 status.load()
71
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070072 while status.step is not None:
Paul Fagerburg3b534f92019-11-07 15:05:22 -070073 status.save()
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070074 if not perform_step(status):
75 logging.debug('perform_step returned False; exiting ...')
Paul Fagerburg3b534f92019-11-07 15:05:22 -070076 return False
77
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070078 move_to_next_step(status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -070079
80 return True
81
82
83def get_args():
84 """Parse the command-line arguments
85
86 There doesn't appear to be a way to specify that --continue is
87 mutually exclusive with --board, --variant, and --bug. As a result,
88 all arguments are optional, and another function will apply the logic
89 to check if there is an illegal combination of arguments.
90
91 Returns a list of:
92 board Name of the base board
93 variant Name of the variant being created
94 bug Text for bug number, if any ('None' otherwise)
95 continue_flag Flag if --continue was specified
96 """
97 parser = argparse.ArgumentParser(
98 description=main.__doc__,
99 formatter_class=argparse.RawTextHelpFormatter)
100 parser.add_argument('--board', type=str, help='Name of the base board')
101 parser.add_argument(
102 '--variant', type=str, help='Name of the new variant to create')
103 parser.add_argument(
104 '--bug', type=str, help='Bug number to reference in commits')
105 parser.add_argument(
106 '--continue', action='store_true',
107 dest='continue_flag', help='Continue the process from where it paused')
108 parser.add_argument(
109 '--verbose', action='store_true',
110 dest='verbose_flag', help='Enable verbose output of progress')
111 args = parser.parse_args()
112
113 if args.verbose_flag:
114 logging.basicConfig(level=logging.DEBUG)
115 else:
116 logging.basicConfig(level=logging.INFO)
117
118 board = args.board
119 if board is not None:
120 board = board.lower()
121
122 variant = args.variant
123 if variant is not None:
124 variant = variant.lower()
125
126 bug = args.bug or 'None'
127
128 return (board, variant, bug, args.continue_flag)
129
130
131def check_flags(board, variant, bug, continue_flag):
132 """Check the flags to ensure no invalid combinations
133
134 We allow any of the following:
135 --continue
136 --board=board_name --variant=variant_name
137 --board=board_name --variant=variant_name --bug=bug_text
138
139 The argument parser does have the functionality to represent the
140 combination of --board and --variant as a single mutually-exclusive
141 argument, so we have to use this function to do the checking.
142
143 Params:
144 board Name of the base board
145 variant Name of the variant being created
146 bug Text for bug number, if any ('None' otherwise)
147 continue_flag Flag if --continue was specified
148
149 Returns:
150 True if the arguments are acceptable, False otherwise
151 """
152 if continue_flag:
153 if board is not None or variant is not None:
154 logging.error('--continue cannot have other options')
155 return False
156
157 if bug != 'None':
158 logging.error('--continue cannot have other options')
159 return False
160 else:
161 if board is None:
162 logging.error('--board must be specified')
163 return False
164
165 if variant is None:
166 logging.error('--variant must be specified')
167 return False
168
169 return True
170
171
172def file_exists(filepath, filename):
173 """Determine if a path and file exists
174
175 Params:
176 filepath Path where build outputs should be found, e.g.
177 /build/hatch/firmware
178 filename File that should exist in that path, e.g.
179 image-sushi.bin
180
181 Returns:
182 True if file exists in that path, False otherwise
183 """
184 fullname = os.path.join(filepath, filename)
185 return os.path.exists(fullname) and os.path.isfile(fullname)
186
187
188def get_status(board, variant, bug, continue_flag):
189 """Create the status file or get the previous status
190
191 This program can stop at several places as we have to wait for CLs
192 to work through CQ or be upstreamed into the chromiumos tree, so just
193 like a git cherry-pick, there is a --continue option to pick up where
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700194 you left off by reading a specially-named status file.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700195
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700196 If --continue is specified, the status file must exist.
197 If the status file exists, then --continue must be specified.
198 When --continue is specified, we read the status file and return
199 with the contents.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700200
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700201 If the status file does not exist, we will create the state file with
202 the board, variant, and (optional) bug details.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700203
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700204 To decouple the list of boards supported from this main program, we
205 try to import a module with the same name as the baseboard,
206 so --board=hatch means that we import hatch.py. If we can't import
207 the file, then we don't support that baseboard.
208
209 The board-specific module will set several variables, which we will
210 copy into the object that we return.
211
212 * step_list - list of steps (named in step_names.py) to run in sequence
213 to create the new variant of the baseboard
214 * fsp - package name for FSP. This may be empty, depending on the
215 processor on the baseboard
216 * fitimage_pkg - package name for the fitimage
217 * fitimage_dir - directory for fitimage; prepend '~/trunk/src/' in chroot,
218 prepend '~/chromiumos/src' outside the chroot
219 * workon_pkgs - list of packages to cros_workon
220 * emerge_cmd - the emerge command, e.g. 'emerge-hatch'
221 * emerge_pkgs - list of packages to emerge
222 * yaml_emerge_pkgs - list of packages to emerge just to build the yaml
223 * private_yaml_dir - directory for the private yaml file
224
225 Additionally, the following fields will be set:
226
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700227 * board - the name of the baseboard, e.g. 'hatch'
228 * variant - the name of the variant, e.g. 'sushi'
229 * bug - optional text for a bug ID, used in the git commit messages.
230 Could be 'None' (as text, not the python None), or something like
231 'b:12345' for buganizer, or 'chromium:12345'
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700232 * step - internal state tracking, what step of the variant creation
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700233 we are at.
234 * yaml_file - internal, just the name of the file where all this data
235 gets saved.
236
237 These data might come from the status file (because we read it), or
238 they might be the initial values after we created the file (because
239 it did not already exist).
240
241 Params:
242 board Name of the base board
243 variant Name of the variant being created
244 bug Text for bug number, if any ('None' otherwise)
245 continue_flag Flag if --continue was specified
246
247 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700248 variant_status object with all the data mentioned above
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700249 """
250 status = variant_status.variant_status()
251 if continue_flag:
252 if not status.yaml_file_exists():
253 logging.error(
254 'new_variant is not in progress; nothing to --continue')
255 return None
256 else:
257 if status.yaml_file_exists():
258 logging.error(
259 'new_variant already in progress; did you forget --continue?')
260 return None
261
262 status.board = board
263 status.variant = variant
264 status.bug = bug
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700265
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700266 # We're just starting out, so load the appropriate module and copy
267 # all the data from it.
268 try:
269 module = importlib.import_module(board)
270 except ImportError:
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700271 print('Unsupported baseboard "' + board + '"')
272 sys.exit(1)
273
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700274 # pylint: disable=bad-whitespace
275 # Allow extra spaces around = so that we can line things up nicely
276 status.emerge_cmd = module.emerge_cmd
277 status.emerge_pkgs = module.emerge_pkgs
278 status.fitimage_dir = module.fitimage_dir
279 status.fitimage_pkg = module.fitimage_pkg
280 status.fsp = module.fsp
281 status.private_yaml_dir = module.private_yaml_dir
282 status.step_list = module.step_list
283 status.workon_pkgs = module.workon_pkgs
284 status.yaml_emerge_pkgs = module.yaml_emerge_pkgs
285 # pylint: enable=bad-whitespace
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700286
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700287 # Start at the first entry in the step list
288 status.step = status.step_list[0]
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700289
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700290 status.save()
291
292 return status
293
294
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700295def perform_step(status):
296 """Call the appropriate function for the current step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700297
298 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700299 status variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700300
301 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700302 True if the step succeeded, False if it failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700303 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700304 # Function to call based on the step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700305 dispatch = {
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700306 step_names.CB_VARIANT: create_coreboot_variant,
307 step_names.CB_CONFIG: create_coreboot_config,
308 step_names.ADD_FIT: add_fitimage,
309 step_names.GEN_FIT: gen_fit_image_outside_chroot,
310 step_names.COMMIT_FIT: commit_fitimage,
311 step_names.EC_IMAGE: create_initial_ec_image,
312 step_names.EC_BUILDALL: ec_buildall,
313 step_names.ADD_YAML: add_variant_to_yaml,
314 step_names.BUILD_YAML: build_yaml,
315 step_names.EMERGE: emerge_all,
316 step_names.PUSH: push_coreboot,
317 step_names.UPLOAD: upload_CLs,
318 step_names.FIND: find_coreboot_upstream,
319 step_names.CQ_DEPEND: add_cq_depends,
320 step_names.CLEAN_UP: clean_up,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700321 }
322
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700323 if status.step not in dispatch:
324 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700325 sys.exit(1)
326
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700327 return dispatch[status.step](status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700328
329
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700330def move_to_next_step(status):
331 """Move to the next step in the list
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700332
333 Params:
334 status variant_status object tracking our board, variant, etc.
335 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700336 if status.step not in status.step_list:
337 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700338 sys.exit(1)
339
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700340 idx = status.step_list.index(status.step)
341 if idx == len(status.step_list)-1:
342 status.step = None
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700343 else:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700344 status.step = status.step_list[idx+1]
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700345
346
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700347def run_process(args, cwd=None, env=None, capture_output=False):
348 """Run a process, log debug messages, return text output of process
349
350 The capture_output parameter allows us to capture the output when we
351 care about it (and not sending it to the screen), or ignoring it when
352 we don't care, and letting the user see the output so they know that
353 the build is still running, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700354
355 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700356 args List of the command and its params
357 cwd If not None, cd to this directory before running
358 env Environment to use for execution; if needed, get
359 os.environ.copy() and add variables. If None, just
360 use the current environment
361 capture_output True if we should capture the stdout, false
362 if we just care about success or not.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700363
364 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700365 If capture_output == True, we return the text output from running
366 the subprocess as a list of lines, or None if the process failed.
367 If capture_output == False, we return a True if it successed, or
368 None if the process failed.
369
370 The caller can evaluate as a bool, because bool(None) == False, and
371 bool() of a non-empty list is True, or the caller can use the returned
372 text for further processing.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700373 """
374 logging.debug('Run %s', str(args))
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700375 try:
376 if capture_output:
377 output = subprocess.check_output(args, cwd=cwd, env=env,
378 stderr=subprocess.STDOUT)
379 else:
380 subprocess.run(args, cwd=cwd, env=env, check=True)
381 # Just something to decode so we don't get an empty list
382 output = b'True'
383
384 logging.debug('process returns 0')
385 # Convert from byte string to ASCII
386 decoded = output.decode('utf-8')
387 # Split into array of individual lines
388 lines = decoded.split('\n')
389 return lines
390 except subprocess.CalledProcessError as err:
391 logging.debug('process returns %s', str(err.returncode))
392 return None
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700393
394
395def cros_workon(status, action):
396 """Call cros_workon for all the 9999 ebuilds we'll be touching
397
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700398 Params:
399 status variant_status object tracking our board, variant, etc.
400 action 'start' or 'stop'
401
402 Returns:
403 True if the call to cros_workon was successful, False if failed
404 """
405
406 # Build up the command from all the packages in the list
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700407 workon_cmd = ['cros_workon', '--board=' + status.board, action] + status.workon_pkgs
408 return bool(run_process(workon_cmd))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700409
410
411def create_coreboot_variant(status):
412 """Create source files for a new variant of the base board in coreboot
413
414 This function calls create_coreboot_variant.sh to set up a new variant
415 of the base board.
416
417 Params:
418 status variant_status object tracking our board, variant, etc.
419
420 Returns:
421 True if everything succeeded, False if something failed
422 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700423 logging.info('Running step create_coreboot_variant')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700424 create_coreboot_variant_sh = os.path.join(
425 os.path.expanduser('~/trunk/src/third_party/coreboot'),
Paul Fagerburg7b516d72019-12-20 13:13:41 -0700426 'util/mainboard/google/create_coreboot_variant.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700427 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700428 [create_coreboot_variant_sh,
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700429 status.board,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700430 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700431 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700432
433
434def create_coreboot_config(status):
435 """Create a coreboot configuration for a new variant
436
437 This function calls create_coreboot_config.sh, which will make a copy
438 of coreboot.${BOARD} into coreboot.${VARIANT}.
439
440 Params:
441 status variant_status object tracking our board, variant, etc.
442
443 Returns:
444 True if the script and test build succeeded, False if something failed
445 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700446 logging.info('Running step create_coreboot_config')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700447 create_coreboot_config_sh = os.path.expanduser(
448 '~/trunk/src/platform/dev/contrib/variant/create_coreboot_config.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700449 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700450 [create_coreboot_config_sh,
451 status.board,
452 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700453 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700454
455
456def add_fitimage(status):
457 """Add the source files for a fitimage for the new variant
458
459 This function calls add_fitimage.sh to create a new XSL file for the
460 variant's fitimage, which can override settings from the base board's XSL.
461 When this is done, the user will have to build the fitimage by running
462 gen_fit_image.sh outside of the chroot (and outside of this program's
463 control) because gen_fit_image.sh uses WINE, which is not installed in
464 the chroot. (There is a linux version of FIT, but it requires Open GL,
465 which is also not installed in the chroot.)
466
467 Params:
468 status variant_status object tracking our board, variant, etc.
469
470 Returns:
471 True if the script succeeded, False otherwise
472 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700473 logging.info('Running step add_fitimage')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700474 add_fitimage_sh = os.path.expanduser(os.path.join(
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700475 '~/trunk/src', status.fitimage_dir, 'files/add_fitimage.sh'))
476 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700477 [add_fitimage_sh,
478 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700479 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700480
481
482def gen_fit_image_outside_chroot(status):
483 """Tell the user to run gen_fit_image.sh outside the chroot
484
485 As noted for add_Fitimage(), gen_fit_image.sh cannot run inside the
486 chroot. This function tells the user to run gen_fit_image.sh in
487 their normal environment, and then come back (--continue) when that
488 is done.
489
490 Params:
491 status variant_status object tracking our board, variant, etc.
492
493 Returns:
494 True
495 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700496 logging.info('Running step gen_fit_image_outside_chroot')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700497 fit_image_files = check_fit_image_files(status)
498 # If the list is empty, then `not` of the list is True, so the files
499 # we need are all present and we can continue.
500 if not fit_image_files:
501 return True
502
503 logging.info('The following files need to be generated:')
504 for filename in fit_image_files:
505 logging.info('* %s', filename)
506 logging.info('The fitimage sources are ready for gen_fit_image.sh to process.')
507 logging.info('gen_fit_image.sh cannot run inside the chroot. Please open a new terminal')
508 logging.info('window, change to the directory where gen_fit_image.sh is located, and run')
509 logging.info('./gen_fit_image.sh %s [location of FIT] -b', status.variant)
510 logging.info('Then re-start this program with --continue.')
511 logging.info('If your chroot is based in ~/chromiumos, then the folder you want is')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700512 logging.info('~/chromiumos/src/%s/asset_generation', status.fitimage_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700513 return False
514
515
516def check_fit_image_files(status):
517 """Check if the fitimage has been generated
518
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700519 This function is not called directly as a step, and so it doesn't need
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700520 to produce any error messages to the user (except with --verbose).
521 gen_fit_image_outside_chroot will call this function to see if the
522 fitimage files exist, and if not, then that function will print the
523 message about how the user needs to run gen_fit_image.sh outside the
524 chroot.
525
526 Params:
527 status variant_status object tracking our board, variant, etc.
528
529 Returns:
530 List of files that *DO NOT* exist and need to be created, [] if
531 all files are present.
532 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700533 outputs_dir = os.path.expanduser(os.path.join(
534 '~/trunk/src', status.fitimage_dir, 'asset_generation/outputs'))
535 logging.debug('outputs_dir = "%s"', outputs_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700536
537 files = []
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700538 if not file_exists(outputs_dir, 'fitimage-' + status.variant + '.bin'):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700539 files.append('fitimage-' + status.variant + '.bin')
540
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700541 if not file_exists(outputs_dir,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700542 'fitimage-' + status.variant + '-versions.txt'):
543 files.append('fitimage-' + status.variant + '-versions.txt')
544
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700545 if not file_exists(outputs_dir, 'fit.log'):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700546 files.append('fit.log')
547
548 return files
549
550
551def move_fitimage_file(fitimage_dir, filename):
552 """Move fitimage files from create-place to commit-place
553
554 commit_fitimage needs to move the fitimage files from the place where
555 they were created to a different directory in the tree. This utility
556 function handles joining paths and calling a file move function.
557
558 Params:
559 fitimage_dir Directory where the fitimage files are
560 filename Name of the file being moved
561
562 Returns:
563 True if the move succeeded, False if it failed
564 """
565 src_dir = os.path.join(fitimage_dir, 'asset_generation/outputs')
566 src = os.path.join(src_dir, filename)
567 dest_dir = os.path.join(fitimage_dir, 'files')
568 dest = os.path.join(dest_dir, filename)
569 # If src does not exist and dest does, the move is already done => success!
570 if not file_exists(src_dir, filename) and file_exists(dest_dir, filename):
571 logging.debug('move "%s", "%s" unnecessary because dest exists and'
572 ' src does not exist', src, dest)
573 return True
574
575 logging.debug('move "%s", "%s"', src, dest)
576 return shutil.move(src, dest)
577
578
579def commit_fitimage(status):
580 """Move the fitimage files and add them to a git commit
581
582 This function moves the fitimage binary and -versions files from
583 asset_generation/outputs to files/ and then adds those files and
584 fit.log to the existing git commit.
585
586 Params:
587 status variant_status object tracking our board, variant, etc.
588
589 Returns:
590 True if the copy, git add, and git commit --amend all succeeded.
591 False if something failed.
592 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700593 logging.info('Running step commit_fitimage')
594 fitimage_dir = os.path.expanduser(os.path.join('~/trunk/src', status.fitimage_dir))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700595 logging.debug('fitimage_dir = "%s"', fitimage_dir)
596
597 # The copy operation will check that the source file exists, so no
598 # need to check separately.
599 if not move_fitimage_file(fitimage_dir,
600 'fitimage-' + status.variant + '.bin'):
601 logging.error('Moving fitimage binary failed')
602 return False
603
604 if not move_fitimage_file(fitimage_dir,
605 'fitimage-' + status.variant + '-versions.txt'):
606 logging.error('Moving fitimage versions.txt failed')
607 return False
608
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700609 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700610 ['git', 'add',
611 'asset_generation/outputs/fit.log',
612 'files/fitimage-' + status.variant + '.bin',
613 'files/fitimage-' + status.variant + '-versions.txt'
614 ],
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700615 cwd=fitimage_dir)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700616 return False
617
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700618 return bool(run_process(['git', 'commit', '--amend', '--no-edit'],
619 cwd=fitimage_dir))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700620
621
622def create_initial_ec_image(status):
623 """Create an EC image for the variant as a clone of the base board
624
625 This function calls create_initial_ec_image.sh, which will clone the
626 base board to create the variant. The shell script will build the
627 EC code for the variant, but the repo upload hook insists that we
628 have done a `make buildall` before it will allow an upload, so this
629 function does the buildall.
630
631 Params:
632 status variant_status object tracking our board, variant, etc.
633
634 Returns:
635 True if the script and test build succeeded, False if something failed
636 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700637 logging.info('Running step create_initial_ec_image')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700638 create_initial_ec_image_sh = os.path.expanduser(
639 '~/trunk/src/platform/dev/contrib/variant/create_initial_ec_image.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700640 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700641 [create_initial_ec_image_sh,
642 status.board,
643 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700644 status.bug])):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700645 return False
646
647 # create_initial_ec_image.sh will build the ec.bin for this variant
648 # if successful.
649 ec = os.path.expanduser('~/trunk/src/platform/ec')
650 logging.debug('ec = "%s"', ec)
651 ec_bin = 'build/' + status.variant + '/ec.bin'
652 logging.debug('ec.bin = "%s"', ec_bin)
653
654 return file_exists(ec, ec_bin)
655
656
657def ec_buildall(status):
658 """Do a make buildall -j for the EC, which is required for repo upload
659
660 The upload hook checks to ensure that the entire EC codebase builds
661 without error, so we have to run make buildall -j before uploading.
662
663 Params:
664 status variant_status object tracking our board, variant, etc.
665
666 Returns:
667 True if the script and test build succeeded, False if something failed
668 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700669 logging.info('Running step ec_buildall')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700670 del status # unused parameter
671 ec = os.path.expanduser('~/trunk/src/platform/ec')
672 logging.debug('ec = "%s"', ec)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700673 return bool(run_process(['make', 'buildall', '-j'], cwd=ec))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700674
675
676def add_variant_to_yaml(status):
677 """Add the new variant to the public and private model.yaml files
678
679 This function calls add_variant_to_yaml.sh (the public yaml) and
680 add_variant.sh (the private yaml) to add the new variant to
681 the yaml files.
682
683 Params:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700684 status variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700685
686 Returns:
687 True if the scripts and build succeeded, False is something failed
688 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700689 logging.info('Running step add_variant_to_yaml')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700690 add_variant_to_yaml_sh = os.path.expanduser(
691 '~/trunk/src/platform/dev/contrib/variant/add_variant_to_yaml.sh')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700692 if not bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700693 [add_variant_to_yaml_sh,
694 status.board,
695 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700696 status.bug])):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700697 return False
698
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700699 add_variant_sh = os.path.expanduser(os.path.join(status.private_yaml_dir, 'add_variant.sh'))
700 return bool(run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700701 [add_variant_sh,
702 status.variant,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700703 status.bug]))
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700704
705
706def build_yaml(status):
707 """Build config files from the yaml files
708
709 This function builds the yaml files into the JSON and C code that
710 mosys and other tools use, then verifies that the new variant's name
711 shows up in all of the output files.
712
713 Params:
714 status variant_status object tracking our board, variant, etc.
715
716 Returns:
717 True if the scripts and build succeeded, False is something failed
718 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700719 logging.info('Running step build_yaml')
720 if not bool(run_process([status.emerge_cmd] + status.yaml_emerge_pkgs)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700721 return False
722
723 # Check generated files for occurences of the variant name.
724 # Each file should have at least one occurence, so use `grep -c` to
725 # count the occurrences of the variant name in each file.
726 # The results will be something like this:
727 # config.json:10
728 # yaml/config.c:6
729 # yaml/config.yaml:27
730 # yaml/model.yaml:6
731 # yaml/private-model.yaml:10
732 # If the variant name doesn't show up in the file, then the count
733 # will be 0, so we would see, e.g.
734 # config.json:0
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700735 # We gather the output from grep, then look for any of the strings
736 # ending in :0. If none of them match, then we're good, but if even
737 # one of them ends with :0 then there was a problem with generating
738 # the files from the yaml.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700739 chromeos_config = '/build/' + status.board + '/usr/share/chromeos-config'
740 logging.debug('chromeos_config = "%s"', chromeos_config)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700741 grep = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700742 ['grep',
743 '-c',
744 status.variant,
745 'config.json',
746 'yaml/config.c',
747 'yaml/config.yaml',
748 'yaml/model.yaml',
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700749 'yaml/private-model.yaml'], cwd=chromeos_config, capture_output=True)
750
751 if grep is None:
752 return False
753
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700754 return not bool([s for s in grep if re.search(r':0$', s)])
755
756
757def emerge_all(status):
758 """Build the coreboot BIOS and EC code for the new variant
759
760 Params:
761 status variant_status object tracking our board, variant, etc.
762
763 Returns:
764 True if the build succeeded, False if something failed
765 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700766 logging.info('Running step emerge_all')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700767 cros_workon(status, 'start')
768 environ = os.environ.copy()
769 environ['FW_NAME'] = status.variant
770 # Build up the command for emerge from all the packages in the list
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700771 emerge_cmd_and_params = [status.emerge_cmd] + status.emerge_pkgs
772 if not bool(run_process(emerge_cmd_and_params, env=environ)):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700773 return False
774
775 cros_workon(status, 'stop')
776 build_path = '/build/' + status.board + '/firmware'
777 logging.debug('build_path = "%s"', build_path)
778 if not file_exists(build_path, 'image-' + status.variant + '.bin'):
779 logging.error('emerge failed because image-%s.bin does not exist',
780 status.variant)
781 return False
782
783 if not file_exists(build_path, 'image-' + status.variant + '.dev.bin'):
784 logging.error('emerge failed because image-%s.dev.bin does not exist',
785 status.variant)
786 return False
787
788 if not file_exists(build_path, 'image-' + status.variant + '.net.bin'):
789 logging.error('emerge failed because image-%s.net.bin does not exist',
790 status.variant)
791 return False
792
793 if not file_exists(build_path, 'image-' + status.variant + '.serial.bin'):
794 logging.error('emerge failed because image-%s.serial.bin does not exist',
795 status.variant)
796 return False
797
798 return True
799
800
801def push_coreboot(status):
802 """Push the coreboot CL to coreboot.org
803
804 Params:
805 status variant_status object tracking our board, variant, etc.
806
807 Returns:
808 True if the build succeeded, False if something failed
809 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700810 logging.info('Running step push_coreboot')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700811 del status # unused parameter
812 logging.error('TODO (pfagerburg): implement push_coreboot')
813 return True
814
815
816def upload_CLs(status):
817 """Upload all CLs to chromiumos
818
819 Params:
820 status variant_status object tracking our board, variant, etc.
821
822 Returns:
823 True if the build succeeded, False if something failed
824 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700825 logging.info('Running step upload_CLs')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700826 del status # unused parameter
827 logging.error('TODO (pfagerburg): implement upload_CLs')
828 return True
829
830
831def find_coreboot_upstream(status):
832 """Find the coreboot CL after it has been upstreamed to chromiumos
833
834 Params:
835 status variant_status object tracking our board, variant, etc.
836
837 Returns:
838 True if the build succeeded, False if something failed
839 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700840 logging.info('Running step find_coreboot_upstream')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700841 del status # unused parameter
842 logging.error('TODO (pfagerburg): implement find_coreboot_upstream')
843 return True
844
845
846def add_cq_depends(status):
847 """Add Cq-Depends to all of the CLs in chromiumos
848
849 The CL in coreboot needs to be pushed to coreboot.org, get merged,
850 and then get upstreamed into the chromiumos tree before the other
851 CLs can cq-depend on it and pass CQ.
852
853 Params:
854 status variant_status object tracking our board, variant, etc.
855
856 Returns:
857 True if the build succeeded, False if something failed
858 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700859 logging.info('Running step add_cq_depends')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700860 del status # unused parameter
861 logging.error('TODO (pfagerburg): implement add_cq_depends')
862 return True
863
864
865def clean_up(status):
866 """Final clean-up, including delete the status file
867
868 Params:
869 status variant_status object tracking our board, variant, etc.
870
871 Returns:
872 True
873 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700874 logging.info('Running step clean_up')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700875 status.rm()
876 return True
877
878
879if __name__ == '__main__':
880 sys.exit(not int(main()))