Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*- |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 3 | """Create a new variant of an existing reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 4 | |
| 5 | This program will call all of the scripts that create the various pieces |
| 6 | of a new variant. For example to create a new variant of the hatch base |
| 7 | board, the following scripts are called: |
| 8 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 9 | * third_party/coreboot/util/mainboard/google/create_coreboot_variant.sh |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 10 | * platform/dev/contrib/variant/create_coreboot_config.sh |
| 11 | * private-overlays/baseboard-hatch-private/sys-boot/ |
| 12 | * coreboot-private-files-hatch/files/add_fitimage.sh |
| 13 | * coreboot-private-files-hatch/asset_generation/gen_fit_image.sh |
| 14 | * Outside the chroot, because it uses WINE to run the FIT tools |
| 15 | * platform/dev/contrib/variant/create_initial_ec_image.sh |
| 16 | * platform/dev/contrib/variant/add_variant_to_yaml.sh |
| 17 | * private-overlays/overlay-hatch-private/chromeos-base/ |
| 18 | * chromeos-config-bsp-hatch-private/add_variant.sh |
| 19 | |
| 20 | Once the scripts are done, the following repos have changes |
| 21 | |
| 22 | * third_party/coreboot |
| 23 | * third_party/chromiumos-overlay |
| 24 | * private-overlays/baseboard-hatch-private |
| 25 | * platform/ec |
| 26 | * private-overlays/overlay-hatch-private |
| 27 | * overlays |
| 28 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 29 | The program has support for multiple reference boards, so the repos, directories, |
| 30 | and scripts above can change depending on what the reference board is. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 31 | |
| 32 | Copyright 2020 The Chromium OS Authors. All rights reserved. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 33 | Use of this source code is governed by a BSD-style license that can be |
| 34 | found in the LICENSE file. |
| 35 | """ |
| 36 | |
| 37 | from __future__ import print_function |
| 38 | import argparse |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 39 | import importlib |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 40 | import logging |
| 41 | import os |
| 42 | import re |
| 43 | import shutil |
| 44 | import subprocess |
| 45 | import sys |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 46 | import step_names |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 47 | import variant_status |
| 48 | |
| 49 | |
| 50 | def main(): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 51 | """Create a new variant of an existing reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 52 | |
| 53 | This program automates the creation of a new variant of an existing |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 54 | reference board by calling various scripts that copy the reference board, |
| 55 | modify files for the new variant, stage commits, and upload to gerrit. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 56 | |
| 57 | Note that one of the following is required: |
| 58 | * --continue |
| 59 | * --board=BOARD --variant=VARIANT [--bug=BUG] |
| 60 | """ |
| 61 | board, variant, bug, continue_flag = get_args() |
| 62 | |
| 63 | if not check_flags(board, variant, bug, continue_flag): |
| 64 | return False |
| 65 | |
| 66 | status = get_status(board, variant, bug, continue_flag) |
| 67 | if status is None: |
| 68 | return False |
| 69 | |
| 70 | status.load() |
| 71 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 72 | while status.step is not None: |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 73 | status.save() |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 74 | if not perform_step(status): |
| 75 | logging.debug('perform_step returned False; exiting ...') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 76 | return False |
| 77 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 78 | move_to_next_step(status) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 79 | |
| 80 | return True |
| 81 | |
| 82 | |
| 83 | def get_args(): |
| 84 | """Parse the command-line arguments |
| 85 | |
| 86 | There doesn't appear to be a way to specify that --continue is |
| 87 | mutually exclusive with --board, --variant, and --bug. As a result, |
| 88 | all arguments are optional, and another function will apply the logic |
| 89 | to check if there is an illegal combination of arguments. |
| 90 | |
| 91 | Returns a list of: |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 92 | board Name of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 93 | variant Name of the variant being created |
| 94 | bug Text for bug number, if any ('None' otherwise) |
| 95 | continue_flag Flag if --continue was specified |
| 96 | """ |
| 97 | parser = argparse.ArgumentParser( |
| 98 | description=main.__doc__, |
| 99 | formatter_class=argparse.RawTextHelpFormatter) |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 100 | parser.add_argument('--board', type=str, help='Name of the reference board') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 101 | parser.add_argument( |
| 102 | '--variant', type=str, help='Name of the new variant to create') |
| 103 | parser.add_argument( |
| 104 | '--bug', type=str, help='Bug number to reference in commits') |
| 105 | parser.add_argument( |
| 106 | '--continue', action='store_true', |
| 107 | dest='continue_flag', help='Continue the process from where it paused') |
| 108 | parser.add_argument( |
| 109 | '--verbose', action='store_true', |
| 110 | dest='verbose_flag', help='Enable verbose output of progress') |
| 111 | args = parser.parse_args() |
| 112 | |
| 113 | if args.verbose_flag: |
| 114 | logging.basicConfig(level=logging.DEBUG) |
| 115 | else: |
| 116 | logging.basicConfig(level=logging.INFO) |
| 117 | |
| 118 | board = args.board |
| 119 | if board is not None: |
| 120 | board = board.lower() |
| 121 | |
| 122 | variant = args.variant |
| 123 | if variant is not None: |
| 124 | variant = variant.lower() |
| 125 | |
| 126 | bug = args.bug or 'None' |
| 127 | |
| 128 | return (board, variant, bug, args.continue_flag) |
| 129 | |
| 130 | |
| 131 | def check_flags(board, variant, bug, continue_flag): |
| 132 | """Check the flags to ensure no invalid combinations |
| 133 | |
| 134 | We allow any of the following: |
| 135 | --continue |
| 136 | --board=board_name --variant=variant_name |
| 137 | --board=board_name --variant=variant_name --bug=bug_text |
| 138 | |
| 139 | The argument parser does have the functionality to represent the |
| 140 | combination of --board and --variant as a single mutually-exclusive |
| 141 | argument, so we have to use this function to do the checking. |
| 142 | |
| 143 | Params: |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 144 | board Name of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 145 | variant Name of the variant being created |
| 146 | bug Text for bug number, if any ('None' otherwise) |
| 147 | continue_flag Flag if --continue was specified |
| 148 | |
| 149 | Returns: |
| 150 | True if the arguments are acceptable, False otherwise |
| 151 | """ |
| 152 | if continue_flag: |
| 153 | if board is not None or variant is not None: |
| 154 | logging.error('--continue cannot have other options') |
| 155 | return False |
| 156 | |
| 157 | if bug != 'None': |
| 158 | logging.error('--continue cannot have other options') |
| 159 | return False |
| 160 | else: |
| 161 | if board is None: |
| 162 | logging.error('--board must be specified') |
| 163 | return False |
| 164 | |
| 165 | if variant is None: |
| 166 | logging.error('--variant must be specified') |
| 167 | return False |
| 168 | |
| 169 | return True |
| 170 | |
| 171 | |
| 172 | def file_exists(filepath, filename): |
| 173 | """Determine if a path and file exists |
| 174 | |
| 175 | Params: |
| 176 | filepath Path where build outputs should be found, e.g. |
| 177 | /build/hatch/firmware |
| 178 | filename File that should exist in that path, e.g. |
| 179 | image-sushi.bin |
| 180 | |
| 181 | Returns: |
| 182 | True if file exists in that path, False otherwise |
| 183 | """ |
| 184 | fullname = os.path.join(filepath, filename) |
| 185 | return os.path.exists(fullname) and os.path.isfile(fullname) |
| 186 | |
| 187 | |
| 188 | def get_status(board, variant, bug, continue_flag): |
| 189 | """Create the status file or get the previous status |
| 190 | |
| 191 | This program can stop at several places as we have to wait for CLs |
| 192 | to work through CQ or be upstreamed into the chromiumos tree, so just |
| 193 | like a git cherry-pick, there is a --continue option to pick up where |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 194 | you left off by reading a specially-named status file. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 195 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 196 | If --continue is specified, the status file must exist. |
| 197 | If the status file exists, then --continue must be specified. |
| 198 | When --continue is specified, we read the status file and return |
| 199 | with the contents. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 200 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 201 | If the status file does not exist, we will create the state file with |
| 202 | the board, variant, and (optional) bug details. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 203 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 204 | To decouple the list of boards supported from this main program, we |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 205 | try to import a module with the same name as the reference board, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 206 | so --board=hatch means that we import hatch.py. If we can't import |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 207 | the file, then we don't support that reference board. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 208 | |
| 209 | The board-specific module will set several variables, which we will |
| 210 | copy into the object that we return. |
| 211 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 212 | * base - the name of the base board, such as Hatch, Volteer, or Zork. |
| 213 | This can be different from the reference board, e.g. the Trembyle |
| 214 | reference board in the Zork project. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 215 | * step_list - list of steps (named in step_names.py) to run in sequence |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 216 | to create the new variant of the reference board |
| 217 | * fsp - package name for FSP. This may be None, depending on the |
| 218 | processor on the reference board |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 219 | * fitimage_pkg - package name for the fitimage |
| 220 | * fitimage_dir - directory for fitimage; prepend '~/trunk/src/' in chroot, |
| 221 | prepend '~/chromiumos/src' outside the chroot |
| 222 | * workon_pkgs - list of packages to cros_workon |
| 223 | * emerge_cmd - the emerge command, e.g. 'emerge-hatch' |
| 224 | * emerge_pkgs - list of packages to emerge |
| 225 | * yaml_emerge_pkgs - list of packages to emerge just to build the yaml |
| 226 | * private_yaml_dir - directory for the private yaml file |
| 227 | |
| 228 | Additionally, the following fields will be set: |
| 229 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 230 | * board - the name of the reference board, e.g. 'hatch' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 231 | * variant - the name of the variant, e.g. 'sushi' |
| 232 | * bug - optional text for a bug ID, used in the git commit messages. |
| 233 | Could be 'None' (as text, not the python None), or something like |
| 234 | 'b:12345' for buganizer, or 'chromium:12345' |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 235 | * step - internal state tracking, what step of the variant creation |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 236 | we are at. |
| 237 | * yaml_file - internal, just the name of the file where all this data |
| 238 | gets saved. |
| 239 | |
| 240 | These data might come from the status file (because we read it), or |
| 241 | they might be the initial values after we created the file (because |
| 242 | it did not already exist). |
| 243 | |
| 244 | Params: |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 245 | board Name of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 246 | variant Name of the variant being created |
| 247 | bug Text for bug number, if any ('None' otherwise) |
| 248 | continue_flag Flag if --continue was specified |
| 249 | |
| 250 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 251 | variant_status object with all the data mentioned above |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 252 | """ |
| 253 | status = variant_status.variant_status() |
| 254 | if continue_flag: |
| 255 | if not status.yaml_file_exists(): |
| 256 | logging.error( |
| 257 | 'new_variant is not in progress; nothing to --continue') |
| 258 | return None |
| 259 | else: |
| 260 | if status.yaml_file_exists(): |
| 261 | logging.error( |
| 262 | 'new_variant already in progress; did you forget --continue?') |
| 263 | return None |
| 264 | |
| 265 | status.board = board |
| 266 | status.variant = variant |
| 267 | status.bug = bug |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 268 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 269 | # We're just starting out, so load the appropriate module and copy |
| 270 | # all the data from it. |
| 271 | try: |
| 272 | module = importlib.import_module(board) |
| 273 | except ImportError: |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 274 | print('Unsupported board "' + board + '"') |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 275 | sys.exit(1) |
| 276 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 277 | # pylint: disable=bad-whitespace |
| 278 | # Allow extra spaces around = so that we can line things up nicely |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 279 | status.base = module.base |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 280 | status.emerge_cmd = module.emerge_cmd |
| 281 | status.emerge_pkgs = module.emerge_pkgs |
| 282 | status.fitimage_dir = module.fitimage_dir |
| 283 | status.fitimage_pkg = module.fitimage_pkg |
| 284 | status.fsp = module.fsp |
| 285 | status.private_yaml_dir = module.private_yaml_dir |
| 286 | status.step_list = module.step_list |
| 287 | status.workon_pkgs = module.workon_pkgs |
| 288 | status.yaml_emerge_pkgs = module.yaml_emerge_pkgs |
| 289 | # pylint: enable=bad-whitespace |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 290 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 291 | # Start at the first entry in the step list |
| 292 | status.step = status.step_list[0] |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 293 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 294 | status.save() |
| 295 | |
| 296 | return status |
| 297 | |
| 298 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 299 | def perform_step(status): |
| 300 | """Call the appropriate function for the current step |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 301 | |
| 302 | Params: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 303 | status variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 304 | |
| 305 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 306 | True if the step succeeded, False if it failed |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 307 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 308 | # Function to call based on the step |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 309 | dispatch = { |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 310 | step_names.CB_VARIANT: create_coreboot_variant, |
| 311 | step_names.CB_CONFIG: create_coreboot_config, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 312 | step_names.COPY_CONFIG: copy_coreboot_config, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 313 | step_names.ADD_FIT: add_fitimage, |
| 314 | step_names.GEN_FIT: gen_fit_image_outside_chroot, |
| 315 | step_names.COMMIT_FIT: commit_fitimage, |
| 316 | step_names.EC_IMAGE: create_initial_ec_image, |
| 317 | step_names.EC_BUILDALL: ec_buildall, |
| 318 | step_names.ADD_YAML: add_variant_to_yaml, |
| 319 | step_names.BUILD_YAML: build_yaml, |
| 320 | step_names.EMERGE: emerge_all, |
| 321 | step_names.PUSH: push_coreboot, |
| 322 | step_names.UPLOAD: upload_CLs, |
| 323 | step_names.FIND: find_coreboot_upstream, |
| 324 | step_names.CQ_DEPEND: add_cq_depends, |
| 325 | step_names.CLEAN_UP: clean_up, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 326 | } |
| 327 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 328 | if status.step not in dispatch: |
| 329 | logging.error('Unknown step "%s", aborting...', status.step) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 330 | sys.exit(1) |
| 331 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 332 | return dispatch[status.step](status) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 333 | |
| 334 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 335 | def move_to_next_step(status): |
| 336 | """Move to the next step in the list |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 337 | |
| 338 | Params: |
| 339 | status variant_status object tracking our board, variant, etc. |
| 340 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 341 | if status.step not in status.step_list: |
| 342 | logging.error('Unknown step "%s", aborting...', status.step) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 343 | sys.exit(1) |
| 344 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 345 | idx = status.step_list.index(status.step) |
| 346 | if idx == len(status.step_list)-1: |
| 347 | status.step = None |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 348 | else: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 349 | status.step = status.step_list[idx+1] |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 350 | |
| 351 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 352 | def run_process(args, cwd=None, env=None, capture_output=False): |
| 353 | """Run a process, log debug messages, return text output of process |
| 354 | |
| 355 | The capture_output parameter allows us to capture the output when we |
| 356 | care about it (and not sending it to the screen), or ignoring it when |
| 357 | we don't care, and letting the user see the output so they know that |
| 358 | the build is still running, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 359 | |
| 360 | Params: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 361 | args List of the command and its params |
| 362 | cwd If not None, cd to this directory before running |
| 363 | env Environment to use for execution; if needed, get |
| 364 | os.environ.copy() and add variables. If None, just |
| 365 | use the current environment |
| 366 | capture_output True if we should capture the stdout, false |
| 367 | if we just care about success or not. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 368 | |
| 369 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 370 | If capture_output == True, we return the text output from running |
| 371 | the subprocess as a list of lines, or None if the process failed. |
| 372 | If capture_output == False, we return a True if it successed, or |
| 373 | None if the process failed. |
| 374 | |
| 375 | The caller can evaluate as a bool, because bool(None) == False, and |
| 376 | bool() of a non-empty list is True, or the caller can use the returned |
| 377 | text for further processing. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 378 | """ |
| 379 | logging.debug('Run %s', str(args)) |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 380 | try: |
| 381 | if capture_output: |
| 382 | output = subprocess.check_output(args, cwd=cwd, env=env, |
| 383 | stderr=subprocess.STDOUT) |
| 384 | else: |
| 385 | subprocess.run(args, cwd=cwd, env=env, check=True) |
| 386 | # Just something to decode so we don't get an empty list |
| 387 | output = b'True' |
| 388 | |
| 389 | logging.debug('process returns 0') |
| 390 | # Convert from byte string to ASCII |
| 391 | decoded = output.decode('utf-8') |
| 392 | # Split into array of individual lines |
| 393 | lines = decoded.split('\n') |
| 394 | return lines |
| 395 | except subprocess.CalledProcessError as err: |
| 396 | logging.debug('process returns %s', str(err.returncode)) |
| 397 | return None |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 398 | |
| 399 | |
| 400 | def cros_workon(status, action): |
| 401 | """Call cros_workon for all the 9999 ebuilds we'll be touching |
| 402 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 403 | Params: |
| 404 | status variant_status object tracking our board, variant, etc. |
| 405 | action 'start' or 'stop' |
| 406 | |
| 407 | Returns: |
| 408 | True if the call to cros_workon was successful, False if failed |
| 409 | """ |
| 410 | |
| 411 | # Build up the command from all the packages in the list |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 412 | workon_cmd = ['cros_workon', '--board=' + status.base, action] + status.workon_pkgs |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 413 | return bool(run_process(workon_cmd)) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 414 | |
| 415 | |
| 416 | def create_coreboot_variant(status): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 417 | """Create source files for a new variant of the reference board in coreboot |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 418 | |
| 419 | This function calls create_coreboot_variant.sh to set up a new variant |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 420 | of the reference board. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 421 | |
| 422 | Params: |
| 423 | status variant_status object tracking our board, variant, etc. |
| 424 | |
| 425 | Returns: |
| 426 | True if everything succeeded, False if something failed |
| 427 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 428 | logging.info('Running step create_coreboot_variant') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 429 | create_coreboot_variant_sh = os.path.join( |
| 430 | os.path.expanduser('~/trunk/src/third_party/coreboot'), |
Paul Fagerburg | 7b516d7 | 2019-12-20 13:13:41 -0700 | [diff] [blame] | 431 | 'util/mainboard/google/create_coreboot_variant.sh') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 432 | return bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 433 | [create_coreboot_variant_sh, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 434 | status.base, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 435 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 436 | status.bug])) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 437 | |
| 438 | |
| 439 | def create_coreboot_config(status): |
| 440 | """Create a coreboot configuration for a new variant |
| 441 | |
| 442 | This function calls create_coreboot_config.sh, which will make a copy |
| 443 | of coreboot.${BOARD} into coreboot.${VARIANT}. |
| 444 | |
| 445 | Params: |
| 446 | status variant_status object tracking our board, variant, etc. |
| 447 | |
| 448 | Returns: |
| 449 | True if the script and test build succeeded, False if something failed |
| 450 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 451 | logging.info('Running step create_coreboot_config') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 452 | create_coreboot_config_sh = os.path.expanduser( |
| 453 | '~/trunk/src/platform/dev/contrib/variant/create_coreboot_config.sh') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 454 | return bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 455 | [create_coreboot_config_sh, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 456 | status.base, |
| 457 | status.board, |
| 458 | status.variant, |
| 459 | status.bug])) |
| 460 | |
| 461 | |
| 462 | def copy_coreboot_config(status): |
| 463 | """Copy the coreboot configuration for a new variant |
| 464 | |
| 465 | This is only necessary for the Zork baseboard right now. |
| 466 | This function calls copy_coreboot_config.sh, which will copy |
| 467 | coreboot.${VARIANT} from |
| 468 | third_party/chromiumos-overlay/sys-boot/coreboot/files/configs |
| 469 | to |
| 470 | src/overlays/overlay-${BASE}/sys-boot/coreboot-${BASE}/files/configs |
| 471 | |
| 472 | Params: |
| 473 | status variant_status object tracking our board, variant, etc. |
| 474 | |
| 475 | Returns: |
| 476 | True if the script and test build succeeded, False if something failed |
| 477 | """ |
| 478 | logging.info('Running step copy_coreboot_config') |
| 479 | copy_coreboot_config_sh = os.path.expanduser( |
| 480 | '~/trunk/src/platform/dev/contrib/variant/copy_coreboot_config.sh') |
| 481 | return bool(run_process( |
| 482 | [copy_coreboot_config_sh, |
| 483 | status.base, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 484 | status.board, |
| 485 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 486 | status.bug])) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 487 | |
| 488 | |
| 489 | def add_fitimage(status): |
| 490 | """Add the source files for a fitimage for the new variant |
| 491 | |
| 492 | This function calls add_fitimage.sh to create a new XSL file for the |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 493 | variant's fitimage, which can override settings from the reference board's |
| 494 | XSL. When this is done, the user will have to build the fitimage by running |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 495 | gen_fit_image.sh outside of the chroot (and outside of this program's |
| 496 | control) because gen_fit_image.sh uses WINE, which is not installed in |
| 497 | the chroot. (There is a linux version of FIT, but it requires Open GL, |
| 498 | which is also not installed in the chroot.) |
| 499 | |
| 500 | Params: |
| 501 | status variant_status object tracking our board, variant, etc. |
| 502 | |
| 503 | Returns: |
| 504 | True if the script succeeded, False otherwise |
| 505 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 506 | logging.info('Running step add_fitimage') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 507 | add_fitimage_sh = os.path.expanduser(os.path.join( |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 508 | '~/trunk/src', status.fitimage_dir, 'files/add_fitimage.sh')) |
| 509 | return bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 510 | [add_fitimage_sh, |
| 511 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 512 | status.bug])) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 513 | |
| 514 | |
| 515 | def gen_fit_image_outside_chroot(status): |
| 516 | """Tell the user to run gen_fit_image.sh outside the chroot |
| 517 | |
| 518 | As noted for add_Fitimage(), gen_fit_image.sh cannot run inside the |
| 519 | chroot. This function tells the user to run gen_fit_image.sh in |
| 520 | their normal environment, and then come back (--continue) when that |
| 521 | is done. |
| 522 | |
| 523 | Params: |
| 524 | status variant_status object tracking our board, variant, etc. |
| 525 | |
| 526 | Returns: |
| 527 | True |
| 528 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 529 | logging.info('Running step gen_fit_image_outside_chroot') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 530 | fit_image_files = check_fit_image_files(status) |
| 531 | # If the list is empty, then `not` of the list is True, so the files |
| 532 | # we need are all present and we can continue. |
| 533 | if not fit_image_files: |
| 534 | return True |
| 535 | |
| 536 | logging.info('The following files need to be generated:') |
| 537 | for filename in fit_image_files: |
| 538 | logging.info('* %s', filename) |
| 539 | logging.info('The fitimage sources are ready for gen_fit_image.sh to process.') |
| 540 | logging.info('gen_fit_image.sh cannot run inside the chroot. Please open a new terminal') |
| 541 | logging.info('window, change to the directory where gen_fit_image.sh is located, and run') |
| 542 | logging.info('./gen_fit_image.sh %s [location of FIT] -b', status.variant) |
| 543 | logging.info('Then re-start this program with --continue.') |
| 544 | logging.info('If your chroot is based in ~/chromiumos, then the folder you want is') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 545 | logging.info('~/chromiumos/src/%s/asset_generation', status.fitimage_dir) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 546 | return False |
| 547 | |
| 548 | |
| 549 | def check_fit_image_files(status): |
| 550 | """Check if the fitimage has been generated |
| 551 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 552 | This function is not called directly as a step, and so it doesn't need |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 553 | to produce any error messages to the user (except with --verbose). |
| 554 | gen_fit_image_outside_chroot will call this function to see if the |
| 555 | fitimage files exist, and if not, then that function will print the |
| 556 | message about how the user needs to run gen_fit_image.sh outside the |
| 557 | chroot. |
| 558 | |
| 559 | Params: |
| 560 | status variant_status object tracking our board, variant, etc. |
| 561 | |
| 562 | Returns: |
| 563 | List of files that *DO NOT* exist and need to be created, [] if |
| 564 | all files are present. |
| 565 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 566 | outputs_dir = os.path.expanduser(os.path.join( |
| 567 | '~/trunk/src', status.fitimage_dir, 'asset_generation/outputs')) |
| 568 | logging.debug('outputs_dir = "%s"', outputs_dir) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 569 | |
| 570 | files = [] |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 571 | if not file_exists(outputs_dir, 'fitimage-' + status.variant + '.bin'): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 572 | files.append('fitimage-' + status.variant + '.bin') |
| 573 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 574 | if not file_exists(outputs_dir, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 575 | 'fitimage-' + status.variant + '-versions.txt'): |
| 576 | files.append('fitimage-' + status.variant + '-versions.txt') |
| 577 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 578 | if not file_exists(outputs_dir, 'fit.log'): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 579 | files.append('fit.log') |
| 580 | |
| 581 | return files |
| 582 | |
| 583 | |
| 584 | def move_fitimage_file(fitimage_dir, filename): |
| 585 | """Move fitimage files from create-place to commit-place |
| 586 | |
| 587 | commit_fitimage needs to move the fitimage files from the place where |
| 588 | they were created to a different directory in the tree. This utility |
| 589 | function handles joining paths and calling a file move function. |
| 590 | |
| 591 | Params: |
| 592 | fitimage_dir Directory where the fitimage files are |
| 593 | filename Name of the file being moved |
| 594 | |
| 595 | Returns: |
| 596 | True if the move succeeded, False if it failed |
| 597 | """ |
| 598 | src_dir = os.path.join(fitimage_dir, 'asset_generation/outputs') |
| 599 | src = os.path.join(src_dir, filename) |
| 600 | dest_dir = os.path.join(fitimage_dir, 'files') |
| 601 | dest = os.path.join(dest_dir, filename) |
| 602 | # If src does not exist and dest does, the move is already done => success! |
| 603 | if not file_exists(src_dir, filename) and file_exists(dest_dir, filename): |
| 604 | logging.debug('move "%s", "%s" unnecessary because dest exists and' |
| 605 | ' src does not exist', src, dest) |
| 606 | return True |
| 607 | |
| 608 | logging.debug('move "%s", "%s"', src, dest) |
| 609 | return shutil.move(src, dest) |
| 610 | |
| 611 | |
| 612 | def commit_fitimage(status): |
| 613 | """Move the fitimage files and add them to a git commit |
| 614 | |
| 615 | This function moves the fitimage binary and -versions files from |
| 616 | asset_generation/outputs to files/ and then adds those files and |
| 617 | fit.log to the existing git commit. |
| 618 | |
| 619 | Params: |
| 620 | status variant_status object tracking our board, variant, etc. |
| 621 | |
| 622 | Returns: |
| 623 | True if the copy, git add, and git commit --amend all succeeded. |
| 624 | False if something failed. |
| 625 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 626 | logging.info('Running step commit_fitimage') |
| 627 | fitimage_dir = os.path.expanduser(os.path.join('~/trunk/src', status.fitimage_dir)) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 628 | logging.debug('fitimage_dir = "%s"', fitimage_dir) |
| 629 | |
| 630 | # The copy operation will check that the source file exists, so no |
| 631 | # need to check separately. |
| 632 | if not move_fitimage_file(fitimage_dir, |
| 633 | 'fitimage-' + status.variant + '.bin'): |
| 634 | logging.error('Moving fitimage binary failed') |
| 635 | return False |
| 636 | |
| 637 | if not move_fitimage_file(fitimage_dir, |
| 638 | 'fitimage-' + status.variant + '-versions.txt'): |
| 639 | logging.error('Moving fitimage versions.txt failed') |
| 640 | return False |
| 641 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 642 | if not bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 643 | ['git', 'add', |
| 644 | 'asset_generation/outputs/fit.log', |
| 645 | 'files/fitimage-' + status.variant + '.bin', |
| 646 | 'files/fitimage-' + status.variant + '-versions.txt' |
| 647 | ], |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 648 | cwd=fitimage_dir)): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 649 | return False |
| 650 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 651 | return bool(run_process(['git', 'commit', '--amend', '--no-edit'], |
| 652 | cwd=fitimage_dir)) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 653 | |
| 654 | |
| 655 | def create_initial_ec_image(status): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 656 | """Create an EC image for the variant as a clone of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 657 | |
| 658 | This function calls create_initial_ec_image.sh, which will clone the |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 659 | reference board to create the variant. The shell script will build the |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 660 | EC code for the variant, but the repo upload hook insists that we |
| 661 | have done a `make buildall` before it will allow an upload, so this |
| 662 | function does the buildall. |
| 663 | |
| 664 | Params: |
| 665 | status variant_status object tracking our board, variant, etc. |
| 666 | |
| 667 | Returns: |
| 668 | True if the script and test build succeeded, False if something failed |
| 669 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 670 | logging.info('Running step create_initial_ec_image') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 671 | create_initial_ec_image_sh = os.path.expanduser( |
| 672 | '~/trunk/src/platform/dev/contrib/variant/create_initial_ec_image.sh') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 673 | if not bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 674 | [create_initial_ec_image_sh, |
| 675 | status.board, |
| 676 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 677 | status.bug])): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 678 | return False |
| 679 | |
| 680 | # create_initial_ec_image.sh will build the ec.bin for this variant |
| 681 | # if successful. |
| 682 | ec = os.path.expanduser('~/trunk/src/platform/ec') |
| 683 | logging.debug('ec = "%s"', ec) |
| 684 | ec_bin = 'build/' + status.variant + '/ec.bin' |
| 685 | logging.debug('ec.bin = "%s"', ec_bin) |
| 686 | |
| 687 | return file_exists(ec, ec_bin) |
| 688 | |
| 689 | |
| 690 | def ec_buildall(status): |
| 691 | """Do a make buildall -j for the EC, which is required for repo upload |
| 692 | |
| 693 | The upload hook checks to ensure that the entire EC codebase builds |
| 694 | without error, so we have to run make buildall -j before uploading. |
| 695 | |
| 696 | Params: |
| 697 | status variant_status object tracking our board, variant, etc. |
| 698 | |
| 699 | Returns: |
| 700 | True if the script and test build succeeded, False if something failed |
| 701 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 702 | logging.info('Running step ec_buildall') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 703 | del status # unused parameter |
| 704 | ec = os.path.expanduser('~/trunk/src/platform/ec') |
| 705 | logging.debug('ec = "%s"', ec) |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 706 | return bool(run_process(['make', 'buildall', '-j'], cwd=ec)) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 707 | |
| 708 | |
| 709 | def add_variant_to_yaml(status): |
| 710 | """Add the new variant to the public and private model.yaml files |
| 711 | |
| 712 | This function calls add_variant_to_yaml.sh (the public yaml) and |
| 713 | add_variant.sh (the private yaml) to add the new variant to |
| 714 | the yaml files. |
| 715 | |
| 716 | Params: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 717 | status variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 718 | |
| 719 | Returns: |
| 720 | True if the scripts and build succeeded, False is something failed |
| 721 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 722 | logging.info('Running step add_variant_to_yaml') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 723 | add_variant_to_yaml_sh = os.path.expanduser( |
| 724 | '~/trunk/src/platform/dev/contrib/variant/add_variant_to_yaml.sh') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 725 | if not bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 726 | [add_variant_to_yaml_sh, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 727 | status.base, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 728 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 729 | status.bug])): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 730 | return False |
| 731 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 732 | add_variant_sh = os.path.expanduser(os.path.join(status.private_yaml_dir, 'add_variant.sh')) |
| 733 | return bool(run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 734 | [add_variant_sh, |
| 735 | status.variant, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 736 | status.bug])) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 737 | |
| 738 | |
| 739 | def build_yaml(status): |
| 740 | """Build config files from the yaml files |
| 741 | |
| 742 | This function builds the yaml files into the JSON and C code that |
| 743 | mosys and other tools use, then verifies that the new variant's name |
| 744 | shows up in all of the output files. |
| 745 | |
| 746 | Params: |
| 747 | status variant_status object tracking our board, variant, etc. |
| 748 | |
| 749 | Returns: |
| 750 | True if the scripts and build succeeded, False is something failed |
| 751 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 752 | logging.info('Running step build_yaml') |
| 753 | if not bool(run_process([status.emerge_cmd] + status.yaml_emerge_pkgs)): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 754 | return False |
| 755 | |
| 756 | # Check generated files for occurences of the variant name. |
| 757 | # Each file should have at least one occurence, so use `grep -c` to |
| 758 | # count the occurrences of the variant name in each file. |
| 759 | # The results will be something like this: |
| 760 | # config.json:10 |
| 761 | # yaml/config.c:6 |
| 762 | # yaml/config.yaml:27 |
| 763 | # yaml/model.yaml:6 |
| 764 | # yaml/private-model.yaml:10 |
| 765 | # If the variant name doesn't show up in the file, then the count |
| 766 | # will be 0, so we would see, e.g. |
| 767 | # config.json:0 |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 768 | # Note that we leave out yaml/model.yaml (the public one) because for |
| 769 | # some boards, there is nothing in the public yaml file. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 770 | # We gather the output from grep, then look for any of the strings |
| 771 | # ending in :0. If none of them match, then we're good, but if even |
| 772 | # one of them ends with :0 then there was a problem with generating |
| 773 | # the files from the yaml. |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 774 | chromeos_config = '/build/' + status.base + '/usr/share/chromeos-config' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 775 | logging.debug('chromeos_config = "%s"', chromeos_config) |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 776 | grep = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 777 | ['grep', |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 778 | '-ci', |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 779 | status.variant, |
| 780 | 'config.json', |
| 781 | 'yaml/config.c', |
| 782 | 'yaml/config.yaml', |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 783 | 'yaml/private-model.yaml'], cwd=chromeos_config, capture_output=True) |
| 784 | |
| 785 | if grep is None: |
| 786 | return False |
| 787 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 788 | return not bool([s for s in grep if re.search(r':0$', s)]) |
| 789 | |
| 790 | |
| 791 | def emerge_all(status): |
| 792 | """Build the coreboot BIOS and EC code for the new variant |
| 793 | |
| 794 | Params: |
| 795 | status variant_status object tracking our board, variant, etc. |
| 796 | |
| 797 | Returns: |
| 798 | True if the build succeeded, False if something failed |
| 799 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 800 | logging.info('Running step emerge_all') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 801 | cros_workon(status, 'start') |
| 802 | environ = os.environ.copy() |
| 803 | environ['FW_NAME'] = status.variant |
| 804 | # Build up the command for emerge from all the packages in the list |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 805 | emerge_cmd_and_params = [status.emerge_cmd] + status.emerge_pkgs |
| 806 | if not bool(run_process(emerge_cmd_and_params, env=environ)): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 807 | return False |
| 808 | |
| 809 | cros_workon(status, 'stop') |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 810 | build_path = '/build/' + status.base + '/firmware' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 811 | logging.debug('build_path = "%s"', build_path) |
| 812 | if not file_exists(build_path, 'image-' + status.variant + '.bin'): |
| 813 | logging.error('emerge failed because image-%s.bin does not exist', |
| 814 | status.variant) |
| 815 | return False |
| 816 | |
| 817 | if not file_exists(build_path, 'image-' + status.variant + '.dev.bin'): |
| 818 | logging.error('emerge failed because image-%s.dev.bin does not exist', |
| 819 | status.variant) |
| 820 | return False |
| 821 | |
| 822 | if not file_exists(build_path, 'image-' + status.variant + '.net.bin'): |
| 823 | logging.error('emerge failed because image-%s.net.bin does not exist', |
| 824 | status.variant) |
| 825 | return False |
| 826 | |
| 827 | if not file_exists(build_path, 'image-' + status.variant + '.serial.bin'): |
| 828 | logging.error('emerge failed because image-%s.serial.bin does not exist', |
| 829 | status.variant) |
| 830 | return False |
| 831 | |
| 832 | return True |
| 833 | |
| 834 | |
| 835 | def push_coreboot(status): |
| 836 | """Push the coreboot CL to coreboot.org |
| 837 | |
| 838 | Params: |
| 839 | status variant_status object tracking our board, variant, etc. |
| 840 | |
| 841 | Returns: |
| 842 | True if the build succeeded, False if something failed |
| 843 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 844 | logging.info('Running step push_coreboot') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 845 | del status # unused parameter |
| 846 | logging.error('TODO (pfagerburg): implement push_coreboot') |
| 847 | return True |
| 848 | |
| 849 | |
| 850 | def upload_CLs(status): |
| 851 | """Upload all CLs to chromiumos |
| 852 | |
| 853 | Params: |
| 854 | status variant_status object tracking our board, variant, etc. |
| 855 | |
| 856 | Returns: |
| 857 | True if the build succeeded, False if something failed |
| 858 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 859 | logging.info('Running step upload_CLs') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 860 | del status # unused parameter |
| 861 | logging.error('TODO (pfagerburg): implement upload_CLs') |
| 862 | return True |
| 863 | |
| 864 | |
| 865 | def find_coreboot_upstream(status): |
| 866 | """Find the coreboot CL after it has been upstreamed to chromiumos |
| 867 | |
| 868 | Params: |
| 869 | status variant_status object tracking our board, variant, etc. |
| 870 | |
| 871 | Returns: |
| 872 | True if the build succeeded, False if something failed |
| 873 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 874 | logging.info('Running step find_coreboot_upstream') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 875 | del status # unused parameter |
| 876 | logging.error('TODO (pfagerburg): implement find_coreboot_upstream') |
| 877 | return True |
| 878 | |
| 879 | |
| 880 | def add_cq_depends(status): |
| 881 | """Add Cq-Depends to all of the CLs in chromiumos |
| 882 | |
| 883 | The CL in coreboot needs to be pushed to coreboot.org, get merged, |
| 884 | and then get upstreamed into the chromiumos tree before the other |
| 885 | CLs can cq-depend on it and pass CQ. |
| 886 | |
| 887 | Params: |
| 888 | status variant_status object tracking our board, variant, etc. |
| 889 | |
| 890 | Returns: |
| 891 | True if the build succeeded, False if something failed |
| 892 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 893 | logging.info('Running step add_cq_depends') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 894 | del status # unused parameter |
| 895 | logging.error('TODO (pfagerburg): implement add_cq_depends') |
| 896 | return True |
| 897 | |
| 898 | |
| 899 | def clean_up(status): |
| 900 | """Final clean-up, including delete the status file |
| 901 | |
| 902 | Params: |
| 903 | status variant_status object tracking our board, variant, etc. |
| 904 | |
| 905 | Returns: |
| 906 | True |
| 907 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 908 | logging.info('Running step clean_up') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 909 | status.rm() |
| 910 | return True |
| 911 | |
| 912 | |
| 913 | if __name__ == '__main__': |
| 914 | sys.exit(not int(main())) |