Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # -*- coding: utf-8 -*- |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 3 | # |
| 4 | # Copyright 2020 The Chromium OS Authors. All rights reserved. |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | # |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 8 | """Create a new variant of an existing reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 9 | |
| 10 | This program will call all of the scripts that create the various pieces |
| 11 | of a new variant. For example to create a new variant of the hatch base |
| 12 | board, the following scripts are called: |
| 13 | |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 14 | * platform/dev/contrib/variant/create_coreboot_variant.sh |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 15 | * platform/dev/contrib/variant/create_coreboot_config.sh |
| 16 | * private-overlays/baseboard-hatch-private/sys-boot/ |
| 17 | * coreboot-private-files-hatch/files/add_fitimage.sh |
| 18 | * coreboot-private-files-hatch/asset_generation/gen_fit_image.sh |
| 19 | * Outside the chroot, because it uses WINE to run the FIT tools |
| 20 | * platform/dev/contrib/variant/create_initial_ec_image.sh |
| 21 | * platform/dev/contrib/variant/add_variant_to_yaml.sh |
| 22 | * private-overlays/overlay-hatch-private/chromeos-base/ |
| 23 | * chromeos-config-bsp-hatch-private/add_variant.sh |
| 24 | |
| 25 | Once the scripts are done, the following repos have changes |
| 26 | |
| 27 | * third_party/coreboot |
| 28 | * third_party/chromiumos-overlay |
| 29 | * private-overlays/baseboard-hatch-private |
| 30 | * platform/ec |
| 31 | * private-overlays/overlay-hatch-private |
| 32 | * overlays |
| 33 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 34 | The program has support for multiple reference boards, so the repos, directories, |
| 35 | and scripts above can change depending on what the reference board is. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 36 | """ |
| 37 | |
| 38 | from __future__ import print_function |
| 39 | import argparse |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 40 | import importlib |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 41 | import json |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 42 | import logging |
| 43 | import os |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 44 | import subprocess |
| 45 | import sys |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 46 | from chromite.lib import build_target_lib |
| 47 | from chromite.lib import cros_build_lib |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 48 | from chromite.lib import git |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 49 | from chromite.lib import gerrit |
Paul Fagerburg | a95dd16 | 2020-03-24 16:27:18 -0600 | [diff] [blame] | 50 | from chromite.lib import osutils |
Paul Fagerburg | 7539807 | 2020-03-16 13:51:24 -0600 | [diff] [blame] | 51 | from chromite.lib import workon_helper |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 52 | import requests |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 53 | import step_names |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 54 | import variant_status |
| 55 | |
| 56 | |
| 57 | def main(): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 58 | """Create a new variant of an existing reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 59 | |
| 60 | This program automates the creation of a new variant of an existing |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 61 | reference board by calling various scripts that copy the reference board, |
| 62 | modify files for the new variant, stage commits, and upload to gerrit. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 63 | |
| 64 | Note that one of the following is required: |
| 65 | * --continue |
| 66 | * --board=BOARD --variant=VARIANT [--bug=BUG] |
| 67 | """ |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 68 | board, variant, bug, continue_flag, abort_flag = get_args() |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 69 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 70 | if not check_flags(board, variant, bug, continue_flag, abort_flag): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 71 | return False |
| 72 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 73 | status = get_status(board, variant, bug, continue_flag, abort_flag) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 74 | if status is None: |
| 75 | return False |
| 76 | |
| 77 | status.load() |
| 78 | |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 79 | # Where is new_variant.py located? |
| 80 | status.my_loc = os.path.dirname(os.path.abspath(__file__)) |
| 81 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 82 | # If the user specified --abort, override the current step. |
| 83 | if abort_flag: |
| 84 | status.step = step_names.ABORT |
| 85 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 86 | while status.step is not None: |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 87 | status.save() |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 88 | if not perform_step(status): |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 89 | logging.debug('perform_step %s returned False; exiting ...', |
| 90 | status.step) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 91 | return False |
| 92 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 93 | move_to_next_step(status) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 94 | |
| 95 | return True |
| 96 | |
| 97 | |
| 98 | def get_args(): |
| 99 | """Parse the command-line arguments |
| 100 | |
| 101 | There doesn't appear to be a way to specify that --continue is |
| 102 | mutually exclusive with --board, --variant, and --bug. As a result, |
| 103 | all arguments are optional, and another function will apply the logic |
| 104 | to check if there is an illegal combination of arguments. |
| 105 | |
| 106 | Returns a list of: |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 107 | board Name of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 108 | variant Name of the variant being created |
| 109 | bug Text for bug number, if any ('None' otherwise) |
| 110 | continue_flag Flag if --continue was specified |
| 111 | """ |
| 112 | parser = argparse.ArgumentParser( |
| 113 | description=main.__doc__, |
| 114 | formatter_class=argparse.RawTextHelpFormatter) |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 115 | parser.add_argument('--board', type=str, help='Name of the reference board') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 116 | parser.add_argument( |
| 117 | '--variant', type=str, help='Name of the new variant to create') |
| 118 | parser.add_argument( |
| 119 | '--bug', type=str, help='Bug number to reference in commits') |
Paul Fagerburg | 2e48a19 | 2020-03-24 17:57:05 -0600 | [diff] [blame] | 120 | # Use a group so that we can enforce mutually-exclusive argurments. |
| 121 | # argparse does not support nesting groups, so we can't put board, |
| 122 | # variant, and bug into a group and have that group as another mutually |
| 123 | # exclusive option. |
| 124 | group = parser.add_mutually_exclusive_group() |
| 125 | group.add_argument( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 126 | '--continue', action='store_true', |
| 127 | dest='continue_flag', help='Continue the process from where it paused') |
Paul Fagerburg | 2e48a19 | 2020-03-24 17:57:05 -0600 | [diff] [blame] | 128 | group.add_argument( |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 129 | '--abort', action='store_true', |
| 130 | dest='abort_flag', help='Cancel the process and abandon all commits') |
| 131 | parser.add_argument( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 132 | '--verbose', action='store_true', |
| 133 | dest='verbose_flag', help='Enable verbose output of progress') |
| 134 | args = parser.parse_args() |
| 135 | |
| 136 | if args.verbose_flag: |
| 137 | logging.basicConfig(level=logging.DEBUG) |
| 138 | else: |
| 139 | logging.basicConfig(level=logging.INFO) |
| 140 | |
| 141 | board = args.board |
| 142 | if board is not None: |
| 143 | board = board.lower() |
| 144 | |
| 145 | variant = args.variant |
| 146 | if variant is not None: |
| 147 | variant = variant.lower() |
| 148 | |
| 149 | bug = args.bug or 'None' |
| 150 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 151 | return (board, variant, bug, args.continue_flag, args.abort_flag) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 152 | |
| 153 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 154 | def check_flags(board, variant, bug, continue_flag, abort_flag): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 155 | """Check the flags to ensure no invalid combinations |
| 156 | |
| 157 | We allow any of the following: |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 158 | --abort |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 159 | --continue |
| 160 | --board=board_name --variant=variant_name |
| 161 | --board=board_name --variant=variant_name --bug=bug_text |
| 162 | |
| 163 | The argument parser does have the functionality to represent the |
| 164 | combination of --board and --variant as a single mutually-exclusive |
| 165 | argument, so we have to use this function to do the checking. |
| 166 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 167 | Args: |
| 168 | board: Name of the reference board |
| 169 | variant: Name of the variant being created |
| 170 | bug: Text for bug number, if any ('None' otherwise) |
| 171 | continue_flag: Flag if --continue was specified |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 172 | abort_flag: Flag if --abort was specified |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 173 | |
| 174 | Returns: |
| 175 | True if the arguments are acceptable, False otherwise |
| 176 | """ |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 177 | # If either --abort or --continue is set, then disallow any of the |
| 178 | # board name, variant name, or bug number to be set. |
| 179 | if continue_flag or abort_flag: |
| 180 | if board is not None or variant is not None or bug != 'None': |
| 181 | logging.error('Do not use --board, --variant, or --bug with ' |
| 182 | '--continue or --abort') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 183 | return False |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 184 | return True |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 185 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 186 | # At this point, neither --continue nor --abort are set, so we must have |
| 187 | # both --board and --variant values. |
| 188 | if board is None or variant is None: |
| 189 | logging.error('Both --board and --variant must be specified') |
| 190 | return False |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 191 | |
| 192 | return True |
| 193 | |
| 194 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 195 | def get_status(board, variant, bug, continue_flag, abort_flag): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 196 | """Create the status file or get the previous status |
| 197 | |
| 198 | This program can stop at several places as we have to wait for CLs |
| 199 | to work through CQ or be upstreamed into the chromiumos tree, so just |
| 200 | like a git cherry-pick, there is a --continue option to pick up where |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 201 | you left off by reading a specially-named status file. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 202 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 203 | If --continue is specified, the status file must exist. |
| 204 | If the status file exists, then --continue must be specified. |
| 205 | When --continue is specified, we read the status file and return |
| 206 | with the contents. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 207 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 208 | If the status file does not exist, we will create the state file with |
| 209 | the board, variant, and (optional) bug details. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 210 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 211 | To decouple the list of boards supported from this main program, we |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 212 | try to import a module with the same name as the reference board, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 213 | so --board=hatch means that we import hatch.py. If we can't import |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 214 | the file, then we don't support that reference board. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 215 | |
| 216 | The board-specific module will set several variables, which we will |
| 217 | copy into the object that we return. |
| 218 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 219 | * base - the name of the base board, such as Hatch, Volteer, or Zork. |
| 220 | This can be different from the reference board, e.g. the Trembyle |
| 221 | reference board in the Zork project. |
Paul Fagerburg | 3ef1cbb | 2020-06-02 16:38:36 -0600 | [diff] [blame] | 222 | * coreboot_base - the name of the base board in coreboot. Usually the same |
| 223 | as base, but can differ, e.g. for Puff, the base is Puff, but the |
| 224 | coreboot_base is Hatch because Puff is based on Hatch. |
Paul Fagerburg | 4d34345 | 2020-01-24 15:23:53 -0700 | [diff] [blame] | 225 | * coreboot_dir - base directory for coreboot, usually third_party/coreboot |
| 226 | but could differ for processors that use a private repo |
| 227 | * cb_config_dir - base directory for coreboot configs, usually |
| 228 | third_party/chromiumos-overlay/sys-boot/coreboot/files/configs but |
| 229 | could differ for processors that use a private repo |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 230 | * step_list - list of steps (named in step_names.py) to run in sequence |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 231 | to create the new variant of the reference board |
| 232 | * fsp - package name for FSP. This may be None, depending on the |
| 233 | processor on the reference board |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 234 | * fitimage_pkg - package name for the fitimage |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 235 | * fitimage_dir - directory for fitimage; prepend '/mnt/host/source/src/' |
| 236 | in chroot, prepend '~/chromiumos/src' outside the chroot |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 237 | * workon_pkgs - list of packages to cros_workon |
| 238 | * emerge_cmd - the emerge command, e.g. 'emerge-hatch' |
| 239 | * emerge_pkgs - list of packages to emerge |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 240 | * config_workon_pkgs - list of packages to cros_workon to build the |
| 241 | project config |
| 242 | * config_emerge_pkgs - list of packages to emerge to build the project |
| 243 | config |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 244 | * private_yaml_dir - directory for the private yaml file |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 245 | * commits - map of commits for the various steps. Indexed by step name, |
| 246 | and the step names used are the same ones in step_names.py |
| 247 | * repo_upload_list - list of commits to upload using `repo upload` |
| 248 | * coreboot_push_list - list of commits to upload using `git push` to |
| 249 | coreboot |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 250 | * depends - maps a step to a list of steps on which it depends, e.g. |
| 251 | depends[step_names.ADD_PRIV_YAML] is a list of other steps that |
| 252 | the 'add_priv_yaml' step depends on. This map is used to amend |
| 253 | the commits with CL numbers for Cq-Depend. |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 254 | |
| 255 | Additionally, the following fields will be set: |
| 256 | |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 257 | * board - the name of the reference board, e.g. 'hatch' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 258 | * variant - the name of the variant, e.g. 'sushi' |
| 259 | * bug - optional text for a bug ID, used in the git commit messages. |
| 260 | Could be 'None' (as text, not the python None), or something like |
| 261 | 'b:12345' for buganizer, or 'chromium:12345' |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 262 | * step - internal state tracking, what step of the variant creation |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 263 | we are at. |
| 264 | * yaml_file - internal, just the name of the file where all this data |
| 265 | gets saved. |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 266 | * commit - a map of maps that tracks all of the git commit and gerrit CL |
| 267 | data for each of the steps in the process. For example, |
| 268 | status.commit['add_priv_yaml'] is a map that has all the information |
| 269 | about the 'add_priv_yaml' step. The keys in the maps allow us to |
| 270 | determine where the commit is, the change_id, if it has been uploaded |
| 271 | to gerrit and where. |
| 272 | |
| 273 | branch_name - the name of the git branch |
| 274 | change_id - the change-id assigned by the commit hook. Gerrit |
| 275 | uses the change_id to track new patchsets in the CL |
| 276 | dir - the directory where the commit has been created |
| 277 | gerrit - the name of the gerrit instance to which the CL has |
| 278 | been uploaded, one of 'chromium', 'chrome-internal', or |
| 279 | 'coreboot' |
| 280 | cl_number - the CL number on the gerrit instance |
| 281 | |
| 282 | When the commit is created, branch_name, change_id, and dir are all |
| 283 | set. The gerrit and cl_number keys are not set until the CL has been |
| 284 | uploaded to a gerrit instance. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 285 | |
| 286 | These data might come from the status file (because we read it), or |
| 287 | they might be the initial values after we created the file (because |
| 288 | it did not already exist). |
| 289 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 290 | Args: |
| 291 | board: Name of the reference board |
| 292 | variant: Name of the variant being created |
| 293 | bug: Text for bug number, if any ('None' otherwise) |
| 294 | continue_flag: Flag if --continue was specified |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 295 | abort_flag: Flag if --abort was specified |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 296 | |
| 297 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 298 | variant_status object with all the data mentioned above |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 299 | """ |
| 300 | status = variant_status.variant_status() |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 301 | if continue_flag or abort_flag: |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 302 | if status.yaml_file_exists(): |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 303 | return status |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 304 | |
| 305 | if continue_flag: |
| 306 | op = '--continue' |
| 307 | if abort_flag: |
| 308 | op = '--abort' |
| 309 | logging.error('%s does not exist; cannot %s', status.yaml_file, op) |
| 310 | return None |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 311 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 312 | # If we get here, the user provided --board and --variant (because |
| 313 | # check_flags() returned Trued), but the yaml file already exists, |
| 314 | # so we print an error message and bail. |
| 315 | if status.yaml_file_exists(): |
| 316 | logging.error( |
| 317 | 'new_variant already in progress; did you forget --continue?') |
| 318 | return None |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 319 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 320 | # At this point, it's not --continue, not --abort, the yaml file doesn't |
| 321 | # exist, and we have valid values for --board, --variant, and --bug (bug |
| 322 | # might be the default value of "None"). Create the yaml file with data |
| 323 | # from the reference board's loadable module. |
| 324 | status.board = board |
| 325 | status.variant = variant |
| 326 | status.bug = bug |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 327 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 328 | # Load the appropriate module and copy all the data from it. |
| 329 | try: |
| 330 | module = importlib.import_module(board) |
| 331 | except ImportError: |
| 332 | print('Unsupported board "' + board + '"') |
| 333 | sys.exit(1) |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 334 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 335 | # pylint: disable=bad-whitespace |
| 336 | # Allow extra spaces around = so that we can line things up nicely |
| 337 | status.base = module.base |
Paul Fagerburg | 3ef1cbb | 2020-06-02 16:38:36 -0600 | [diff] [blame] | 338 | status.coreboot_base = getattr(module, 'coreboot_base', module.base) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 339 | status.coreboot_dir = module.coreboot_dir |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 340 | status.cb_config_dir = getattr(module, 'cb_config_dir', None) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 341 | status.emerge_cmd = module.emerge_cmd |
| 342 | status.emerge_pkgs = module.emerge_pkgs |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 343 | status.fitimage_dir = getattr(module, 'fitimage_dir', None) |
| 344 | status.fitimage_pkg = getattr(module, 'fitimage_pkg', None) |
| 345 | status.fitimage_cmd = getattr(module, 'fitimage_cmd', None) |
| 346 | status.fsp = getattr(module, 'fsp', None) |
| 347 | status.private_yaml_dir = getattr(module, 'private_yaml_dir', None) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 348 | status.step_list = module.step_list |
| 349 | status.workon_pkgs = module.workon_pkgs |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 350 | status.config_workon_pkgs = module.config_workon_pkgs |
| 351 | status.config_emerge_pkgs = module.config_emerge_pkgs |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 352 | status.coreboot_push_list = module.coreboot_push_list |
| 353 | status.repo_upload_list = module.repo_upload_list |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 354 | status.depends = module.depends |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 355 | # pylint: enable=bad-whitespace |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 356 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 357 | # Start at the first entry in the step list |
| 358 | status.step = status.step_list[0] |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 359 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 360 | # Start an empty map for tracking CL data |
| 361 | status.commits = {} |
| 362 | |
| 363 | status.save() |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 364 | |
| 365 | return status |
| 366 | |
| 367 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 368 | def perform_step(status): |
| 369 | """Call the appropriate function for the current step |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 370 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 371 | Args: |
| 372 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 373 | |
| 374 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 375 | True if the step succeeded, False if it failed |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 376 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 377 | # Function to call based on the step |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 378 | dispatch = { |
Paul Fagerburg | bc18424 | 2020-04-28 17:00:54 -0600 | [diff] [blame] | 379 | step_names.PROJECT_CONFIG: project_config, |
Paul Fagerburg | 8f60b80 | 2020-06-09 15:38:28 -0600 | [diff] [blame^] | 380 | step_names.FW_BUILD_CONFIG: fw_build_config, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 381 | step_names.CB_VARIANT: create_coreboot_variant, |
| 382 | step_names.CB_CONFIG: create_coreboot_config, |
Paul Fagerburg | 2fd23f4 | 2020-02-07 14:04:48 -0700 | [diff] [blame] | 383 | step_names.CRAS_CONFIG: copy_cras_config, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 384 | step_names.ADD_FIT: add_fitimage, |
| 385 | step_names.GEN_FIT: gen_fit_image_outside_chroot, |
| 386 | step_names.COMMIT_FIT: commit_fitimage, |
| 387 | step_names.EC_IMAGE: create_initial_ec_image, |
| 388 | step_names.EC_BUILDALL: ec_buildall, |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 389 | step_names.ADD_PUB_YAML: add_variant_to_public_yaml, |
| 390 | step_names.ADD_PRIV_YAML: add_variant_to_private_yaml, |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 391 | step_names.BUILD_CONFIG: build_config, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 392 | step_names.EMERGE: emerge_all, |
| 393 | step_names.PUSH: push_coreboot, |
| 394 | step_names.UPLOAD: upload_CLs, |
| 395 | step_names.FIND: find_coreboot_upstream, |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 396 | step_names.CALC_CQ_DEPEND: calc_cq_depend, |
| 397 | step_names.ADD_CQ_DEPEND: add_cq_depend, |
| 398 | step_names.RE_UPLOAD: re_upload, |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 399 | step_names.CLEAN_UP: clean_up, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 400 | step_names.ABORT: abort, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 401 | } |
| 402 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 403 | if status.step not in dispatch: |
| 404 | logging.error('Unknown step "%s", aborting...', status.step) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 405 | sys.exit(1) |
| 406 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 407 | return dispatch[status.step](status) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 408 | |
| 409 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 410 | def move_to_next_step(status): |
| 411 | """Move to the next step in the list |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 412 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 413 | Args: |
| 414 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 415 | """ |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 416 | # Special case: the next step after 'abort' is 'clean_up'. Always. |
| 417 | if status.step == step_names.ABORT: |
| 418 | status.step = step_names.CLEAN_UP |
| 419 | return |
| 420 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 421 | if status.step not in status.step_list: |
| 422 | logging.error('Unknown step "%s", aborting...', status.step) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 423 | sys.exit(1) |
| 424 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 425 | idx = status.step_list.index(status.step) |
| 426 | if idx == len(status.step_list)-1: |
| 427 | status.step = None |
Paul Fagerburg | dd3e9c3 | 2020-01-07 14:00:35 -0700 | [diff] [blame] | 428 | else: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 429 | status.step = status.step_list[idx+1] |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 430 | |
| 431 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 432 | def run_process(args, cwd=None, env=None, capture_output=False): |
| 433 | """Run a process, log debug messages, return text output of process |
| 434 | |
| 435 | The capture_output parameter allows us to capture the output when we |
| 436 | care about it (and not sending it to the screen), or ignoring it when |
| 437 | we don't care, and letting the user see the output so they know that |
| 438 | the build is still running, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 439 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 440 | Args: |
| 441 | args: List of the command and its params |
| 442 | cwd: If not None, cd to this directory before running |
| 443 | env: Environment to use for execution; if needed, get os.environ.copy() |
| 444 | and add variables. If None, just use the current environment |
| 445 | capture_output: True if we should capture the stdout, false if we |
| 446 | just care about success or not. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 447 | |
| 448 | Returns: |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 449 | If capture_output == True, we return the text output from running |
| 450 | the subprocess as a list of lines, or None if the process failed. |
| 451 | If capture_output == False, we return a True if it successed, or |
| 452 | None if the process failed. |
| 453 | |
| 454 | The caller can evaluate as a bool, because bool(None) == False, and |
| 455 | bool() of a non-empty list is True, or the caller can use the returned |
| 456 | text for further processing. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 457 | """ |
| 458 | logging.debug('Run %s', str(args)) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 459 | if cwd is not None: |
| 460 | logging.debug('cwd = %s', cwd) |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 461 | try: |
| 462 | if capture_output: |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 463 | output = subprocess.run(args, cwd=cwd, env=env, check=True, |
| 464 | stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 465 | else: |
| 466 | subprocess.run(args, cwd=cwd, env=env, check=True) |
| 467 | # Just something to decode so we don't get an empty list |
| 468 | output = b'True' |
| 469 | |
| 470 | logging.debug('process returns 0') |
| 471 | # Convert from byte string to ASCII |
| 472 | decoded = output.decode('utf-8') |
| 473 | # Split into array of individual lines |
| 474 | lines = decoded.split('\n') |
| 475 | return lines |
| 476 | except subprocess.CalledProcessError as err: |
| 477 | logging.debug('process returns %s', str(err.returncode)) |
| 478 | return None |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 479 | |
| 480 | |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 481 | def get_git_commit_data(cwd): |
| 482 | """Get the branch name and change id of the current commit |
| 483 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 484 | Args: |
| 485 | cwd: The current working directory, where we want to get the branch |
| 486 | name and change id |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 487 | |
| 488 | Returns: |
| 489 | Map with 'dir', 'branch_name' and 'change_id' keys. The 'dir' |
| 490 | key maps to the value of os.path.expanduser(cwd) |
| 491 | """ |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 492 | cwd = git.FindGitTopLevel(os.path.expanduser(cwd)) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 493 | logging.debug('get_git_commit_data(%s)', cwd) |
| 494 | |
| 495 | branch_name = git.GetCurrentBranch(cwd) |
| 496 | if branch_name is None: |
| 497 | logging.error('Cannot determine git branch name in %s; exiting', cwd) |
| 498 | sys.exit(1) |
| 499 | logging.debug('git current branch is %s', branch_name) |
| 500 | |
| 501 | change_id = git.GetChangeId(cwd) |
| 502 | if change_id is None: |
| 503 | logging.error('Cannot determine Change-Id in %s; exiting', cwd) |
| 504 | sys.exit(1) |
| 505 | logging.debug('git Change-Id is %s', change_id) |
| 506 | |
| 507 | return { |
| 508 | 'dir': cwd, |
| 509 | 'branch_name': branch_name, |
| 510 | 'change_id': change_id |
| 511 | } |
| 512 | |
| 513 | |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 514 | def change_id_to_sha(git_repo, change_id): |
| 515 | """Find the SHA for a given Change-Id. |
| 516 | |
| 517 | Args: |
| 518 | git_repo: Directory of git repository. |
| 519 | change_id: The Change-Id to search for. |
| 520 | |
| 521 | Returns: |
| 522 | The SHA hash for the Change-Id if only one commit is found. |
| 523 | None if the Change-Id was not found. |
| 524 | Raises a ValueError if more than one commit is found with the |
| 525 | same Change-Id. |
| 526 | """ |
| 527 | output = git.Log(git_repo, max_count=1, format='format:%H', |
| 528 | grep=fr'^Change-Id: {change_id}$') |
| 529 | sha_hashes = output.splitlines() |
| 530 | if not sha_hashes: |
| 531 | return None |
| 532 | if len(sha_hashes) > 1: |
| 533 | raise ValueError('More than one SHA with that Change-Id found') |
| 534 | return sha_hashes[0] |
| 535 | |
| 536 | |
| 537 | def get_commit_msg(git_repo, rev): |
| 538 | """Get the commit message for a given revision. |
| 539 | |
| 540 | Because git.Log doesn't allow specifying check=False or getting the |
| 541 | returncode, we have to catch the CalledProcessError instead. |
| 542 | |
| 543 | Args: |
| 544 | git_repo: Directory of git repository. |
| 545 | rev: The revision to search for, a SHA or a label. |
| 546 | |
| 547 | Returns: |
| 548 | The commit message as a list of strings, if the revision exists. |
| 549 | None if the revision was not found. |
| 550 | """ |
| 551 | try: |
| 552 | msg = git.Log(git_repo, max_count=1, format='format:%B', rev=rev) |
| 553 | return msg.splitlines() |
| 554 | except cros_build_lib.CalledProcessError: |
| 555 | raise ValueError('SHA was not found') |
| 556 | |
| 557 | |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 558 | def emerge_with_workon(status, workon_pkgs, emerge_cmd, emerge_pkgs, env=None): |
| 559 | """Emerge a list of packages after `cros_workon start`ing them |
| 560 | |
| 561 | This function will `cros_workon start` a list of packages, then `emerge` |
| 562 | another list of packages, and finally, `cros_workon stop` only those |
| 563 | packages that were actually started by the `cros_workon start` command. |
| 564 | Any package already in a `cros_workon start` state prior to this function |
| 565 | will still be in that state when this function exits. |
| 566 | |
| 567 | To determine which packages this program started and which ones were |
| 568 | already started, we query the list of packages being worked on, then |
| 569 | cros_workon start the entire list (which will produce a "package already |
| 570 | being worked on" type of message for anything already started), and then |
| 571 | query the list of packages being worked on again. The difference between |
| 572 | the before and after lists are the packages that this program started, |
| 573 | and so that's the list of packages to cros_workon stop after the emerge |
| 574 | is done. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 575 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 576 | Args: |
| 577 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 578 | workon_pkgs: list of packages to `cros_workon start` |
| 579 | emerge_cmd: the emerge command to run, e.g. 'emerge-volteer' |
| 580 | emerge_pkgs: list of packages to `emerge` |
| 581 | env: environment to pass to run_process, or None to pass default |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 582 | |
| 583 | Returns: |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 584 | True if everything succeeded, False if something failed |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 585 | """ |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 586 | # Get the list of packages that are already cros_workon started. |
| 587 | build_target = build_target_lib.BuildTarget(status.base) |
| 588 | workon = workon_helper.WorkonHelper(build_target.root, build_target.name) |
| 589 | before_workon = workon.ListAtoms() |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 590 | |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 591 | workon.StartWorkingOnPackages(workon_pkgs) |
| 592 | |
| 593 | # Determine which packages we need to cros_workon stop. |
| 594 | after_workon = workon.ListAtoms() |
| 595 | stop_packages = list(set(after_workon) - set(before_workon)) |
| 596 | |
| 597 | # Run the emerge command. |
| 598 | emerge_result = run_process([emerge_cmd] + emerge_pkgs, env=env) |
| 599 | |
| 600 | # cros_workon stop before returning the result. |
| 601 | workon.StopWorkingOnPackages(stop_packages) |
| 602 | |
| 603 | return emerge_result |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 604 | |
| 605 | |
Paul Fagerburg | bc18424 | 2020-04-28 17:00:54 -0600 | [diff] [blame] | 606 | def project_config(status): |
| 607 | """Check if the project config is correct and complete |
| 608 | |
| 609 | For programs that use the new project/config structure with starlark |
| 610 | configuration files, this function will check that emerging the |
| 611 | project's configuration will result in a project-config.json that |
| 612 | includes the new name of the new hwdesign (a.k.a. "variant"). |
| 613 | |
| 614 | Args: |
| 615 | status: variant_status object tracking our board, variant, etc. |
| 616 | |
| 617 | Returns: |
| 618 | True if everything succeeded, False if something failed |
| 619 | """ |
| 620 | logging.info('Running step project_config') |
| 621 | try: |
| 622 | if not emerge_with_workon(status, status.config_workon_pkgs, |
| 623 | status.emerge_cmd, status.config_emerge_pkgs): |
| 624 | raise RuntimeError(f'Building the configuration failed.') |
| 625 | |
| 626 | # Make sure project-config.json exists in the /build tree |
| 627 | emerged_json = os.path.join( |
| 628 | '/build', |
| 629 | status.base, |
| 630 | 'usr/share/chromeos-config/yaml/project-config.json') |
| 631 | if not os.path.isfile(emerged_json): |
| 632 | raise RuntimeError( |
| 633 | f'project-config.json {emerged_json} does not exist.') |
| 634 | |
| 635 | # Search emerged_json to see if the new variant's name shows up there. |
| 636 | if not status.variant in osutils.ReadFile(emerged_json): |
| 637 | raise RuntimeError( |
| 638 | f'variant name {status.variant} not found in {emerged_json}') |
| 639 | |
| 640 | except RuntimeError as e: |
| 641 | logging.error(str(e)) |
| 642 | logging.error('Please file a bug in Infra > ChromeOS > Product > Device' |
| 643 | ' to have the project configuration updated.') |
| 644 | logging.error('(https://bugs.chromium.org/p/chromium/issues/list?' |
| 645 | 'q=component:Infra%3EChromeOS%3EProduct%3EDevice)') |
| 646 | return False |
| 647 | |
| 648 | return True |
| 649 | |
| 650 | |
Paul Fagerburg | 8f60b80 | 2020-06-09 15:38:28 -0600 | [diff] [blame^] | 651 | def fw_build_config(status): |
| 652 | """Add the _FW_BUILD_CONFIG setting to the project config |
| 653 | |
| 654 | For programs that use the new project/config structure with starlark |
| 655 | configuration files, this function calls fw_build_config.sh, which will |
| 656 | modify the config.star file to have a default _FW_BUILD_CONFIG entry. |
| 657 | |
| 658 | Args: |
| 659 | status: variant_status object tracking our board, variant, etc. |
| 660 | |
| 661 | Returns: |
| 662 | True if everything succeeded, False if something failed |
| 663 | """ |
| 664 | logging.info('Running step fw_build_config') |
| 665 | fw_build_config_sh = os.path.join(status.my_loc, 'fw_build_config.sh') |
| 666 | rc = run_process( |
| 667 | [fw_build_config_sh, |
| 668 | status.base, |
| 669 | status.variant, |
| 670 | status.bug]) |
| 671 | if rc: |
| 672 | status.commits[step_names.FW_BUILD_CONFIG] = get_git_commit_data( |
| 673 | os.path.join('/mnt/host/source/src/project', status.base, status.variant)) |
| 674 | return rc |
| 675 | |
| 676 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 677 | def create_coreboot_variant(status): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 678 | """Create source files for a new variant of the reference board in coreboot |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 679 | |
| 680 | This function calls create_coreboot_variant.sh to set up a new variant |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 681 | of the reference board. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 682 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 683 | Args: |
| 684 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 685 | |
| 686 | Returns: |
| 687 | True if everything succeeded, False if something failed |
| 688 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 689 | logging.info('Running step create_coreboot_variant') |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 690 | cb_src_dir = os.path.join('/mnt/host/source/src/', status.coreboot_dir) |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 691 | environ = {**os.environ, 'CB_SRC_DIR': cb_src_dir} |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 692 | create_coreboot_variant_sh = os.path.join(status.my_loc, |
| 693 | 'create_coreboot_variant.sh') |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 694 | rc = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 695 | [create_coreboot_variant_sh, |
Paul Fagerburg | 3ef1cbb | 2020-06-02 16:38:36 -0600 | [diff] [blame] | 696 | status.coreboot_base, |
Paul Fagerburg | abb1562 | 2020-02-07 15:41:29 -0700 | [diff] [blame] | 697 | status.board, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 698 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 699 | status.bug], env=environ) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 700 | if rc: |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 701 | status.commits[step_names.CB_VARIANT] = get_git_commit_data(cb_src_dir) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 702 | return rc |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 703 | |
| 704 | |
| 705 | def create_coreboot_config(status): |
| 706 | """Create a coreboot configuration for a new variant |
| 707 | |
| 708 | This function calls create_coreboot_config.sh, which will make a copy |
| 709 | of coreboot.${BOARD} into coreboot.${VARIANT}. |
| 710 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 711 | Args: |
| 712 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 713 | |
| 714 | Returns: |
| 715 | True if the script and test build succeeded, False if something failed |
| 716 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 717 | logging.info('Running step create_coreboot_config') |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 718 | # Only set CB_CONFIG_DIR if it's not None, so here we have to copy |
| 719 | # the environment first and then optionally add a key. |
Paul Fagerburg | 4d34345 | 2020-01-24 15:23:53 -0700 | [diff] [blame] | 720 | environ = os.environ.copy() |
| 721 | if status.cb_config_dir is not None: |
| 722 | environ['CB_CONFIG_DIR'] = status.cb_config_dir |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 723 | create_coreboot_config_sh = os.path.join(status.my_loc, |
| 724 | 'create_coreboot_config.sh') |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 725 | rc = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 726 | [create_coreboot_config_sh, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 727 | status.base, |
| 728 | status.board, |
| 729 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 730 | status.bug], env=environ) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 731 | if rc: |
| 732 | # Use status.cb_config_dir if defined, or if not, use |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 733 | # '/mnt/host/source/src/third_party/chromiumos-overlay' |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 734 | if status.cb_config_dir is not None: |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 735 | cb_config_dir = os.path.join('/mnt/host/source/src/', status.cb_config_dir) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 736 | else: |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 737 | cb_config_dir = '/mnt/host/source/src/third_party/chromiumos-overlay' |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 738 | status.commits[step_names.CB_CONFIG] = get_git_commit_data(cb_config_dir) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 739 | return rc |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 740 | |
| 741 | |
Paul Fagerburg | 2fd23f4 | 2020-02-07 14:04:48 -0700 | [diff] [blame] | 742 | def copy_cras_config(status): |
| 743 | """Copy the cras config for a new variant |
| 744 | |
| 745 | This is only necessary for the Zork baseboard right now. |
| 746 | This function calls copy_cras_config.sh, which will copy the |
| 747 | cras config in |
| 748 | overlays/overlay-${BASE}/chromeos-base/chromeos-bsp-${BASE}/files/cras-config/${BASE} |
| 749 | to .../${VARIANT} |
| 750 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 751 | Args: |
| 752 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 2fd23f4 | 2020-02-07 14:04:48 -0700 | [diff] [blame] | 753 | |
| 754 | Returns: |
| 755 | True if the script and test build succeeded, False if something failed |
| 756 | """ |
| 757 | logging.info('Running step copy_cras_config') |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 758 | copy_cras_config_sh = os.path.join(status.my_loc, 'copy_cras_config.sh') |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 759 | rc = run_process( |
Paul Fagerburg | 2fd23f4 | 2020-02-07 14:04:48 -0700 | [diff] [blame] | 760 | [copy_cras_config_sh, |
| 761 | status.base, |
| 762 | status.board, |
| 763 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 764 | status.bug]) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 765 | if rc: |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 766 | status.commits[step_names.CRAS_CONFIG] = get_git_commit_data( |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 767 | '/mnt/host/source/src/overlays') |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 768 | return rc |
Paul Fagerburg | 2fd23f4 | 2020-02-07 14:04:48 -0700 | [diff] [blame] | 769 | |
| 770 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 771 | def add_fitimage(status): |
| 772 | """Add the source files for a fitimage for the new variant |
| 773 | |
| 774 | This function calls add_fitimage.sh to create a new XSL file for the |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 775 | variant's fitimage, which can override settings from the reference board's |
| 776 | XSL. When this is done, the user will have to build the fitimage by running |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 777 | gen_fit_image.sh outside of the chroot (and outside of this program's |
| 778 | control) because gen_fit_image.sh uses WINE, which is not installed in |
| 779 | the chroot. (There is a linux version of FIT, but it requires Open GL, |
| 780 | which is also not installed in the chroot.) |
| 781 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 782 | Args: |
| 783 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 784 | |
| 785 | Returns: |
| 786 | True if the script succeeded, False otherwise |
| 787 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 788 | logging.info('Running step add_fitimage') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 789 | add_fitimage_sh = os.path.expanduser(os.path.join( |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 790 | '/mnt/host/source/src', status.fitimage_dir, 'files/add_fitimage.sh')) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 791 | rc = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 792 | [add_fitimage_sh, |
| 793 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 794 | status.bug]) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 795 | if rc: |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 796 | fitimage_dir = os.path.join('/mnt/host/source/src', status.fitimage_dir) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 797 | status.commits[step_names.COMMIT_FIT] = get_git_commit_data(fitimage_dir) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 798 | return rc |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 799 | |
| 800 | |
| 801 | def gen_fit_image_outside_chroot(status): |
| 802 | """Tell the user to run gen_fit_image.sh outside the chroot |
| 803 | |
| 804 | As noted for add_Fitimage(), gen_fit_image.sh cannot run inside the |
| 805 | chroot. This function tells the user to run gen_fit_image.sh in |
| 806 | their normal environment, and then come back (--continue) when that |
| 807 | is done. |
| 808 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 809 | Args: |
| 810 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 811 | |
| 812 | Returns: |
| 813 | True |
| 814 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 815 | logging.info('Running step gen_fit_image_outside_chroot') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 816 | fit_image_files = check_fit_image_files(status) |
| 817 | # If the list is empty, then `not` of the list is True, so the files |
| 818 | # we need are all present and we can continue. |
| 819 | if not fit_image_files: |
| 820 | return True |
| 821 | |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 822 | logging.error('The following files need to be generated:') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 823 | for filename in fit_image_files: |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 824 | logging.error('* %s', filename) |
| 825 | logging.error('The fitimage sources are ready for gen_fit_image.sh to process.') |
| 826 | logging.error('gen_fit_image.sh cannot run inside the chroot. Please open a new terminal') |
| 827 | logging.error('window, change to the directory where gen_fit_image.sh is located, and run') |
| 828 | logging.error(status.fitimage_cmd, status.variant) |
| 829 | logging.error('Then re-start this program with --continue.') |
| 830 | logging.error('If your chroot is based in ~/chromiumos, then the folder you want is') |
| 831 | logging.error('~/chromiumos/src/%s/asset_generation', status.fitimage_dir) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 832 | return False |
| 833 | |
| 834 | |
| 835 | def check_fit_image_files(status): |
| 836 | """Check if the fitimage has been generated |
| 837 | |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 838 | This function is not called directly as a step, and so it doesn't need |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 839 | to produce any error messages to the user (except with --verbose). |
| 840 | gen_fit_image_outside_chroot will call this function to see if the |
| 841 | fitimage files exist, and if not, then that function will print the |
| 842 | message about how the user needs to run gen_fit_image.sh outside the |
| 843 | chroot. |
| 844 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 845 | Args: |
| 846 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 847 | |
| 848 | Returns: |
| 849 | List of files that *DO NOT* exist and need to be created, [] if |
| 850 | all files are present. |
| 851 | """ |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 852 | outputs_dir = os.path.join('/mnt/host/source/src', status.fitimage_dir, |
| 853 | 'asset_generation/outputs') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 854 | logging.debug('outputs_dir = "%s"', outputs_dir) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 855 | |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 856 | files_not_found = [] |
| 857 | fitimage_bin = 'fitimage-' + status.variant + '.bin' |
| 858 | if not os.path.isfile(os.path.join(outputs_dir, fitimage_bin)): |
| 859 | files_not_found.append(fitimage_bin) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 860 | |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 861 | fitimage_versions = 'fitimage-' + status.variant + '-versions.txt' |
| 862 | if not os.path.isfile(os.path.join(outputs_dir, fitimage_versions)): |
| 863 | files_not_found.append(fitimage_versions) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 864 | |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 865 | return files_not_found |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 866 | |
| 867 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 868 | def commit_fitimage(status): |
Paul Fagerburg | 8f71ac0 | 2020-04-22 10:53:45 -0600 | [diff] [blame] | 869 | """Add the fitimage files to the git commit |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 870 | |
Paul Fagerburg | 8f71ac0 | 2020-04-22 10:53:45 -0600 | [diff] [blame] | 871 | This function calls commit_fitimage.sh to move the fitimage binary and |
| 872 | -versions files from asset_generation/outputs to files/ and then adds |
| 873 | those files and fit.log to the existing git commit. |
| 874 | Depending on the baseboard, there may be different file names (such |
| 875 | as fit-${VARIANT}.log for volteer) and/or additional files (such as |
| 876 | files/blobs/description-${VARIANT}.bin for volteer) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 877 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 878 | Args: |
| 879 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 880 | |
| 881 | Returns: |
Paul Fagerburg | 8f71ac0 | 2020-04-22 10:53:45 -0600 | [diff] [blame] | 882 | True if the script succeeded, False if something failed |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 883 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 884 | logging.info('Running step commit_fitimage') |
Paul Fagerburg | 8f71ac0 | 2020-04-22 10:53:45 -0600 | [diff] [blame] | 885 | commit_fitimage_sh = os.path.expanduser(os.path.join( |
| 886 | '/mnt/host/source/src', status.fitimage_dir, 'files/commit_fitimage.sh')) |
| 887 | return run_process([commit_fitimage_sh, status.variant]) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 888 | |
| 889 | |
| 890 | def create_initial_ec_image(status): |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 891 | """Create an EC image for the variant as a clone of the reference board |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 892 | |
| 893 | This function calls create_initial_ec_image.sh, which will clone the |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 894 | reference board to create the variant. The shell script will build the |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 895 | EC code for the variant, but the repo upload hook insists that we |
| 896 | have done a `make buildall` before it will allow an upload, so this |
| 897 | function does the buildall. |
| 898 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 899 | Args: |
| 900 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 901 | |
| 902 | Returns: |
| 903 | True if the script and test build succeeded, False if something failed |
| 904 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 905 | logging.info('Running step create_initial_ec_image') |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 906 | create_initial_ec_image_sh = os.path.join(status.my_loc, |
| 907 | 'create_initial_ec_image.sh') |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 908 | if not run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 909 | [create_initial_ec_image_sh, |
| 910 | status.board, |
| 911 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 912 | status.bug]): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 913 | return False |
| 914 | |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 915 | # No need to `if rc:` because we already tested the run_process result above |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 916 | status.commits[step_names.EC_IMAGE] = get_git_commit_data( |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 917 | '/mnt/host/source/src/platform/ec/board') |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 918 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 919 | # create_initial_ec_image.sh will build the ec.bin for this variant |
| 920 | # if successful. |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 921 | ec_dir = '/mnt/host/source/src/platform/ec' |
| 922 | ec_bin = os.path.join(ec_dir, 'build', status.variant, 'ec.bin') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 923 | logging.debug('ec.bin = "%s"', ec_bin) |
| 924 | |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 925 | if not os.path.isfile(ec_bin): |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 926 | logging.error('EC binary %s not found', ec_bin) |
| 927 | return False |
| 928 | return True |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 929 | |
| 930 | |
| 931 | def ec_buildall(status): |
| 932 | """Do a make buildall -j for the EC, which is required for repo upload |
| 933 | |
| 934 | The upload hook checks to ensure that the entire EC codebase builds |
| 935 | without error, so we have to run make buildall -j before uploading. |
| 936 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 937 | Args: |
| 938 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 939 | |
| 940 | Returns: |
| 941 | True if the script and test build succeeded, False if something failed |
| 942 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 943 | logging.info('Running step ec_buildall') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 944 | del status # unused parameter |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 945 | ec = '/mnt/host/source/src/platform/ec' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 946 | logging.debug('ec = "%s"', ec) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 947 | return run_process(['make', 'buildall', '-j'], cwd=ec) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 948 | |
| 949 | |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 950 | def add_variant_to_public_yaml(status): |
| 951 | """Add the new variant to the public model.yaml file |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 952 | |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 953 | This function calls add_variant_to_yaml.sh to add the new variant to |
| 954 | the public model.yaml file. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 955 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 956 | Args: |
| 957 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 958 | |
| 959 | Returns: |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 960 | True if the script succeeded, False is something failed |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 961 | """ |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 962 | logging.info('Running step add_variant_to_public_yaml') |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 963 | add_variant_to_yaml_sh = os.path.join(status.my_loc, |
| 964 | 'add_variant_to_yaml.sh') |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 965 | rc = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 966 | [add_variant_to_yaml_sh, |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 967 | status.base, |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 968 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 969 | status.bug]) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 970 | if rc: |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 971 | status.commits[step_names.ADD_PUB_YAML] = get_git_commit_data( |
Paul Fagerburg | 5f794bf | 2020-02-12 13:01:36 -0700 | [diff] [blame] | 972 | '/mnt/host/source/src/overlays') |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 973 | return rc |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 974 | |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 975 | |
| 976 | def add_variant_to_private_yaml(status): |
| 977 | """Add the new variant to the private model.yaml file |
| 978 | |
| 979 | This function calls add_variant.sh to add the new variant to |
| 980 | the private model.yaml file. |
| 981 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 982 | Args: |
| 983 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 984 | |
| 985 | Returns: |
| 986 | True if the script succeeded, False is something failed |
| 987 | """ |
| 988 | logging.info('Running step add_variant_to_private_yaml') |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 989 | add_variant_sh = os.path.expanduser(os.path.join(status.private_yaml_dir, 'add_variant.sh')) |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 990 | rc = run_process( |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 991 | [add_variant_sh, |
| 992 | status.variant, |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 993 | status.bug]) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 994 | if rc: |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 995 | status.commits[step_names.ADD_PRIV_YAML] = get_git_commit_data(status.private_yaml_dir) |
Paul Fagerburg | 1d043c3 | 2020-02-03 08:57:08 -0700 | [diff] [blame] | 996 | return rc |
| 997 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 998 | |
| 999 | |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1000 | def build_config(status): |
| 1001 | """Build project config files, from yaml or starlark |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1002 | |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1003 | This function builds the project config files that mosys and other tools |
| 1004 | use, then verifies that the new variant's name shows up in all of the |
| 1005 | output files. Depending on the baseboard, the input may be the model.yaml |
| 1006 | files, or the starlark configuration files. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1007 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1008 | Args: |
| 1009 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1010 | |
| 1011 | Returns: |
| 1012 | True if the scripts and build succeeded, False is something failed |
| 1013 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1014 | logging.info('Running step build_yaml') |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1015 | if not emerge_with_workon(status, status.config_workon_pkgs, |
| 1016 | status.emerge_cmd, status.config_emerge_pkgs): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1017 | return False |
| 1018 | |
Paul Fagerburg | a95dd16 | 2020-03-24 16:27:18 -0600 | [diff] [blame] | 1019 | # Check the generated config.yaml file for occurences of the variant |
| 1020 | # name to determine if the emerge was successful. |
| 1021 | config_yaml = os.path.join( |
| 1022 | '/build', status.base, 'usr/share/chromeos-config/yaml/config.yaml') |
| 1023 | logging.debug('config_yaml = "%s"', config_yaml) |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 1024 | if not os.path.isfile(config_yaml): |
Paul Fagerburg | a95dd16 | 2020-03-24 16:27:18 -0600 | [diff] [blame] | 1025 | logging.error('%s does not exist', config_yaml) |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1026 | return False |
| 1027 | |
Paul Fagerburg | a95dd16 | 2020-03-24 16:27:18 -0600 | [diff] [blame] | 1028 | if not status.variant in osutils.ReadFile(config_yaml): |
| 1029 | logging.error('variant name %s not found in yaml file %s', |
| 1030 | status.variant, config_yaml) |
| 1031 | return False |
| 1032 | |
| 1033 | return True |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1034 | |
| 1035 | |
| 1036 | def emerge_all(status): |
| 1037 | """Build the coreboot BIOS and EC code for the new variant |
| 1038 | |
Paul Fagerburg | 7539807 | 2020-03-16 13:51:24 -0600 | [diff] [blame] | 1039 | This build step will cros_workon start a list of packages provided by |
| 1040 | the reference board data as status.workon_pkgs, then emerge a list of |
| 1041 | packages (status.emerge_pkgs), and then cros_workon stop any packages |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1042 | that it started. Any packages that were already being worked on will |
| 1043 | not be stopped. |
Paul Fagerburg | 7539807 | 2020-03-16 13:51:24 -0600 | [diff] [blame] | 1044 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1045 | Args: |
| 1046 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1047 | |
| 1048 | Returns: |
| 1049 | True if the build succeeded, False if something failed |
| 1050 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1051 | logging.info('Running step emerge_all') |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1052 | environ = {**os.environ, 'FW_NAME': status.variant} |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1053 | if not emerge_with_workon(status, status.workon_pkgs, |
| 1054 | status.emerge_cmd, status.emerge_pkgs, |
| 1055 | env=environ): |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1056 | return False |
| 1057 | |
Paul Fagerburg | 20559bc | 2020-04-29 10:37:14 -0600 | [diff] [blame] | 1058 | # Check if the expected build outputs exist. |
Paul Fagerburg | e868e83 | 2020-01-22 17:14:04 -0700 | [diff] [blame] | 1059 | build_path = '/build/' + status.base + '/firmware' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1060 | logging.debug('build_path = "%s"', build_path) |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 1061 | image_bin = 'image-' + status.variant + '.bin' |
| 1062 | if not os.path.isfile(os.path.join(build_path, image_bin)): |
| 1063 | logging.error('emerge failed because %s does not exist', image_bin) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1064 | return False |
| 1065 | |
Paul Fagerburg | 8021dab | 2020-03-25 21:23:33 -0600 | [diff] [blame] | 1066 | serial_bin = 'image-' + status.variant + '.serial.bin' |
| 1067 | if not os.path.isfile(os.path.join(build_path, serial_bin)): |
| 1068 | logging.error('emerge failed because %s does not exist', serial_bin) |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1069 | return False |
| 1070 | |
| 1071 | return True |
| 1072 | |
| 1073 | |
| 1074 | def push_coreboot(status): |
| 1075 | """Push the coreboot CL to coreboot.org |
| 1076 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1077 | Args: |
| 1078 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1079 | |
| 1080 | Returns: |
| 1081 | True if the build succeeded, False if something failed |
| 1082 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1083 | logging.info('Running step push_coreboot') |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1084 | |
| 1085 | # Set up a return code that may change to False if we find that a |
| 1086 | # coreboot CL has not been uploaded. |
| 1087 | rc = True |
| 1088 | |
| 1089 | for commit_key in status.coreboot_push_list: |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1090 | logging.debug('Processing key %s', commit_key) |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1091 | commit = status.commits[commit_key] |
| 1092 | if 'gerrit' not in commit or 'cl_number' not in commit: |
| 1093 | change_id = commit['change_id'] |
| 1094 | cl = find_change_id(change_id) |
| 1095 | if cl is not None: |
| 1096 | save_cl_data(status, commit_key, cl) |
| 1097 | else: |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1098 | logging.debug('Not found %s, need to upload', change_id) |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 1099 | logging.error('The following commit needs to be pushed to coreboot.org:') |
| 1100 | logging.error(' Branch "%s"', commit['branch_name']) |
| 1101 | logging.error(' in directory "%s"', commit['dir']) |
| 1102 | logging.error(' with change-id "%s"', commit['change_id']) |
| 1103 | logging.error('Please push the branch to review.coreboot.org, ' |
| 1104 | 'and then re-start this program with --continue') |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1105 | # Since this commit needs to be uploaded, do not continue after |
| 1106 | # this step returns. |
| 1107 | rc = False |
| 1108 | else: |
| 1109 | instance_name = commit['gerrit'] |
| 1110 | cl_number = commit['cl_number'] |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1111 | logging.debug('Already uploaded (%s, %s)', instance_name, cl_number) |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1112 | |
| 1113 | return rc |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1114 | |
| 1115 | |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1116 | def query_gerrit(instance, change_id): |
| 1117 | """Search a gerrit instance for a specific change_id |
| 1118 | |
| 1119 | Args: |
| 1120 | instance: gerrit instance to query. Suitable values come from |
| 1121 | gerrit.GetCrosInternal() and gerrit.GetCrosExternal() |
| 1122 | change_id: The change_id to search for |
| 1123 | |
| 1124 | Returns: |
| 1125 | CL number if found, None if not |
| 1126 | """ |
| 1127 | raw = instance.Query(change=change_id, raw=True) |
| 1128 | if raw: |
| 1129 | # If the CL was found by change_id, there will be only one, |
| 1130 | # because the change_id is used to recognize a new patchset |
| 1131 | # on an existing CL. |
| 1132 | return raw[0]['number'] |
| 1133 | |
| 1134 | return None |
| 1135 | |
| 1136 | |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1137 | def query_coreboot_gerrit(change_id): |
| 1138 | """Search the coreboot gerrit for a specific change_id |
| 1139 | |
| 1140 | Use the REST API to look for the change_id. See |
| 1141 | https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html |
| 1142 | for details on the REST API to search for a change-id. |
| 1143 | |
| 1144 | We can't use query_gerrit with a manually constructed GerritHelper |
| 1145 | because we need the user's private SSH key to access review.coreboot.org, |
| 1146 | but these are not available inside the chroot. |
| 1147 | |
| 1148 | Args: |
| 1149 | change_id: The change_id to search for |
| 1150 | |
| 1151 | Returns: |
| 1152 | CL number if found, None if not |
| 1153 | """ |
| 1154 | r = requests.get('https://review.coreboot.org/changes/' + change_id) |
| 1155 | response = r.content.decode('utf-8') |
| 1156 | # Check if the response starts with 'Not found', in which case return None |
| 1157 | if response.startswith('Not found:'): |
| 1158 | return None |
| 1159 | # Strip off the initial )]}'\n that is used as XSS protections, see |
| 1160 | # https://gerrit-review.googlesource.com/Documentation/rest-api.html#output |
| 1161 | # and decode as JSON. |
| 1162 | data = json.loads(response[5:]) |
| 1163 | if '_number' in data: |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 1164 | return str(data['_number']) |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1165 | return None |
| 1166 | |
| 1167 | |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1168 | def find_change_id(change_id): |
| 1169 | """Search the public and private ChromeOS gerrit instances for a change-id |
| 1170 | |
| 1171 | Args: |
| 1172 | change_id: Change-Id to search for in both gerrit instances |
| 1173 | |
| 1174 | Returns: |
| 1175 | Tuple of the gerrit instance ('chromium' or 'chrome-internal') and |
| 1176 | the CL number if the Change-Id is found. |
| 1177 | None if not found. |
| 1178 | """ |
| 1179 | cl_number = query_gerrit(gerrit.GetCrosExternal(), change_id) |
| 1180 | if cl_number: |
| 1181 | return 'chromium', cl_number |
| 1182 | cl_number = query_gerrit(gerrit.GetCrosInternal(), change_id) |
| 1183 | if cl_number: |
| 1184 | return 'chrome-internal', cl_number |
Paul Fagerburg | 8d85093 | 2020-02-25 14:13:32 -0700 | [diff] [blame] | 1185 | cl_number = query_coreboot_gerrit(change_id) |
| 1186 | if cl_number: |
| 1187 | return 'coreboot', cl_number |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1188 | return None |
| 1189 | |
| 1190 | |
| 1191 | def save_cl_data(status, commit_key, cl): |
| 1192 | """Save the gerrit instance and CL number to the yaml file |
| 1193 | |
| 1194 | Args: |
| 1195 | status: variant_status object tracking our board, variant, etc. |
| 1196 | commit_key: Which key in the commits map we're processing |
| 1197 | cl: Value returned by find_change_id, should be a tuple |
| 1198 | of instance_name, cl_number |
| 1199 | """ |
| 1200 | instance_name, cl_number = cl |
| 1201 | print(f'Found ({instance_name}, {cl_number}), saving to yaml') |
| 1202 | status.commits[commit_key]['gerrit'] = instance_name |
| 1203 | status.commits[commit_key]['cl_number'] = cl_number |
| 1204 | status.save() |
| 1205 | |
| 1206 | |
| 1207 | def repo_upload(branch_name, cwd): |
| 1208 | """Upload a branch to gerrit |
| 1209 | |
| 1210 | This function runs `repo upload` in the specified directory to upload |
| 1211 | a branch to gerrit. Because it's operating in a directory and with a |
| 1212 | branch name, it could upload more than one commit, which is OK because |
| 1213 | we'll look for each commit by change-id before trying to upload in that |
| 1214 | directory. For example, this happens in Zork, where the cb_config step |
| 1215 | and the cras_config step both have a commit in src/overlays. When we're |
| 1216 | processing the cb_config step and we `repo upload` in src/overlays, it |
| 1217 | will also upload the commit for cras_config. Then we come around to the |
| 1218 | cras_config step, and since we can find a CL with the change-id, we don't |
| 1219 | try to upload again. |
| 1220 | |
| 1221 | Args: |
| 1222 | branch_name: the name of the branch to upload. Gets passed to |
| 1223 | repo upload with the --br flag |
| 1224 | cwd: directory where we want to upload. Gets set as the working |
| 1225 | directory for executing repo upload. |
| 1226 | |
| 1227 | Returns: |
| 1228 | True if repo upload exits with a successful error code, false otherwise |
| 1229 | """ |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 1230 | return run_process( |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1231 | ['repo', |
| 1232 | 'upload', |
| 1233 | '.', |
| 1234 | '--br=' + branch_name, |
| 1235 | '--wip', |
| 1236 | '--verify', |
| 1237 | '--yes'], |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 1238 | cwd=cwd) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1239 | |
| 1240 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1241 | def upload_CLs(status): |
| 1242 | """Upload all CLs to chromiumos |
| 1243 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1244 | Args: |
| 1245 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1246 | |
| 1247 | Returns: |
| 1248 | True if the build succeeded, False if something failed |
| 1249 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1250 | logging.info('Running step upload_CLs') |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1251 | |
| 1252 | for commit_key in status.repo_upload_list: |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1253 | logging.debug('Processing key %s', commit_key) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1254 | commit = status.commits[commit_key] |
| 1255 | if 'gerrit' not in commit or 'cl_number' not in commit: |
| 1256 | change_id = commit['change_id'] |
| 1257 | cl = find_change_id(change_id) |
| 1258 | if cl is not None: |
| 1259 | save_cl_data(status, commit_key, cl) |
| 1260 | else: |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1261 | logging.debug('Not found %s, need to upload', change_id) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1262 | if not repo_upload(commit['branch_name'], commit['dir']): |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1263 | logging.error('Repo upload %s in %s failed!', |
| 1264 | commit['branch_name'], |
| 1265 | commit['dir']) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1266 | return False |
| 1267 | cl = find_change_id(change_id) |
| 1268 | if cl is None: |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1269 | logging.error('repo upload %s succeeded, but change_id is not found!', |
| 1270 | commit_key) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1271 | return False |
| 1272 | save_cl_data(status, commit_key, cl) |
| 1273 | else: |
| 1274 | instance_name = commit['gerrit'] |
| 1275 | cl_number = commit['cl_number'] |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1276 | logging.debug('Already uploaded (%s, %s)', instance_name, cl_number) |
Paul Fagerburg | a8c7e34 | 2020-02-25 13:30:49 -0700 | [diff] [blame] | 1277 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1278 | return True |
| 1279 | |
| 1280 | |
| 1281 | def find_coreboot_upstream(status): |
| 1282 | """Find the coreboot CL after it has been upstreamed to chromiumos |
| 1283 | |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 1284 | When the coreboot variant CL is first uploaded to review.coreboot.org, |
| 1285 | it is not visible in the chromiumos tree (and also cannot be used as |
| 1286 | a target for cq-depend). There is a process for upstreaming CLs from |
| 1287 | coreboot after they have been reviewed, approved, and merged. We can |
| 1288 | track a specific coreboot CL if we know the change-id that it used on |
| 1289 | the coreboot gerrit instance, by looking for that change-id as |
| 1290 | 'original-change-id' in the public chromium gerrit instance. |
| 1291 | |
| 1292 | The change-id for the coreboot variant will be under the 'cb_variant' key, |
| 1293 | but this is for the 'coreboot' gerrit instance. |
| 1294 | |
| 1295 | When we find the upstreamed CL, we will record the gerrit instance and |
| 1296 | CL number in the yaml file under the 'find' key ("find upstream coreboot") |
| 1297 | so that we don't need to search coreboot again. |
| 1298 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1299 | Args: |
| 1300 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1301 | |
| 1302 | Returns: |
| 1303 | True if the build succeeded, False if something failed |
| 1304 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1305 | logging.info('Running step find_coreboot_upstream') |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 1306 | |
| 1307 | # If we have already found the upstream coreboot CL, then exit with success |
| 1308 | if step_names.FIND in status.commits: |
| 1309 | commit = status.commits[step_names.FIND] |
| 1310 | if 'gerrit' in commit and 'cl_number' in commit: |
| 1311 | instance_name = commit['gerrit'] |
| 1312 | cl_number = commit['cl_number'] |
Paul Fagerburg | 6cba149 | 2020-06-02 16:44:05 -0600 | [diff] [blame] | 1313 | logging.debug('Already found (%s, %s)', instance_name, cl_number) |
Paul Fagerburg | aec8d99 | 2020-02-27 15:51:47 -0700 | [diff] [blame] | 1314 | return True |
| 1315 | |
| 1316 | # Make sure we have a CB_VARIANT commit and a change_id for it |
| 1317 | if step_names.CB_VARIANT not in status.commits: |
| 1318 | logging.error('Key %s not found in status.commits', |
| 1319 | step_names.CB_VARIANT) |
| 1320 | return False |
| 1321 | if 'change_id' not in status.commits[step_names.CB_VARIANT]: |
| 1322 | logging.error('Key change_id not found in status.commits[%s]', |
| 1323 | step_names.CB_VARIANT) |
| 1324 | return False |
| 1325 | |
| 1326 | # Find the CL by the Original-Change-Id |
| 1327 | original_change_id = status.commits[step_names.CB_VARIANT]['change_id'] |
| 1328 | gerrit_query_args = { |
| 1329 | 'Original-Change-Id': original_change_id |
| 1330 | } |
| 1331 | cros = gerrit.GetCrosExternal() |
| 1332 | upstream = cros.Query(**gerrit_query_args) |
| 1333 | # If nothing is found, the patch hasn't been upstreamed yet |
| 1334 | if not upstream: |
| 1335 | logging.error('Program cannot continue until coreboot CL is upstreamed.') |
| 1336 | logging.error('(coreboot:%s, change-id %s)', |
| 1337 | status.commits[step_names.CB_VARIANT]['cl_number'], |
| 1338 | status.commits[step_names.CB_VARIANT]['change_id']) |
| 1339 | logging.error('Please wait for the CL to be upstreamed, then run this' |
| 1340 | ' program again with --continue') |
| 1341 | return False |
| 1342 | |
| 1343 | # If more than one CL is found, something is very wrong |
| 1344 | if len(upstream) != 1: |
| 1345 | logging.error('More than one CL was found with Original-Change-Id %s', |
| 1346 | original_change_id) |
| 1347 | return False |
| 1348 | |
| 1349 | # At this point, we know there is only one CL and we can get the |
| 1350 | # repo and CL number by splitting on the colon between them. |
| 1351 | patchlink = upstream[0].PatchLink() |
| 1352 | instance_name, cl_number = patchlink.split(':') |
| 1353 | |
| 1354 | # Can't use get_git_commit_data because we're not pulling this |
| 1355 | # information from a git commit, but rather from gerrit. |
| 1356 | # We only need the gerrit instance and the CL number so we can have |
| 1357 | # other CLs cq-depend on this CL. The other keys are not needed because: |
| 1358 | # dir - not needed because we're not going to `cd` there to `repo upload` |
| 1359 | # branch_name - not valid; the CL is already merged |
| 1360 | # change_id - we use the change_id to find a CL number, and since we |
| 1361 | # just found the CL number via original-change-id, this is moot. |
| 1362 | status.commits[step_names.FIND] = { |
| 1363 | 'gerrit': instance_name, |
| 1364 | 'cl_number': str(cl_number) |
| 1365 | } |
| 1366 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1367 | return True |
| 1368 | |
| 1369 | |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1370 | def calc_cq_depend(status): |
| 1371 | """Determine the list of CLs for each commit that has dependencies. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1372 | |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1373 | status.depends is a map of dependencies from step name to a list of |
| 1374 | steps that the step depends on. For each step, find the SHA of the |
| 1375 | commit, then find the gerrit instance and CL number of the commits |
| 1376 | that it depends on. Construct the Cq-Depends list and save it under |
| 1377 | the 'cq_depend' key, i.e. commit['add_priv_yaml']['cq_depend'] or |
| 1378 | as it will be stored in the yaml: |
| 1379 | commits: |
| 1380 | add_priv_yaml: |
| 1381 | cq_depend: 'chromium:1629121, chromium:1638243' |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1382 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1383 | Args: |
| 1384 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1385 | |
| 1386 | Returns: |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1387 | True if all dependencies have been calculated. False if something |
| 1388 | failed, usually a commit not found by Change-Id. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1389 | """ |
Paul Fagerburg | f5f8c7e | 2020-03-25 10:06:52 -0600 | [diff] [blame] | 1390 | logging.info('Running step calc_cq_depend') |
| 1391 | # Iterate through the commits that have dependencies. |
| 1392 | for key in status.depends: |
| 1393 | logging.debug('Processing %s to add dependencies', key) |
| 1394 | # For every commit that has dependencies, find the gerrit instance |
| 1395 | # and CL number of the dependencies. |
| 1396 | cq_depend_list = [] |
| 1397 | for depend_key in status.depends[key]: |
| 1398 | depend_commit = status.commits[depend_key] |
| 1399 | if not 'gerrit' in depend_commit: |
| 1400 | logging.error('Commit %s does not have a gerrit instance', |
| 1401 | depend_key) |
| 1402 | return False |
| 1403 | if not 'cl_number' in depend_commit: |
| 1404 | logging.error('Commit %s does not have a CL number', |
| 1405 | depend_key) |
| 1406 | return False |
| 1407 | |
| 1408 | instance_name = depend_commit['gerrit'] |
| 1409 | cl_number = depend_commit['cl_number'] |
| 1410 | cq_depend_list.append(f'{instance_name}:{cl_number}') |
| 1411 | |
| 1412 | # Add the 'cq_depend' key to the commit. |
| 1413 | cq_depend_str = 'Cq-Depend: %s' % ', '.join(cq_depend_list) |
| 1414 | logging.debug('Add to commit %s %s', key, cq_depend_str) |
| 1415 | status.commits[key]['cq_depend'] = cq_depend_str |
| 1416 | |
| 1417 | return True |
| 1418 | |
| 1419 | |
| 1420 | def add_cq_depend_to_commit_msg(git_repo, change_id, cq_depend_str): |
| 1421 | """Update the commit message with a Cq-Depends line. |
| 1422 | |
| 1423 | Find the SHA of the commit, then use git filter-branch --msg-filter |
| 1424 | to add the Cq-Depend line just before the Change-Id line. See |
| 1425 | https://chromium.googlesource.com/chromiumos/docs/+/HEAD/contributing.md#cq-depend |
| 1426 | for details about Cq-Depend format and location. |
| 1427 | |
| 1428 | Args: |
| 1429 | git_repo: Directory of git repository. |
| 1430 | change_id: The Change-Id to search for. |
| 1431 | cq_depend_str: The Cq-Depend string. It must be in the correct format |
| 1432 | per chromeos documentation, ready to insert into the commit msg |
| 1433 | on the line before Change-Id. |
| 1434 | |
| 1435 | Returns: |
| 1436 | True if `git filter-branch` was successful. False if the command |
| 1437 | failed. |
| 1438 | """ |
| 1439 | logging.debug('find SHA of Change-Id %s in %s', change_id, git_repo) |
| 1440 | sha = change_id_to_sha(git_repo, change_id) |
| 1441 | if sha is None: |
| 1442 | logging.error('Cannot find the SHA for Change-Id %s in %s', |
| 1443 | change_id, git_repo) |
| 1444 | return False |
| 1445 | logging.debug('SHA = %s', sha) |
| 1446 | |
| 1447 | # Check if the commit message already has a Cq-Depend line. |
| 1448 | msg = get_commit_msg(git_repo, sha) |
| 1449 | if any('Cq-Depend' in tmpstr for tmpstr in msg): |
| 1450 | logging.debug('Already has Cq-Depend') |
| 1451 | return True |
| 1452 | |
| 1453 | # Use git filter-branch --msg-filter to add the Cq-Depend line just |
| 1454 | # before the Change-Id line. |
| 1455 | environ = {**os.environ, 'FILTER_BRANCH_SQUELCH_WARNING': '1'} |
| 1456 | cmd = [ |
| 1457 | 'git', |
| 1458 | 'filter-branch', |
| 1459 | '--msg-filter', |
| 1460 | f'sed -E "s/^(Change-Id: {change_id})$/{cq_depend_str}\\n\\1/"', |
| 1461 | '--', |
| 1462 | f'{sha}^..'] |
| 1463 | return run_process(cmd, cwd=git_repo, env=environ) |
| 1464 | |
| 1465 | |
| 1466 | def add_cq_depend(status): |
| 1467 | """Add Cq-Depend to commits and flag them for re-upload. |
| 1468 | |
| 1469 | Args: |
| 1470 | status: variant_status object tracking our board, variant, etc. |
| 1471 | |
| 1472 | Returns: |
| 1473 | True if the commit messages have been successfully amended, False if |
| 1474 | something failed. |
| 1475 | """ |
| 1476 | logging.info('Running step add_cq_depend') |
| 1477 | for key in status.commits: |
| 1478 | commit = status.commits[key] |
| 1479 | if 'cq_depend' in commit: |
| 1480 | logging.debug('%s has %s', key, commit['cq_depend']) |
| 1481 | # Make sure the commit has a working directory and a change_id |
| 1482 | # before trying to amend its commit message. |
| 1483 | if 'dir' not in commit or 'change_id' not in commit: |
| 1484 | logging.error('Missing dir and/or change_id from %s', key) |
| 1485 | return False |
| 1486 | |
| 1487 | if not add_cq_depend_to_commit_msg(commit['dir'], |
| 1488 | commit['change_id'], |
| 1489 | commit['cq_depend']): |
| 1490 | return False |
| 1491 | commit['needs_re_upload'] = True |
| 1492 | else: |
| 1493 | logging.debug('%s no dependencies', key) |
| 1494 | |
| 1495 | return True |
| 1496 | |
| 1497 | |
| 1498 | def re_upload(status): |
| 1499 | """Re-upload commits that have changed. |
| 1500 | |
| 1501 | Args: |
| 1502 | status: variant_status object tracking our board, variant, etc. |
| 1503 | |
| 1504 | Returns: |
| 1505 | True if the uploads succeeded. False if a repo upload failed. |
| 1506 | """ |
| 1507 | logging.info('Running step re_upload') |
| 1508 | for key in status.commits: |
| 1509 | commit = status.commits[key] |
| 1510 | if commit.get('needs_re_upload'): |
| 1511 | logging.debug('Re-upload branch %s in %s', commit['branch_name'], |
| 1512 | commit['dir']) |
| 1513 | if not repo_upload(commit['branch_name'], commit['dir']): |
| 1514 | logging.error('Repo upload %s in %s failed!', |
| 1515 | commit['branch_name'], |
| 1516 | commit['dir']) |
| 1517 | return False |
| 1518 | commit['needs_re_upload'] = False |
| 1519 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1520 | return True |
| 1521 | |
| 1522 | |
| 1523 | def clean_up(status): |
| 1524 | """Final clean-up, including delete the status file |
| 1525 | |
Paul Fagerburg | 92fcb4b | 2020-02-19 21:21:43 -0700 | [diff] [blame] | 1526 | Args: |
| 1527 | status: variant_status object tracking our board, variant, etc. |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1528 | |
| 1529 | Returns: |
| 1530 | True |
| 1531 | """ |
Paul Fagerburg | bab5dde | 2020-01-10 15:10:29 -0700 | [diff] [blame] | 1532 | logging.info('Running step clean_up') |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1533 | status.rm() |
| 1534 | return True |
| 1535 | |
| 1536 | |
Paul Fagerburg | 042a525 | 2020-03-16 21:49:18 -0600 | [diff] [blame] | 1537 | def abort(status): |
| 1538 | """Abort the creation of a new variant by abandoning commits |
| 1539 | |
| 1540 | When the user specifies the --abort flag, we override status.step to |
| 1541 | be 'abort' and there is no transition from 'abort' to anything else. |
| 1542 | We look at status.commits and for each key, see if we have already |
| 1543 | been in that directory and abandoned that specific branch. If not, |
| 1544 | abandon the commit and then add the branch+dir to a list of abandoned |
| 1545 | commits. We do this because some boards (such as Zork) can have multiple |
| 1546 | commits in the same directory and with the same branch name, and we only |
| 1547 | want to repo abandon that branch once. |
| 1548 | |
| 1549 | Args: |
| 1550 | status: variant_status object tracking our board, variant, etc. |
| 1551 | |
| 1552 | Returns: |
| 1553 | True |
| 1554 | """ |
| 1555 | logging.info('Running step abort') |
| 1556 | # Use the set 'abandoned' to keep track of each branch+dir abandoned. |
| 1557 | abandoned = set() |
| 1558 | for step in status.commits: |
| 1559 | logging.debug('Processing step %s', step) |
| 1560 | commit = status.commits[step] |
| 1561 | branch = commit['branch_name'] |
| 1562 | cwd = commit['dir'] |
| 1563 | if (branch, cwd) in abandoned: |
| 1564 | logging.debug('Branch %s in directory %s already abandoned', |
| 1565 | branch, cwd) |
| 1566 | else: |
| 1567 | logging.info('Abandoning branch %s in directory %s', |
| 1568 | branch, cwd) |
| 1569 | if run_process(['repo', 'abandon', branch, '.'], cwd=cwd): |
| 1570 | abandoned.add((branch, cwd)) |
| 1571 | else: |
| 1572 | logging.error('Error while abandoning branch %s', branch) |
| 1573 | return False |
| 1574 | |
| 1575 | return True |
| 1576 | |
| 1577 | |
Paul Fagerburg | 3b534f9 | 2019-11-07 15:05:22 -0700 | [diff] [blame] | 1578 | if __name__ == '__main__': |
| 1579 | sys.exit(not int(main())) |