blob: 6330129c2754fcf1849933aac1b067e9129c6e74 [file] [log] [blame]
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07003#
4# Copyright 2020 The Chromium OS Authors. All rights reserved.
5# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7#
Paul Fagerburge868e832020-01-22 17:14:04 -07008"""Create a new variant of an existing reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -07009
10This program will call all of the scripts that create the various pieces
11of a new variant. For example to create a new variant of the hatch base
12board, the following scripts are called:
13
Paul Fagerburg5f794bf2020-02-12 13:01:36 -070014* platform/dev/contrib/variant/create_coreboot_variant.sh
Paul Fagerburg3b534f92019-11-07 15:05:22 -070015* platform/dev/contrib/variant/create_coreboot_config.sh
16* private-overlays/baseboard-hatch-private/sys-boot/
17 * coreboot-private-files-hatch/files/add_fitimage.sh
18 * coreboot-private-files-hatch/asset_generation/gen_fit_image.sh
19 * Outside the chroot, because it uses WINE to run the FIT tools
20* platform/dev/contrib/variant/create_initial_ec_image.sh
21* platform/dev/contrib/variant/add_variant_to_yaml.sh
22* private-overlays/overlay-hatch-private/chromeos-base/
23 * chromeos-config-bsp-hatch-private/add_variant.sh
24
25Once the scripts are done, the following repos have changes
26
27* third_party/coreboot
28* third_party/chromiumos-overlay
29* private-overlays/baseboard-hatch-private
30* platform/ec
31* private-overlays/overlay-hatch-private
32* overlays
33
Paul Fagerburge868e832020-01-22 17:14:04 -070034The program has support for multiple reference boards, so the repos, directories,
35and scripts above can change depending on what the reference board is.
Paul Fagerburg3b534f92019-11-07 15:05:22 -070036"""
37
38from __future__ import print_function
39import argparse
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070040import importlib
Paul Fagerburg8d850932020-02-25 14:13:32 -070041import json
Paul Fagerburg3b534f92019-11-07 15:05:22 -070042import logging
43import os
Paul Fagerburg3b534f92019-11-07 15:05:22 -070044import subprocess
45import sys
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -060046from chromite.lib import build_target_lib
47from chromite.lib import cros_build_lib
Paul Fagerburg1d043c32020-02-03 08:57:08 -070048from chromite.lib import git
Paul Fagerburga8c7e342020-02-25 13:30:49 -070049from chromite.lib import gerrit
Paul Fagerburga95dd162020-03-24 16:27:18 -060050from chromite.lib import osutils
Paul Fagerburg75398072020-03-16 13:51:24 -060051from chromite.lib import workon_helper
Paul Fagerburg8d850932020-02-25 14:13:32 -070052import requests
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070053import step_names
Paul Fagerburg3b534f92019-11-07 15:05:22 -070054import variant_status
55
56
57def main():
Paul Fagerburge868e832020-01-22 17:14:04 -070058 """Create a new variant of an existing reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -070059
60 This program automates the creation of a new variant of an existing
Paul Fagerburge868e832020-01-22 17:14:04 -070061 reference board by calling various scripts that copy the reference board,
62 modify files for the new variant, stage commits, and upload to gerrit.
Paul Fagerburg3b534f92019-11-07 15:05:22 -070063
64 Note that one of the following is required:
65 * --continue
66 * --board=BOARD --variant=VARIANT [--bug=BUG]
67 """
Paul Fagerburg042a5252020-03-16 21:49:18 -060068 board, variant, bug, continue_flag, abort_flag = get_args()
Paul Fagerburg3b534f92019-11-07 15:05:22 -070069
Paul Fagerburg042a5252020-03-16 21:49:18 -060070 if not check_flags(board, variant, bug, continue_flag, abort_flag):
Paul Fagerburg3b534f92019-11-07 15:05:22 -070071 return False
72
Paul Fagerburg042a5252020-03-16 21:49:18 -060073 status = get_status(board, variant, bug, continue_flag, abort_flag)
Paul Fagerburg3b534f92019-11-07 15:05:22 -070074 if status is None:
75 return False
76
77 status.load()
78
Paul Fagerburg5f794bf2020-02-12 13:01:36 -070079 # Where is new_variant.py located?
80 status.my_loc = os.path.dirname(os.path.abspath(__file__))
81
Paul Fagerburg042a5252020-03-16 21:49:18 -060082 # If the user specified --abort, override the current step.
83 if abort_flag:
84 status.step = step_names.ABORT
85
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070086 while status.step is not None:
Paul Fagerburg3b534f92019-11-07 15:05:22 -070087 status.save()
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070088 if not perform_step(status):
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -060089 logging.debug('perform_step %s returned False; exiting ...',
90 status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -070091 return False
92
Paul Fagerburgbab5dde2020-01-10 15:10:29 -070093 move_to_next_step(status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -070094
95 return True
96
97
98def get_args():
99 """Parse the command-line arguments
100
101 There doesn't appear to be a way to specify that --continue is
102 mutually exclusive with --board, --variant, and --bug. As a result,
103 all arguments are optional, and another function will apply the logic
104 to check if there is an illegal combination of arguments.
105
106 Returns a list of:
Paul Fagerburge868e832020-01-22 17:14:04 -0700107 board Name of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700108 variant Name of the variant being created
109 bug Text for bug number, if any ('None' otherwise)
110 continue_flag Flag if --continue was specified
111 """
112 parser = argparse.ArgumentParser(
113 description=main.__doc__,
114 formatter_class=argparse.RawTextHelpFormatter)
Paul Fagerburge868e832020-01-22 17:14:04 -0700115 parser.add_argument('--board', type=str, help='Name of the reference board')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700116 parser.add_argument(
117 '--variant', type=str, help='Name of the new variant to create')
118 parser.add_argument(
119 '--bug', type=str, help='Bug number to reference in commits')
Paul Fagerburg2e48a192020-03-24 17:57:05 -0600120 # Use a group so that we can enforce mutually-exclusive argurments.
121 # argparse does not support nesting groups, so we can't put board,
122 # variant, and bug into a group and have that group as another mutually
123 # exclusive option.
124 group = parser.add_mutually_exclusive_group()
125 group.add_argument(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700126 '--continue', action='store_true',
127 dest='continue_flag', help='Continue the process from where it paused')
Paul Fagerburg2e48a192020-03-24 17:57:05 -0600128 group.add_argument(
Paul Fagerburg042a5252020-03-16 21:49:18 -0600129 '--abort', action='store_true',
130 dest='abort_flag', help='Cancel the process and abandon all commits')
131 parser.add_argument(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700132 '--verbose', action='store_true',
133 dest='verbose_flag', help='Enable verbose output of progress')
134 args = parser.parse_args()
135
136 if args.verbose_flag:
137 logging.basicConfig(level=logging.DEBUG)
138 else:
139 logging.basicConfig(level=logging.INFO)
140
141 board = args.board
142 if board is not None:
143 board = board.lower()
144
145 variant = args.variant
146 if variant is not None:
147 variant = variant.lower()
148
149 bug = args.bug or 'None'
150
Paul Fagerburg042a5252020-03-16 21:49:18 -0600151 return (board, variant, bug, args.continue_flag, args.abort_flag)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700152
153
Paul Fagerburg042a5252020-03-16 21:49:18 -0600154def check_flags(board, variant, bug, continue_flag, abort_flag):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700155 """Check the flags to ensure no invalid combinations
156
157 We allow any of the following:
Paul Fagerburg042a5252020-03-16 21:49:18 -0600158 --abort
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700159 --continue
160 --board=board_name --variant=variant_name
161 --board=board_name --variant=variant_name --bug=bug_text
162
163 The argument parser does have the functionality to represent the
164 combination of --board and --variant as a single mutually-exclusive
165 argument, so we have to use this function to do the checking.
166
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700167 Args:
168 board: Name of the reference board
169 variant: Name of the variant being created
170 bug: Text for bug number, if any ('None' otherwise)
171 continue_flag: Flag if --continue was specified
Paul Fagerburg042a5252020-03-16 21:49:18 -0600172 abort_flag: Flag if --abort was specified
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700173
174 Returns:
175 True if the arguments are acceptable, False otherwise
176 """
Paul Fagerburg042a5252020-03-16 21:49:18 -0600177 # If either --abort or --continue is set, then disallow any of the
178 # board name, variant name, or bug number to be set.
179 if continue_flag or abort_flag:
180 if board is not None or variant is not None or bug != 'None':
181 logging.error('Do not use --board, --variant, or --bug with '
182 '--continue or --abort')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700183 return False
Paul Fagerburg042a5252020-03-16 21:49:18 -0600184 return True
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700185
Paul Fagerburg042a5252020-03-16 21:49:18 -0600186 # At this point, neither --continue nor --abort are set, so we must have
187 # both --board and --variant values.
188 if board is None or variant is None:
189 logging.error('Both --board and --variant must be specified')
190 return False
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700191
192 return True
193
194
Paul Fagerburg042a5252020-03-16 21:49:18 -0600195def get_status(board, variant, bug, continue_flag, abort_flag):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700196 """Create the status file or get the previous status
197
198 This program can stop at several places as we have to wait for CLs
199 to work through CQ or be upstreamed into the chromiumos tree, so just
200 like a git cherry-pick, there is a --continue option to pick up where
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700201 you left off by reading a specially-named status file.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700202
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700203 If --continue is specified, the status file must exist.
204 If the status file exists, then --continue must be specified.
205 When --continue is specified, we read the status file and return
206 with the contents.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700207
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700208 If the status file does not exist, we will create the state file with
209 the board, variant, and (optional) bug details.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700210
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700211 To decouple the list of boards supported from this main program, we
Paul Fagerburge868e832020-01-22 17:14:04 -0700212 try to import a module with the same name as the reference board,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700213 so --board=hatch means that we import hatch.py. If we can't import
Paul Fagerburge868e832020-01-22 17:14:04 -0700214 the file, then we don't support that reference board.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700215
216 The board-specific module will set several variables, which we will
217 copy into the object that we return.
218
Paul Fagerburge868e832020-01-22 17:14:04 -0700219 * base - the name of the base board, such as Hatch, Volteer, or Zork.
220 This can be different from the reference board, e.g. the Trembyle
221 reference board in the Zork project.
Paul Fagerburg3ef1cbb2020-06-02 16:38:36 -0600222 * coreboot_base - the name of the base board in coreboot. Usually the same
223 as base, but can differ, e.g. for Puff, the base is Puff, but the
224 coreboot_base is Hatch because Puff is based on Hatch.
Paul Fagerburg4d343452020-01-24 15:23:53 -0700225 * coreboot_dir - base directory for coreboot, usually third_party/coreboot
226 but could differ for processors that use a private repo
227 * cb_config_dir - base directory for coreboot configs, usually
228 third_party/chromiumos-overlay/sys-boot/coreboot/files/configs but
229 could differ for processors that use a private repo
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700230 * step_list - list of steps (named in step_names.py) to run in sequence
Paul Fagerburge868e832020-01-22 17:14:04 -0700231 to create the new variant of the reference board
232 * fsp - package name for FSP. This may be None, depending on the
233 processor on the reference board
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700234 * fitimage_pkg - package name for the fitimage
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700235 * fitimage_dir - directory for fitimage; prepend '/mnt/host/source/src/'
236 in chroot, prepend '~/chromiumos/src' outside the chroot
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700237 * workon_pkgs - list of packages to cros_workon
238 * emerge_cmd - the emerge command, e.g. 'emerge-hatch'
239 * emerge_pkgs - list of packages to emerge
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600240 * config_workon_pkgs - list of packages to cros_workon to build the
241 project config
242 * config_emerge_pkgs - list of packages to emerge to build the project
243 config
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700244 * private_yaml_dir - directory for the private yaml file
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700245 * commits - map of commits for the various steps. Indexed by step name,
246 and the step names used are the same ones in step_names.py
247 * repo_upload_list - list of commits to upload using `repo upload`
248 * coreboot_push_list - list of commits to upload using `git push` to
249 coreboot
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600250 * depends - maps a step to a list of steps on which it depends, e.g.
251 depends[step_names.ADD_PRIV_YAML] is a list of other steps that
252 the 'add_priv_yaml' step depends on. This map is used to amend
253 the commits with CL numbers for Cq-Depend.
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700254
255 Additionally, the following fields will be set:
256
Paul Fagerburge868e832020-01-22 17:14:04 -0700257 * board - the name of the reference board, e.g. 'hatch'
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700258 * variant - the name of the variant, e.g. 'sushi'
259 * bug - optional text for a bug ID, used in the git commit messages.
260 Could be 'None' (as text, not the python None), or something like
261 'b:12345' for buganizer, or 'chromium:12345'
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700262 * step - internal state tracking, what step of the variant creation
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700263 we are at.
264 * yaml_file - internal, just the name of the file where all this data
265 gets saved.
Paul Fagerburgaec8d992020-02-27 15:51:47 -0700266 * commit - a map of maps that tracks all of the git commit and gerrit CL
267 data for each of the steps in the process. For example,
268 status.commit['add_priv_yaml'] is a map that has all the information
269 about the 'add_priv_yaml' step. The keys in the maps allow us to
270 determine where the commit is, the change_id, if it has been uploaded
271 to gerrit and where.
272
273 branch_name - the name of the git branch
274 change_id - the change-id assigned by the commit hook. Gerrit
275 uses the change_id to track new patchsets in the CL
276 dir - the directory where the commit has been created
277 gerrit - the name of the gerrit instance to which the CL has
278 been uploaded, one of 'chromium', 'chrome-internal', or
279 'coreboot'
280 cl_number - the CL number on the gerrit instance
281
282 When the commit is created, branch_name, change_id, and dir are all
283 set. The gerrit and cl_number keys are not set until the CL has been
284 uploaded to a gerrit instance.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700285
286 These data might come from the status file (because we read it), or
287 they might be the initial values after we created the file (because
288 it did not already exist).
289
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700290 Args:
291 board: Name of the reference board
292 variant: Name of the variant being created
293 bug: Text for bug number, if any ('None' otherwise)
294 continue_flag: Flag if --continue was specified
Paul Fagerburg042a5252020-03-16 21:49:18 -0600295 abort_flag: Flag if --abort was specified
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700296
297 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700298 variant_status object with all the data mentioned above
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700299 """
300 status = variant_status.variant_status()
Paul Fagerburg042a5252020-03-16 21:49:18 -0600301 if continue_flag or abort_flag:
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700302 if status.yaml_file_exists():
Paul Fagerburg042a5252020-03-16 21:49:18 -0600303 return status
Paul Fagerburg6cba1492020-06-02 16:44:05 -0600304
305 if continue_flag:
306 op = '--continue'
307 if abort_flag:
308 op = '--abort'
309 logging.error('%s does not exist; cannot %s', status.yaml_file, op)
310 return None
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700311
Paul Fagerburg042a5252020-03-16 21:49:18 -0600312 # If we get here, the user provided --board and --variant (because
313 # check_flags() returned Trued), but the yaml file already exists,
314 # so we print an error message and bail.
315 if status.yaml_file_exists():
316 logging.error(
317 'new_variant already in progress; did you forget --continue?')
318 return None
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700319
Paul Fagerburg042a5252020-03-16 21:49:18 -0600320 # At this point, it's not --continue, not --abort, the yaml file doesn't
321 # exist, and we have valid values for --board, --variant, and --bug (bug
322 # might be the default value of "None"). Create the yaml file with data
323 # from the reference board's loadable module.
324 status.board = board
325 status.variant = variant
326 status.bug = bug
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700327
Paul Fagerburg042a5252020-03-16 21:49:18 -0600328 # Load the appropriate module and copy all the data from it.
329 try:
330 module = importlib.import_module(board)
331 except ImportError:
332 print('Unsupported board "' + board + '"')
333 sys.exit(1)
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700334
Paul Fagerburg042a5252020-03-16 21:49:18 -0600335 # pylint: disable=bad-whitespace
336 # Allow extra spaces around = so that we can line things up nicely
337 status.base = module.base
Paul Fagerburg3ef1cbb2020-06-02 16:38:36 -0600338 status.coreboot_base = getattr(module, 'coreboot_base', module.base)
Paul Fagerburg042a5252020-03-16 21:49:18 -0600339 status.coreboot_dir = module.coreboot_dir
Paul Fagerburg6cba1492020-06-02 16:44:05 -0600340 status.cb_config_dir = getattr(module, 'cb_config_dir', None)
Paul Fagerburg042a5252020-03-16 21:49:18 -0600341 status.emerge_cmd = module.emerge_cmd
342 status.emerge_pkgs = module.emerge_pkgs
Paul Fagerburg6cba1492020-06-02 16:44:05 -0600343 status.fitimage_dir = getattr(module, 'fitimage_dir', None)
344 status.fitimage_pkg = getattr(module, 'fitimage_pkg', None)
345 status.fitimage_cmd = getattr(module, 'fitimage_cmd', None)
346 status.fsp = getattr(module, 'fsp', None)
347 status.private_yaml_dir = getattr(module, 'private_yaml_dir', None)
Paul Fagerburg042a5252020-03-16 21:49:18 -0600348 status.step_list = module.step_list
349 status.workon_pkgs = module.workon_pkgs
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600350 status.config_workon_pkgs = module.config_workon_pkgs
351 status.config_emerge_pkgs = module.config_emerge_pkgs
Paul Fagerburg042a5252020-03-16 21:49:18 -0600352 status.coreboot_push_list = module.coreboot_push_list
353 status.repo_upload_list = module.repo_upload_list
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600354 status.depends = module.depends
Paul Fagerburg042a5252020-03-16 21:49:18 -0600355 # pylint: enable=bad-whitespace
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700356
Paul Fagerburg042a5252020-03-16 21:49:18 -0600357 # Start at the first entry in the step list
358 status.step = status.step_list[0]
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700359
Paul Fagerburg042a5252020-03-16 21:49:18 -0600360 # Start an empty map for tracking CL data
361 status.commits = {}
362
363 status.save()
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700364
365 return status
366
367
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700368def perform_step(status):
369 """Call the appropriate function for the current step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700370
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700371 Args:
372 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700373
374 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700375 True if the step succeeded, False if it failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700376 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700377 # Function to call based on the step
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700378 dispatch = {
Paul Fagerburgbc184242020-04-28 17:00:54 -0600379 step_names.PROJECT_CONFIG: project_config,
Paul Fagerburg8f60b802020-06-09 15:38:28 -0600380 step_names.FW_BUILD_CONFIG: fw_build_config,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700381 step_names.CB_VARIANT: create_coreboot_variant,
382 step_names.CB_CONFIG: create_coreboot_config,
Paul Fagerburg2fd23f42020-02-07 14:04:48 -0700383 step_names.CRAS_CONFIG: copy_cras_config,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700384 step_names.ADD_FIT: add_fitimage,
385 step_names.GEN_FIT: gen_fit_image_outside_chroot,
386 step_names.COMMIT_FIT: commit_fitimage,
387 step_names.EC_IMAGE: create_initial_ec_image,
388 step_names.EC_BUILDALL: ec_buildall,
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700389 step_names.ADD_PUB_YAML: add_variant_to_public_yaml,
390 step_names.ADD_PRIV_YAML: add_variant_to_private_yaml,
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600391 step_names.BUILD_CONFIG: build_config,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700392 step_names.EMERGE: emerge_all,
393 step_names.PUSH: push_coreboot,
394 step_names.UPLOAD: upload_CLs,
395 step_names.FIND: find_coreboot_upstream,
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600396 step_names.CALC_CQ_DEPEND: calc_cq_depend,
397 step_names.ADD_CQ_DEPEND: add_cq_depend,
398 step_names.RE_UPLOAD: re_upload,
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700399 step_names.CLEAN_UP: clean_up,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600400 step_names.ABORT: abort,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700401 }
402
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700403 if status.step not in dispatch:
404 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700405 sys.exit(1)
406
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700407 return dispatch[status.step](status)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700408
409
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700410def move_to_next_step(status):
411 """Move to the next step in the list
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700412
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700413 Args:
414 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700415 """
Paul Fagerburg042a5252020-03-16 21:49:18 -0600416 # Special case: the next step after 'abort' is 'clean_up'. Always.
417 if status.step == step_names.ABORT:
418 status.step = step_names.CLEAN_UP
419 return
420
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700421 if status.step not in status.step_list:
422 logging.error('Unknown step "%s", aborting...', status.step)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700423 sys.exit(1)
424
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700425 idx = status.step_list.index(status.step)
426 if idx == len(status.step_list)-1:
427 status.step = None
Paul Fagerburgdd3e9c32020-01-07 14:00:35 -0700428 else:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700429 status.step = status.step_list[idx+1]
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700430
431
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700432def run_process(args, cwd=None, env=None, capture_output=False):
433 """Run a process, log debug messages, return text output of process
434
435 The capture_output parameter allows us to capture the output when we
436 care about it (and not sending it to the screen), or ignoring it when
437 we don't care, and letting the user see the output so they know that
438 the build is still running, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700439
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700440 Args:
441 args: List of the command and its params
442 cwd: If not None, cd to this directory before running
443 env: Environment to use for execution; if needed, get os.environ.copy()
444 and add variables. If None, just use the current environment
445 capture_output: True if we should capture the stdout, false if we
446 just care about success or not.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700447
448 Returns:
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700449 If capture_output == True, we return the text output from running
450 the subprocess as a list of lines, or None if the process failed.
451 If capture_output == False, we return a True if it successed, or
452 None if the process failed.
453
454 The caller can evaluate as a bool, because bool(None) == False, and
455 bool() of a non-empty list is True, or the caller can use the returned
456 text for further processing.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700457 """
458 logging.debug('Run %s', str(args))
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700459 if cwd is not None:
460 logging.debug('cwd = %s', cwd)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700461 try:
462 if capture_output:
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700463 output = subprocess.run(args, cwd=cwd, env=env, check=True,
464 stderr=subprocess.STDOUT, stdout=subprocess.PIPE).stdout
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700465 else:
466 subprocess.run(args, cwd=cwd, env=env, check=True)
467 # Just something to decode so we don't get an empty list
468 output = b'True'
469
470 logging.debug('process returns 0')
471 # Convert from byte string to ASCII
472 decoded = output.decode('utf-8')
473 # Split into array of individual lines
474 lines = decoded.split('\n')
475 return lines
476 except subprocess.CalledProcessError as err:
477 logging.debug('process returns %s', str(err.returncode))
478 return None
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700479
480
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700481def get_git_commit_data(cwd):
482 """Get the branch name and change id of the current commit
483
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700484 Args:
485 cwd: The current working directory, where we want to get the branch
486 name and change id
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700487
488 Returns:
489 Map with 'dir', 'branch_name' and 'change_id' keys. The 'dir'
490 key maps to the value of os.path.expanduser(cwd)
491 """
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700492 cwd = git.FindGitTopLevel(os.path.expanduser(cwd))
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700493 logging.debug('get_git_commit_data(%s)', cwd)
494
495 branch_name = git.GetCurrentBranch(cwd)
496 if branch_name is None:
497 logging.error('Cannot determine git branch name in %s; exiting', cwd)
498 sys.exit(1)
499 logging.debug('git current branch is %s', branch_name)
500
501 change_id = git.GetChangeId(cwd)
502 if change_id is None:
503 logging.error('Cannot determine Change-Id in %s; exiting', cwd)
504 sys.exit(1)
505 logging.debug('git Change-Id is %s', change_id)
506
507 return {
508 'dir': cwd,
509 'branch_name': branch_name,
510 'change_id': change_id
511 }
512
513
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600514def change_id_to_sha(git_repo, change_id):
515 """Find the SHA for a given Change-Id.
516
517 Args:
518 git_repo: Directory of git repository.
519 change_id: The Change-Id to search for.
520
521 Returns:
522 The SHA hash for the Change-Id if only one commit is found.
523 None if the Change-Id was not found.
524 Raises a ValueError if more than one commit is found with the
525 same Change-Id.
526 """
527 output = git.Log(git_repo, max_count=1, format='format:%H',
528 grep=fr'^Change-Id: {change_id}$')
529 sha_hashes = output.splitlines()
530 if not sha_hashes:
531 return None
532 if len(sha_hashes) > 1:
533 raise ValueError('More than one SHA with that Change-Id found')
534 return sha_hashes[0]
535
536
537def get_commit_msg(git_repo, rev):
538 """Get the commit message for a given revision.
539
540 Because git.Log doesn't allow specifying check=False or getting the
541 returncode, we have to catch the CalledProcessError instead.
542
543 Args:
544 git_repo: Directory of git repository.
545 rev: The revision to search for, a SHA or a label.
546
547 Returns:
548 The commit message as a list of strings, if the revision exists.
549 None if the revision was not found.
550 """
551 try:
552 msg = git.Log(git_repo, max_count=1, format='format:%B', rev=rev)
553 return msg.splitlines()
554 except cros_build_lib.CalledProcessError:
555 raise ValueError('SHA was not found')
556
557
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600558def emerge_with_workon(status, workon_pkgs, emerge_cmd, emerge_pkgs, env=None):
559 """Emerge a list of packages after `cros_workon start`ing them
560
561 This function will `cros_workon start` a list of packages, then `emerge`
562 another list of packages, and finally, `cros_workon stop` only those
563 packages that were actually started by the `cros_workon start` command.
564 Any package already in a `cros_workon start` state prior to this function
565 will still be in that state when this function exits.
566
567 To determine which packages this program started and which ones were
568 already started, we query the list of packages being worked on, then
569 cros_workon start the entire list (which will produce a "package already
570 being worked on" type of message for anything already started), and then
571 query the list of packages being worked on again. The difference between
572 the before and after lists are the packages that this program started,
573 and so that's the list of packages to cros_workon stop after the emerge
574 is done.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700575
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700576 Args:
577 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600578 workon_pkgs: list of packages to `cros_workon start`
579 emerge_cmd: the emerge command to run, e.g. 'emerge-volteer'
580 emerge_pkgs: list of packages to `emerge`
581 env: environment to pass to run_process, or None to pass default
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700582
583 Returns:
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600584 True if everything succeeded, False if something failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700585 """
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600586 # Get the list of packages that are already cros_workon started.
587 build_target = build_target_lib.BuildTarget(status.base)
588 workon = workon_helper.WorkonHelper(build_target.root, build_target.name)
589 before_workon = workon.ListAtoms()
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700590
Paul Fagerburg20559bc2020-04-29 10:37:14 -0600591 workon.StartWorkingOnPackages(workon_pkgs)
592
593 # Determine which packages we need to cros_workon stop.
594 after_workon = workon.ListAtoms()
595 stop_packages = list(set(after_workon) - set(before_workon))
596
597 # Run the emerge command.
598 emerge_result = run_process([emerge_cmd] + emerge_pkgs, env=env)
599
600 # cros_workon stop before returning the result.
601 workon.StopWorkingOnPackages(stop_packages)
602
603 return emerge_result
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700604
605
Paul Fagerburgbc184242020-04-28 17:00:54 -0600606def project_config(status):
607 """Check if the project config is correct and complete
608
609 For programs that use the new project/config structure with starlark
610 configuration files, this function will check that emerging the
611 project's configuration will result in a project-config.json that
612 includes the new name of the new hwdesign (a.k.a. "variant").
613
614 Args:
615 status: variant_status object tracking our board, variant, etc.
616
617 Returns:
618 True if everything succeeded, False if something failed
619 """
620 logging.info('Running step project_config')
621 try:
622 if not emerge_with_workon(status, status.config_workon_pkgs,
623 status.emerge_cmd, status.config_emerge_pkgs):
624 raise RuntimeError(f'Building the configuration failed.')
625
626 # Make sure project-config.json exists in the /build tree
627 emerged_json = os.path.join(
628 '/build',
629 status.base,
630 'usr/share/chromeos-config/yaml/project-config.json')
631 if not os.path.isfile(emerged_json):
632 raise RuntimeError(
633 f'project-config.json {emerged_json} does not exist.')
634
635 # Search emerged_json to see if the new variant's name shows up there.
636 if not status.variant in osutils.ReadFile(emerged_json):
637 raise RuntimeError(
638 f'variant name {status.variant} not found in {emerged_json}')
639
640 except RuntimeError as e:
641 logging.error(str(e))
642 logging.error('Please file a bug in Infra > ChromeOS > Product > Device'
643 ' to have the project configuration updated.')
644 logging.error('(https://bugs.chromium.org/p/chromium/issues/list?'
645 'q=component:Infra%3EChromeOS%3EProduct%3EDevice)')
646 return False
647
648 return True
649
650
Paul Fagerburg8f60b802020-06-09 15:38:28 -0600651def fw_build_config(status):
652 """Add the _FW_BUILD_CONFIG setting to the project config
653
654 For programs that use the new project/config structure with starlark
655 configuration files, this function calls fw_build_config.sh, which will
656 modify the config.star file to have a default _FW_BUILD_CONFIG entry.
657
658 Args:
659 status: variant_status object tracking our board, variant, etc.
660
661 Returns:
662 True if everything succeeded, False if something failed
663 """
664 logging.info('Running step fw_build_config')
665 fw_build_config_sh = os.path.join(status.my_loc, 'fw_build_config.sh')
666 rc = run_process(
667 [fw_build_config_sh,
668 status.base,
669 status.variant,
670 status.bug])
671 if rc:
672 status.commits[step_names.FW_BUILD_CONFIG] = get_git_commit_data(
673 os.path.join('/mnt/host/source/src/project', status.base, status.variant))
674 return rc
675
676
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700677def create_coreboot_variant(status):
Paul Fagerburge868e832020-01-22 17:14:04 -0700678 """Create source files for a new variant of the reference board in coreboot
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700679
680 This function calls create_coreboot_variant.sh to set up a new variant
Paul Fagerburge868e832020-01-22 17:14:04 -0700681 of the reference board.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700682
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700683 Args:
684 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700685
686 Returns:
687 True if everything succeeded, False if something failed
688 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700689 logging.info('Running step create_coreboot_variant')
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700690 cb_src_dir = os.path.join('/mnt/host/source/src/', status.coreboot_dir)
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600691 environ = {**os.environ, 'CB_SRC_DIR': cb_src_dir}
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700692 create_coreboot_variant_sh = os.path.join(status.my_loc,
693 'create_coreboot_variant.sh')
Paul Fagerburg042a5252020-03-16 21:49:18 -0600694 rc = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700695 [create_coreboot_variant_sh,
Paul Fagerburg3ef1cbb2020-06-02 16:38:36 -0600696 status.coreboot_base,
Paul Fagerburgabb15622020-02-07 15:41:29 -0700697 status.board,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700698 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600699 status.bug], env=environ)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700700 if rc:
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700701 status.commits[step_names.CB_VARIANT] = get_git_commit_data(cb_src_dir)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700702 return rc
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700703
704
705def create_coreboot_config(status):
706 """Create a coreboot configuration for a new variant
707
708 This function calls create_coreboot_config.sh, which will make a copy
709 of coreboot.${BOARD} into coreboot.${VARIANT}.
710
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700711 Args:
712 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700713
714 Returns:
715 True if the script and test build succeeded, False if something failed
716 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700717 logging.info('Running step create_coreboot_config')
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -0600718 # Only set CB_CONFIG_DIR if it's not None, so here we have to copy
719 # the environment first and then optionally add a key.
Paul Fagerburg4d343452020-01-24 15:23:53 -0700720 environ = os.environ.copy()
721 if status.cb_config_dir is not None:
722 environ['CB_CONFIG_DIR'] = status.cb_config_dir
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700723 create_coreboot_config_sh = os.path.join(status.my_loc,
724 'create_coreboot_config.sh')
Paul Fagerburg042a5252020-03-16 21:49:18 -0600725 rc = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700726 [create_coreboot_config_sh,
Paul Fagerburge868e832020-01-22 17:14:04 -0700727 status.base,
728 status.board,
729 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600730 status.bug], env=environ)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700731 if rc:
732 # Use status.cb_config_dir if defined, or if not, use
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700733 # '/mnt/host/source/src/third_party/chromiumos-overlay'
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700734 if status.cb_config_dir is not None:
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700735 cb_config_dir = os.path.join('/mnt/host/source/src/', status.cb_config_dir)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700736 else:
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700737 cb_config_dir = '/mnt/host/source/src/third_party/chromiumos-overlay'
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700738 status.commits[step_names.CB_CONFIG] = get_git_commit_data(cb_config_dir)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700739 return rc
Paul Fagerburge868e832020-01-22 17:14:04 -0700740
741
Paul Fagerburg2fd23f42020-02-07 14:04:48 -0700742def copy_cras_config(status):
743 """Copy the cras config for a new variant
744
745 This is only necessary for the Zork baseboard right now.
746 This function calls copy_cras_config.sh, which will copy the
747 cras config in
748 overlays/overlay-${BASE}/chromeos-base/chromeos-bsp-${BASE}/files/cras-config/${BASE}
749 to .../${VARIANT}
750
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700751 Args:
752 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg2fd23f42020-02-07 14:04:48 -0700753
754 Returns:
755 True if the script and test build succeeded, False if something failed
756 """
757 logging.info('Running step copy_cras_config')
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700758 copy_cras_config_sh = os.path.join(status.my_loc, 'copy_cras_config.sh')
Paul Fagerburg042a5252020-03-16 21:49:18 -0600759 rc = run_process(
Paul Fagerburg2fd23f42020-02-07 14:04:48 -0700760 [copy_cras_config_sh,
761 status.base,
762 status.board,
763 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600764 status.bug])
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700765 if rc:
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700766 status.commits[step_names.CRAS_CONFIG] = get_git_commit_data(
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700767 '/mnt/host/source/src/overlays')
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700768 return rc
Paul Fagerburg2fd23f42020-02-07 14:04:48 -0700769
770
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700771def add_fitimage(status):
772 """Add the source files for a fitimage for the new variant
773
774 This function calls add_fitimage.sh to create a new XSL file for the
Paul Fagerburge868e832020-01-22 17:14:04 -0700775 variant's fitimage, which can override settings from the reference board's
776 XSL. When this is done, the user will have to build the fitimage by running
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700777 gen_fit_image.sh outside of the chroot (and outside of this program's
778 control) because gen_fit_image.sh uses WINE, which is not installed in
779 the chroot. (There is a linux version of FIT, but it requires Open GL,
780 which is also not installed in the chroot.)
781
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700782 Args:
783 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700784
785 Returns:
786 True if the script succeeded, False otherwise
787 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700788 logging.info('Running step add_fitimage')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700789 add_fitimage_sh = os.path.expanduser(os.path.join(
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700790 '/mnt/host/source/src', status.fitimage_dir, 'files/add_fitimage.sh'))
Paul Fagerburg042a5252020-03-16 21:49:18 -0600791 rc = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700792 [add_fitimage_sh,
793 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600794 status.bug])
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700795 if rc:
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700796 fitimage_dir = os.path.join('/mnt/host/source/src', status.fitimage_dir)
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700797 status.commits[step_names.COMMIT_FIT] = get_git_commit_data(fitimage_dir)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700798 return rc
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700799
800
801def gen_fit_image_outside_chroot(status):
802 """Tell the user to run gen_fit_image.sh outside the chroot
803
804 As noted for add_Fitimage(), gen_fit_image.sh cannot run inside the
805 chroot. This function tells the user to run gen_fit_image.sh in
806 their normal environment, and then come back (--continue) when that
807 is done.
808
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700809 Args:
810 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700811
812 Returns:
813 True
814 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700815 logging.info('Running step gen_fit_image_outside_chroot')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700816 fit_image_files = check_fit_image_files(status)
817 # If the list is empty, then `not` of the list is True, so the files
818 # we need are all present and we can continue.
819 if not fit_image_files:
820 return True
821
Paul Fagerburgaec8d992020-02-27 15:51:47 -0700822 logging.error('The following files need to be generated:')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700823 for filename in fit_image_files:
Paul Fagerburgaec8d992020-02-27 15:51:47 -0700824 logging.error('* %s', filename)
825 logging.error('The fitimage sources are ready for gen_fit_image.sh to process.')
826 logging.error('gen_fit_image.sh cannot run inside the chroot. Please open a new terminal')
827 logging.error('window, change to the directory where gen_fit_image.sh is located, and run')
828 logging.error(status.fitimage_cmd, status.variant)
829 logging.error('Then re-start this program with --continue.')
830 logging.error('If your chroot is based in ~/chromiumos, then the folder you want is')
831 logging.error('~/chromiumos/src/%s/asset_generation', status.fitimage_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700832 return False
833
834
835def check_fit_image_files(status):
836 """Check if the fitimage has been generated
837
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700838 This function is not called directly as a step, and so it doesn't need
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700839 to produce any error messages to the user (except with --verbose).
840 gen_fit_image_outside_chroot will call this function to see if the
841 fitimage files exist, and if not, then that function will print the
842 message about how the user needs to run gen_fit_image.sh outside the
843 chroot.
844
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700845 Args:
846 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700847
848 Returns:
849 List of files that *DO NOT* exist and need to be created, [] if
850 all files are present.
851 """
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700852 outputs_dir = os.path.join('/mnt/host/source/src', status.fitimage_dir,
853 'asset_generation/outputs')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700854 logging.debug('outputs_dir = "%s"', outputs_dir)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700855
Paul Fagerburg8021dab2020-03-25 21:23:33 -0600856 files_not_found = []
857 fitimage_bin = 'fitimage-' + status.variant + '.bin'
858 if not os.path.isfile(os.path.join(outputs_dir, fitimage_bin)):
859 files_not_found.append(fitimage_bin)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700860
Paul Fagerburg8021dab2020-03-25 21:23:33 -0600861 fitimage_versions = 'fitimage-' + status.variant + '-versions.txt'
862 if not os.path.isfile(os.path.join(outputs_dir, fitimage_versions)):
863 files_not_found.append(fitimage_versions)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700864
Paul Fagerburg8021dab2020-03-25 21:23:33 -0600865 return files_not_found
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700866
867
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700868def commit_fitimage(status):
Paul Fagerburg8f71ac02020-04-22 10:53:45 -0600869 """Add the fitimage files to the git commit
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700870
Paul Fagerburg8f71ac02020-04-22 10:53:45 -0600871 This function calls commit_fitimage.sh to move the fitimage binary and
872 -versions files from asset_generation/outputs to files/ and then adds
873 those files and fit.log to the existing git commit.
874 Depending on the baseboard, there may be different file names (such
875 as fit-${VARIANT}.log for volteer) and/or additional files (such as
876 files/blobs/description-${VARIANT}.bin for volteer)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700877
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700878 Args:
879 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700880
881 Returns:
Paul Fagerburg8f71ac02020-04-22 10:53:45 -0600882 True if the script succeeded, False if something failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700883 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700884 logging.info('Running step commit_fitimage')
Paul Fagerburg8f71ac02020-04-22 10:53:45 -0600885 commit_fitimage_sh = os.path.expanduser(os.path.join(
886 '/mnt/host/source/src', status.fitimage_dir, 'files/commit_fitimage.sh'))
887 return run_process([commit_fitimage_sh, status.variant])
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700888
889
890def create_initial_ec_image(status):
Paul Fagerburge868e832020-01-22 17:14:04 -0700891 """Create an EC image for the variant as a clone of the reference board
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700892
893 This function calls create_initial_ec_image.sh, which will clone the
Paul Fagerburge868e832020-01-22 17:14:04 -0700894 reference board to create the variant. The shell script will build the
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700895 EC code for the variant, but the repo upload hook insists that we
896 have done a `make buildall` before it will allow an upload, so this
897 function does the buildall.
898
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700899 Args:
900 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700901
902 Returns:
903 True if the script and test build succeeded, False if something failed
904 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700905 logging.info('Running step create_initial_ec_image')
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700906 create_initial_ec_image_sh = os.path.join(status.my_loc,
907 'create_initial_ec_image.sh')
Paul Fagerburg042a5252020-03-16 21:49:18 -0600908 if not run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700909 [create_initial_ec_image_sh,
910 status.board,
911 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600912 status.bug]):
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700913 return False
914
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700915 # No need to `if rc:` because we already tested the run_process result above
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700916 status.commits[step_names.EC_IMAGE] = get_git_commit_data(
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700917 '/mnt/host/source/src/platform/ec/board')
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700918
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700919 # create_initial_ec_image.sh will build the ec.bin for this variant
920 # if successful.
Paul Fagerburg8021dab2020-03-25 21:23:33 -0600921 ec_dir = '/mnt/host/source/src/platform/ec'
922 ec_bin = os.path.join(ec_dir, 'build', status.variant, 'ec.bin')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700923 logging.debug('ec.bin = "%s"', ec_bin)
924
Paul Fagerburg8021dab2020-03-25 21:23:33 -0600925 if not os.path.isfile(ec_bin):
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700926 logging.error('EC binary %s not found', ec_bin)
927 return False
928 return True
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700929
930
931def ec_buildall(status):
932 """Do a make buildall -j for the EC, which is required for repo upload
933
934 The upload hook checks to ensure that the entire EC codebase builds
935 without error, so we have to run make buildall -j before uploading.
936
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700937 Args:
938 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700939
940 Returns:
941 True if the script and test build succeeded, False if something failed
942 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700943 logging.info('Running step ec_buildall')
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700944 del status # unused parameter
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700945 ec = '/mnt/host/source/src/platform/ec'
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700946 logging.debug('ec = "%s"', ec)
Paul Fagerburg042a5252020-03-16 21:49:18 -0600947 return run_process(['make', 'buildall', '-j'], cwd=ec)
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700948
949
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700950def add_variant_to_public_yaml(status):
951 """Add the new variant to the public model.yaml file
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700952
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700953 This function calls add_variant_to_yaml.sh to add the new variant to
954 the public model.yaml file.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700955
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700956 Args:
957 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700958
959 Returns:
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700960 True if the script succeeded, False is something failed
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700961 """
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700962 logging.info('Running step add_variant_to_public_yaml')
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700963 add_variant_to_yaml_sh = os.path.join(status.my_loc,
964 'add_variant_to_yaml.sh')
Paul Fagerburg042a5252020-03-16 21:49:18 -0600965 rc = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700966 [add_variant_to_yaml_sh,
Paul Fagerburge868e832020-01-22 17:14:04 -0700967 status.base,
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700968 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600969 status.bug])
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700970 if rc:
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700971 status.commits[step_names.ADD_PUB_YAML] = get_git_commit_data(
Paul Fagerburg5f794bf2020-02-12 13:01:36 -0700972 '/mnt/host/source/src/overlays')
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700973 return rc
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700974
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700975
976def add_variant_to_private_yaml(status):
977 """Add the new variant to the private model.yaml file
978
979 This function calls add_variant.sh to add the new variant to
980 the private model.yaml file.
981
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -0700982 Args:
983 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700984
985 Returns:
986 True if the script succeeded, False is something failed
987 """
988 logging.info('Running step add_variant_to_private_yaml')
Paul Fagerburgbab5dde2020-01-10 15:10:29 -0700989 add_variant_sh = os.path.expanduser(os.path.join(status.private_yaml_dir, 'add_variant.sh'))
Paul Fagerburg042a5252020-03-16 21:49:18 -0600990 rc = run_process(
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700991 [add_variant_sh,
992 status.variant,
Paul Fagerburg042a5252020-03-16 21:49:18 -0600993 status.bug])
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700994 if rc:
Paul Fagerburga8c7e342020-02-25 13:30:49 -0700995 status.commits[step_names.ADD_PRIV_YAML] = get_git_commit_data(status.private_yaml_dir)
Paul Fagerburg1d043c32020-02-03 08:57:08 -0700996 return rc
997
Paul Fagerburg3b534f92019-11-07 15:05:22 -0700998
999
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001000def build_config(status):
1001 """Build project config files, from yaml or starlark
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001002
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001003 This function builds the project config files that mosys and other tools
1004 use, then verifies that the new variant's name shows up in all of the
1005 output files. Depending on the baseboard, the input may be the model.yaml
1006 files, or the starlark configuration files.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001007
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001008 Args:
1009 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001010
1011 Returns:
1012 True if the scripts and build succeeded, False is something failed
1013 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001014 logging.info('Running step build_yaml')
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001015 if not emerge_with_workon(status, status.config_workon_pkgs,
1016 status.emerge_cmd, status.config_emerge_pkgs):
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001017 return False
1018
Paul Fagerburga95dd162020-03-24 16:27:18 -06001019 # Check the generated config.yaml file for occurences of the variant
1020 # name to determine if the emerge was successful.
1021 config_yaml = os.path.join(
1022 '/build', status.base, 'usr/share/chromeos-config/yaml/config.yaml')
1023 logging.debug('config_yaml = "%s"', config_yaml)
Paul Fagerburg8021dab2020-03-25 21:23:33 -06001024 if not os.path.isfile(config_yaml):
Paul Fagerburga95dd162020-03-24 16:27:18 -06001025 logging.error('%s does not exist', config_yaml)
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001026 return False
1027
Paul Fagerburga95dd162020-03-24 16:27:18 -06001028 if not status.variant in osutils.ReadFile(config_yaml):
1029 logging.error('variant name %s not found in yaml file %s',
1030 status.variant, config_yaml)
1031 return False
1032
1033 return True
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001034
1035
1036def emerge_all(status):
1037 """Build the coreboot BIOS and EC code for the new variant
1038
Paul Fagerburg75398072020-03-16 13:51:24 -06001039 This build step will cros_workon start a list of packages provided by
1040 the reference board data as status.workon_pkgs, then emerge a list of
1041 packages (status.emerge_pkgs), and then cros_workon stop any packages
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001042 that it started. Any packages that were already being worked on will
1043 not be stopped.
Paul Fagerburg75398072020-03-16 13:51:24 -06001044
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001045 Args:
1046 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001047
1048 Returns:
1049 True if the build succeeded, False if something failed
1050 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001051 logging.info('Running step emerge_all')
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001052 environ = {**os.environ, 'FW_NAME': status.variant}
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001053 if not emerge_with_workon(status, status.workon_pkgs,
1054 status.emerge_cmd, status.emerge_pkgs,
1055 env=environ):
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001056 return False
1057
Paul Fagerburg20559bc2020-04-29 10:37:14 -06001058 # Check if the expected build outputs exist.
Paul Fagerburge868e832020-01-22 17:14:04 -07001059 build_path = '/build/' + status.base + '/firmware'
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001060 logging.debug('build_path = "%s"', build_path)
Paul Fagerburg8021dab2020-03-25 21:23:33 -06001061 image_bin = 'image-' + status.variant + '.bin'
1062 if not os.path.isfile(os.path.join(build_path, image_bin)):
1063 logging.error('emerge failed because %s does not exist', image_bin)
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001064 return False
1065
Paul Fagerburg8021dab2020-03-25 21:23:33 -06001066 serial_bin = 'image-' + status.variant + '.serial.bin'
1067 if not os.path.isfile(os.path.join(build_path, serial_bin)):
1068 logging.error('emerge failed because %s does not exist', serial_bin)
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001069 return False
1070
1071 return True
1072
1073
1074def push_coreboot(status):
1075 """Push the coreboot CL to coreboot.org
1076
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001077 Args:
1078 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001079
1080 Returns:
1081 True if the build succeeded, False if something failed
1082 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001083 logging.info('Running step push_coreboot')
Paul Fagerburg8d850932020-02-25 14:13:32 -07001084
1085 # Set up a return code that may change to False if we find that a
1086 # coreboot CL has not been uploaded.
1087 rc = True
1088
1089 for commit_key in status.coreboot_push_list:
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001090 logging.debug('Processing key %s', commit_key)
Paul Fagerburg8d850932020-02-25 14:13:32 -07001091 commit = status.commits[commit_key]
1092 if 'gerrit' not in commit or 'cl_number' not in commit:
1093 change_id = commit['change_id']
1094 cl = find_change_id(change_id)
1095 if cl is not None:
1096 save_cl_data(status, commit_key, cl)
1097 else:
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001098 logging.debug('Not found %s, need to upload', change_id)
Paul Fagerburgaec8d992020-02-27 15:51:47 -07001099 logging.error('The following commit needs to be pushed to coreboot.org:')
1100 logging.error(' Branch "%s"', commit['branch_name'])
1101 logging.error(' in directory "%s"', commit['dir'])
1102 logging.error(' with change-id "%s"', commit['change_id'])
1103 logging.error('Please push the branch to review.coreboot.org, '
1104 'and then re-start this program with --continue')
Paul Fagerburg8d850932020-02-25 14:13:32 -07001105 # Since this commit needs to be uploaded, do not continue after
1106 # this step returns.
1107 rc = False
1108 else:
1109 instance_name = commit['gerrit']
1110 cl_number = commit['cl_number']
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001111 logging.debug('Already uploaded (%s, %s)', instance_name, cl_number)
Paul Fagerburg8d850932020-02-25 14:13:32 -07001112
1113 return rc
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001114
1115
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001116def query_gerrit(instance, change_id):
1117 """Search a gerrit instance for a specific change_id
1118
1119 Args:
1120 instance: gerrit instance to query. Suitable values come from
1121 gerrit.GetCrosInternal() and gerrit.GetCrosExternal()
1122 change_id: The change_id to search for
1123
1124 Returns:
1125 CL number if found, None if not
1126 """
1127 raw = instance.Query(change=change_id, raw=True)
1128 if raw:
1129 # If the CL was found by change_id, there will be only one,
1130 # because the change_id is used to recognize a new patchset
1131 # on an existing CL.
1132 return raw[0]['number']
1133
1134 return None
1135
1136
Paul Fagerburg8d850932020-02-25 14:13:32 -07001137def query_coreboot_gerrit(change_id):
1138 """Search the coreboot gerrit for a specific change_id
1139
1140 Use the REST API to look for the change_id. See
1141 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html
1142 for details on the REST API to search for a change-id.
1143
1144 We can't use query_gerrit with a manually constructed GerritHelper
1145 because we need the user's private SSH key to access review.coreboot.org,
1146 but these are not available inside the chroot.
1147
1148 Args:
1149 change_id: The change_id to search for
1150
1151 Returns:
1152 CL number if found, None if not
1153 """
1154 r = requests.get('https://review.coreboot.org/changes/' + change_id)
1155 response = r.content.decode('utf-8')
1156 # Check if the response starts with 'Not found', in which case return None
1157 if response.startswith('Not found:'):
1158 return None
1159 # Strip off the initial )]}'\n that is used as XSS protections, see
1160 # https://gerrit-review.googlesource.com/Documentation/rest-api.html#output
1161 # and decode as JSON.
1162 data = json.loads(response[5:])
1163 if '_number' in data:
Paul Fagerburgaec8d992020-02-27 15:51:47 -07001164 return str(data['_number'])
Paul Fagerburg8d850932020-02-25 14:13:32 -07001165 return None
1166
1167
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001168def find_change_id(change_id):
1169 """Search the public and private ChromeOS gerrit instances for a change-id
1170
1171 Args:
1172 change_id: Change-Id to search for in both gerrit instances
1173
1174 Returns:
1175 Tuple of the gerrit instance ('chromium' or 'chrome-internal') and
1176 the CL number if the Change-Id is found.
1177 None if not found.
1178 """
1179 cl_number = query_gerrit(gerrit.GetCrosExternal(), change_id)
1180 if cl_number:
1181 return 'chromium', cl_number
1182 cl_number = query_gerrit(gerrit.GetCrosInternal(), change_id)
1183 if cl_number:
1184 return 'chrome-internal', cl_number
Paul Fagerburg8d850932020-02-25 14:13:32 -07001185 cl_number = query_coreboot_gerrit(change_id)
1186 if cl_number:
1187 return 'coreboot', cl_number
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001188 return None
1189
1190
1191def save_cl_data(status, commit_key, cl):
1192 """Save the gerrit instance and CL number to the yaml file
1193
1194 Args:
1195 status: variant_status object tracking our board, variant, etc.
1196 commit_key: Which key in the commits map we're processing
1197 cl: Value returned by find_change_id, should be a tuple
1198 of instance_name, cl_number
1199 """
1200 instance_name, cl_number = cl
1201 print(f'Found ({instance_name}, {cl_number}), saving to yaml')
1202 status.commits[commit_key]['gerrit'] = instance_name
1203 status.commits[commit_key]['cl_number'] = cl_number
1204 status.save()
1205
1206
1207def repo_upload(branch_name, cwd):
1208 """Upload a branch to gerrit
1209
1210 This function runs `repo upload` in the specified directory to upload
1211 a branch to gerrit. Because it's operating in a directory and with a
1212 branch name, it could upload more than one commit, which is OK because
1213 we'll look for each commit by change-id before trying to upload in that
1214 directory. For example, this happens in Zork, where the cb_config step
1215 and the cras_config step both have a commit in src/overlays. When we're
1216 processing the cb_config step and we `repo upload` in src/overlays, it
1217 will also upload the commit for cras_config. Then we come around to the
1218 cras_config step, and since we can find a CL with the change-id, we don't
1219 try to upload again.
1220
1221 Args:
1222 branch_name: the name of the branch to upload. Gets passed to
1223 repo upload with the --br flag
1224 cwd: directory where we want to upload. Gets set as the working
1225 directory for executing repo upload.
1226
1227 Returns:
1228 True if repo upload exits with a successful error code, false otherwise
1229 """
Paul Fagerburg042a5252020-03-16 21:49:18 -06001230 return run_process(
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001231 ['repo',
1232 'upload',
1233 '.',
1234 '--br=' + branch_name,
1235 '--wip',
1236 '--verify',
1237 '--yes'],
Paul Fagerburg042a5252020-03-16 21:49:18 -06001238 cwd=cwd)
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001239
1240
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001241def upload_CLs(status):
1242 """Upload all CLs to chromiumos
1243
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001244 Args:
1245 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001246
1247 Returns:
1248 True if the build succeeded, False if something failed
1249 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001250 logging.info('Running step upload_CLs')
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001251
1252 for commit_key in status.repo_upload_list:
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001253 logging.debug('Processing key %s', commit_key)
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001254 commit = status.commits[commit_key]
1255 if 'gerrit' not in commit or 'cl_number' not in commit:
1256 change_id = commit['change_id']
1257 cl = find_change_id(change_id)
1258 if cl is not None:
1259 save_cl_data(status, commit_key, cl)
1260 else:
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001261 logging.debug('Not found %s, need to upload', change_id)
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001262 if not repo_upload(commit['branch_name'], commit['dir']):
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001263 logging.error('Repo upload %s in %s failed!',
1264 commit['branch_name'],
1265 commit['dir'])
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001266 return False
1267 cl = find_change_id(change_id)
1268 if cl is None:
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001269 logging.error('repo upload %s succeeded, but change_id is not found!',
1270 commit_key)
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001271 return False
1272 save_cl_data(status, commit_key, cl)
1273 else:
1274 instance_name = commit['gerrit']
1275 cl_number = commit['cl_number']
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001276 logging.debug('Already uploaded (%s, %s)', instance_name, cl_number)
Paul Fagerburga8c7e342020-02-25 13:30:49 -07001277
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001278 return True
1279
1280
1281def find_coreboot_upstream(status):
1282 """Find the coreboot CL after it has been upstreamed to chromiumos
1283
Paul Fagerburgaec8d992020-02-27 15:51:47 -07001284 When the coreboot variant CL is first uploaded to review.coreboot.org,
1285 it is not visible in the chromiumos tree (and also cannot be used as
1286 a target for cq-depend). There is a process for upstreaming CLs from
1287 coreboot after they have been reviewed, approved, and merged. We can
1288 track a specific coreboot CL if we know the change-id that it used on
1289 the coreboot gerrit instance, by looking for that change-id as
1290 'original-change-id' in the public chromium gerrit instance.
1291
1292 The change-id for the coreboot variant will be under the 'cb_variant' key,
1293 but this is for the 'coreboot' gerrit instance.
1294
1295 When we find the upstreamed CL, we will record the gerrit instance and
1296 CL number in the yaml file under the 'find' key ("find upstream coreboot")
1297 so that we don't need to search coreboot again.
1298
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001299 Args:
1300 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001301
1302 Returns:
1303 True if the build succeeded, False if something failed
1304 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001305 logging.info('Running step find_coreboot_upstream')
Paul Fagerburgaec8d992020-02-27 15:51:47 -07001306
1307 # If we have already found the upstream coreboot CL, then exit with success
1308 if step_names.FIND in status.commits:
1309 commit = status.commits[step_names.FIND]
1310 if 'gerrit' in commit and 'cl_number' in commit:
1311 instance_name = commit['gerrit']
1312 cl_number = commit['cl_number']
Paul Fagerburg6cba1492020-06-02 16:44:05 -06001313 logging.debug('Already found (%s, %s)', instance_name, cl_number)
Paul Fagerburgaec8d992020-02-27 15:51:47 -07001314 return True
1315
1316 # Make sure we have a CB_VARIANT commit and a change_id for it
1317 if step_names.CB_VARIANT not in status.commits:
1318 logging.error('Key %s not found in status.commits',
1319 step_names.CB_VARIANT)
1320 return False
1321 if 'change_id' not in status.commits[step_names.CB_VARIANT]:
1322 logging.error('Key change_id not found in status.commits[%s]',
1323 step_names.CB_VARIANT)
1324 return False
1325
1326 # Find the CL by the Original-Change-Id
1327 original_change_id = status.commits[step_names.CB_VARIANT]['change_id']
1328 gerrit_query_args = {
1329 'Original-Change-Id': original_change_id
1330 }
1331 cros = gerrit.GetCrosExternal()
1332 upstream = cros.Query(**gerrit_query_args)
1333 # If nothing is found, the patch hasn't been upstreamed yet
1334 if not upstream:
1335 logging.error('Program cannot continue until coreboot CL is upstreamed.')
1336 logging.error('(coreboot:%s, change-id %s)',
1337 status.commits[step_names.CB_VARIANT]['cl_number'],
1338 status.commits[step_names.CB_VARIANT]['change_id'])
1339 logging.error('Please wait for the CL to be upstreamed, then run this'
1340 ' program again with --continue')
1341 return False
1342
1343 # If more than one CL is found, something is very wrong
1344 if len(upstream) != 1:
1345 logging.error('More than one CL was found with Original-Change-Id %s',
1346 original_change_id)
1347 return False
1348
1349 # At this point, we know there is only one CL and we can get the
1350 # repo and CL number by splitting on the colon between them.
1351 patchlink = upstream[0].PatchLink()
1352 instance_name, cl_number = patchlink.split(':')
1353
1354 # Can't use get_git_commit_data because we're not pulling this
1355 # information from a git commit, but rather from gerrit.
1356 # We only need the gerrit instance and the CL number so we can have
1357 # other CLs cq-depend on this CL. The other keys are not needed because:
1358 # dir - not needed because we're not going to `cd` there to `repo upload`
1359 # branch_name - not valid; the CL is already merged
1360 # change_id - we use the change_id to find a CL number, and since we
1361 # just found the CL number via original-change-id, this is moot.
1362 status.commits[step_names.FIND] = {
1363 'gerrit': instance_name,
1364 'cl_number': str(cl_number)
1365 }
1366
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001367 return True
1368
1369
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001370def calc_cq_depend(status):
1371 """Determine the list of CLs for each commit that has dependencies.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001372
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001373 status.depends is a map of dependencies from step name to a list of
1374 steps that the step depends on. For each step, find the SHA of the
1375 commit, then find the gerrit instance and CL number of the commits
1376 that it depends on. Construct the Cq-Depends list and save it under
1377 the 'cq_depend' key, i.e. commit['add_priv_yaml']['cq_depend'] or
1378 as it will be stored in the yaml:
1379 commits:
1380 add_priv_yaml:
1381 cq_depend: 'chromium:1629121, chromium:1638243'
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001382
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001383 Args:
1384 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001385
1386 Returns:
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001387 True if all dependencies have been calculated. False if something
1388 failed, usually a commit not found by Change-Id.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001389 """
Paul Fagerburgf5f8c7e2020-03-25 10:06:52 -06001390 logging.info('Running step calc_cq_depend')
1391 # Iterate through the commits that have dependencies.
1392 for key in status.depends:
1393 logging.debug('Processing %s to add dependencies', key)
1394 # For every commit that has dependencies, find the gerrit instance
1395 # and CL number of the dependencies.
1396 cq_depend_list = []
1397 for depend_key in status.depends[key]:
1398 depend_commit = status.commits[depend_key]
1399 if not 'gerrit' in depend_commit:
1400 logging.error('Commit %s does not have a gerrit instance',
1401 depend_key)
1402 return False
1403 if not 'cl_number' in depend_commit:
1404 logging.error('Commit %s does not have a CL number',
1405 depend_key)
1406 return False
1407
1408 instance_name = depend_commit['gerrit']
1409 cl_number = depend_commit['cl_number']
1410 cq_depend_list.append(f'{instance_name}:{cl_number}')
1411
1412 # Add the 'cq_depend' key to the commit.
1413 cq_depend_str = 'Cq-Depend: %s' % ', '.join(cq_depend_list)
1414 logging.debug('Add to commit %s %s', key, cq_depend_str)
1415 status.commits[key]['cq_depend'] = cq_depend_str
1416
1417 return True
1418
1419
1420def add_cq_depend_to_commit_msg(git_repo, change_id, cq_depend_str):
1421 """Update the commit message with a Cq-Depends line.
1422
1423 Find the SHA of the commit, then use git filter-branch --msg-filter
1424 to add the Cq-Depend line just before the Change-Id line. See
1425 https://chromium.googlesource.com/chromiumos/docs/+/HEAD/contributing.md#cq-depend
1426 for details about Cq-Depend format and location.
1427
1428 Args:
1429 git_repo: Directory of git repository.
1430 change_id: The Change-Id to search for.
1431 cq_depend_str: The Cq-Depend string. It must be in the correct format
1432 per chromeos documentation, ready to insert into the commit msg
1433 on the line before Change-Id.
1434
1435 Returns:
1436 True if `git filter-branch` was successful. False if the command
1437 failed.
1438 """
1439 logging.debug('find SHA of Change-Id %s in %s', change_id, git_repo)
1440 sha = change_id_to_sha(git_repo, change_id)
1441 if sha is None:
1442 logging.error('Cannot find the SHA for Change-Id %s in %s',
1443 change_id, git_repo)
1444 return False
1445 logging.debug('SHA = %s', sha)
1446
1447 # Check if the commit message already has a Cq-Depend line.
1448 msg = get_commit_msg(git_repo, sha)
1449 if any('Cq-Depend' in tmpstr for tmpstr in msg):
1450 logging.debug('Already has Cq-Depend')
1451 return True
1452
1453 # Use git filter-branch --msg-filter to add the Cq-Depend line just
1454 # before the Change-Id line.
1455 environ = {**os.environ, 'FILTER_BRANCH_SQUELCH_WARNING': '1'}
1456 cmd = [
1457 'git',
1458 'filter-branch',
1459 '--msg-filter',
1460 f'sed -E "s/^(Change-Id: {change_id})$/{cq_depend_str}\\n\\1/"',
1461 '--',
1462 f'{sha}^..']
1463 return run_process(cmd, cwd=git_repo, env=environ)
1464
1465
1466def add_cq_depend(status):
1467 """Add Cq-Depend to commits and flag them for re-upload.
1468
1469 Args:
1470 status: variant_status object tracking our board, variant, etc.
1471
1472 Returns:
1473 True if the commit messages have been successfully amended, False if
1474 something failed.
1475 """
1476 logging.info('Running step add_cq_depend')
1477 for key in status.commits:
1478 commit = status.commits[key]
1479 if 'cq_depend' in commit:
1480 logging.debug('%s has %s', key, commit['cq_depend'])
1481 # Make sure the commit has a working directory and a change_id
1482 # before trying to amend its commit message.
1483 if 'dir' not in commit or 'change_id' not in commit:
1484 logging.error('Missing dir and/or change_id from %s', key)
1485 return False
1486
1487 if not add_cq_depend_to_commit_msg(commit['dir'],
1488 commit['change_id'],
1489 commit['cq_depend']):
1490 return False
1491 commit['needs_re_upload'] = True
1492 else:
1493 logging.debug('%s no dependencies', key)
1494
1495 return True
1496
1497
1498def re_upload(status):
1499 """Re-upload commits that have changed.
1500
1501 Args:
1502 status: variant_status object tracking our board, variant, etc.
1503
1504 Returns:
1505 True if the uploads succeeded. False if a repo upload failed.
1506 """
1507 logging.info('Running step re_upload')
1508 for key in status.commits:
1509 commit = status.commits[key]
1510 if commit.get('needs_re_upload'):
1511 logging.debug('Re-upload branch %s in %s', commit['branch_name'],
1512 commit['dir'])
1513 if not repo_upload(commit['branch_name'], commit['dir']):
1514 logging.error('Repo upload %s in %s failed!',
1515 commit['branch_name'],
1516 commit['dir'])
1517 return False
1518 commit['needs_re_upload'] = False
1519
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001520 return True
1521
1522
1523def clean_up(status):
1524 """Final clean-up, including delete the status file
1525
Paul Fagerburg92fcb4b2020-02-19 21:21:43 -07001526 Args:
1527 status: variant_status object tracking our board, variant, etc.
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001528
1529 Returns:
1530 True
1531 """
Paul Fagerburgbab5dde2020-01-10 15:10:29 -07001532 logging.info('Running step clean_up')
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001533 status.rm()
1534 return True
1535
1536
Paul Fagerburg042a5252020-03-16 21:49:18 -06001537def abort(status):
1538 """Abort the creation of a new variant by abandoning commits
1539
1540 When the user specifies the --abort flag, we override status.step to
1541 be 'abort' and there is no transition from 'abort' to anything else.
1542 We look at status.commits and for each key, see if we have already
1543 been in that directory and abandoned that specific branch. If not,
1544 abandon the commit and then add the branch+dir to a list of abandoned
1545 commits. We do this because some boards (such as Zork) can have multiple
1546 commits in the same directory and with the same branch name, and we only
1547 want to repo abandon that branch once.
1548
1549 Args:
1550 status: variant_status object tracking our board, variant, etc.
1551
1552 Returns:
1553 True
1554 """
1555 logging.info('Running step abort')
1556 # Use the set 'abandoned' to keep track of each branch+dir abandoned.
1557 abandoned = set()
1558 for step in status.commits:
1559 logging.debug('Processing step %s', step)
1560 commit = status.commits[step]
1561 branch = commit['branch_name']
1562 cwd = commit['dir']
1563 if (branch, cwd) in abandoned:
1564 logging.debug('Branch %s in directory %s already abandoned',
1565 branch, cwd)
1566 else:
1567 logging.info('Abandoning branch %s in directory %s',
1568 branch, cwd)
1569 if run_process(['repo', 'abandon', branch, '.'], cwd=cwd):
1570 abandoned.add((branch, cwd))
1571 else:
1572 logging.error('Error while abandoning branch %s', branch)
1573 return False
1574
1575 return True
1576
1577
Paul Fagerburg3b534f92019-11-07 15:05:22 -07001578if __name__ == '__main__':
1579 sys.exit(not int(main()))