blob: 71028a5822200567dea6c6773fbb4b6d7828ec87 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2013 The ChromiumOS Authors
Mike Frysingerd13faeb2013-09-05 16:00:46 -04002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysingere852b072021-05-21 12:39:03 -040011import configparser
Mike Frysingerd13faeb2013-09-05 16:00:46 -040012import getpass
Mike Frysingere852b072021-05-21 12:39:03 -040013import io
Chris McDonald59650c32021-07-20 15:29:28 -060014import logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import os
16import re
Mike Frysinger09fe0122014-02-09 02:44:05 -050017import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040018
Mike Frysingerd13faeb2013-09-05 16:00:46 -040019from chromite.lib import commandline
Chris McDonald59650c32021-07-20 15:29:28 -060020from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import gs
23from chromite.lib import osutils
24from chromite.lib import signing
25
26
27# This will split a fully qualified ChromeOS version string up.
28# R34-5126.0.0 will break into "34" and "5126.0.0".
Alex Klein1699fab2022-09-08 08:46:06 -060029VERSION_REGEX = r"^R([0-9]+)-([^-]+)"
Mike Frysingerd13faeb2013-09-05 16:00:46 -040030
Mike Frysingerdad40d62014-02-09 02:18:02 -050031# The test signers will scan this dir looking for test work.
32# Keep it in sync with the signer config files [gs_test_buckets].
Alex Klein1699fab2022-09-08 08:46:06 -060033TEST_SIGN_BUCKET_BASE = "gs://chromeos-throw-away-bucket/signer-tests"
Mike Frysingerdad40d62014-02-09 02:18:02 -050034
David Rileyf8205122015-09-04 13:46:36 -070035# Keysets that are only valid in the above test bucket.
Alex Klein1699fab2022-09-08 08:46:06 -060036TEST_KEYSET_PREFIX = "test-keys"
Alex Klein041edd82023-04-17 12:23:23 -060037TEST_KEYSETS = {"mp", "premp", "nvidia-premp"}
Mike Frysingerdad40d62014-02-09 02:18:02 -050038
Amey Deshpandea936c622015-08-12 17:27:54 -070039# Supported image types for signing.
40_SUPPORTED_IMAGE_TYPES = (
41 constants.IMAGE_TYPE_RECOVERY,
42 constants.IMAGE_TYPE_FACTORY,
43 constants.IMAGE_TYPE_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070044 constants.IMAGE_TYPE_ACCESSORY_USBPD,
Evan Benn4d061102022-02-14 12:50:45 +110045 constants.IMAGE_TYPE_HPS_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070046 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070047 constants.IMAGE_TYPE_BASE,
Vadim Bendeburyfe37f282020-11-04 19:05:49 -080048 constants.IMAGE_TYPE_GSC_FIRMWARE,
Amey Deshpandea936c622015-08-12 17:27:54 -070049)
50
Mike Frysingerd13faeb2013-09-05 16:00:46 -040051
Mike Frysinger4495b032014-03-05 17:24:03 -050052class PushError(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060053 """When an (unknown) error happened while trying to push artifacts."""
Mike Frysinger4495b032014-03-05 17:24:03 -050054
55
Mike Frysingerd13faeb2013-09-05 16:00:46 -040056class MissingBoardInstructions(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060057 """Raised when a board lacks any signer instructions."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -040058
Alex Klein1699fab2022-09-08 08:46:06 -060059 def __init__(self, board, image_type, input_insns):
60 Exception.__init__(
61 self,
62 "Board %s lacks insns for %s image: %s not found"
63 % (board, image_type, input_insns),
64 )
Mike Frysingerd84d91e2015-11-05 18:02:24 -050065
Mike Frysingerd13faeb2013-09-05 16:00:46 -040066
Alex Klein074f94f2023-06-22 10:32:06 -060067class InputInsns:
Alex Klein1699fab2022-09-08 08:46:06 -060068 """Object to hold settings for a signable board.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050069
Mike Frysingerd13faeb2013-09-05 16:00:46 -040070 Note: The format of the instruction file pushimage outputs (and the signer
71 reads) is not exactly the same as the instruction file pushimage reads.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040072 """
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050073
Alex Klein1699fab2022-09-08 08:46:06 -060074 def __init__(self, board, image_type=None, buildroot=None):
75 """Initialization.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040076
Alex Klein1699fab2022-09-08 08:46:06 -060077 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +100078 board: The board to look up details.
79 image_type: The type of image we will be signing (see --sign-types).
80 buildroot: Buildroot in which to look for signing instructions.
Alex Klein1699fab2022-09-08 08:46:06 -060081 """
82 self.board = board
83 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040084
Alex Klein1699fab2022-09-08 08:46:06 -060085 config = configparser.ConfigParser()
Mike Frysinger31fdddd2023-02-24 15:50:55 -050086 with open(self.GetInsnFile("DEFAULT"), encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060087 config.read_file(fp)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050088
Trent Apted4a0812b2023-05-15 15:33:55 +100089 # What pushimage internally refers to as 'recovery', are the basic
90 # signing instructions in practice, and other types are stacked on top.
Alex Klein1699fab2022-09-08 08:46:06 -060091 if image_type is None:
92 image_type = constants.IMAGE_TYPE_RECOVERY
93 self.image_type = image_type
94 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
95 if not os.path.exists(input_insns):
96 # This board doesn't have any signing instructions.
97 raise MissingBoardInstructions(self.board, image_type, input_insns)
Mike Frysinger31fdddd2023-02-24 15:50:55 -050098 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060099 config.read_file(fp)
100
Alex Klein511a7c12023-08-11 15:28:46 -0600101 if image_type != constants.IMAGE_TYPE_RECOVERY:
Alex Klein1699fab2022-09-08 08:46:06 -0600102 input_insns = self.GetInsnFile(image_type)
103 if not os.path.exists(input_insns):
104 # This type doesn't have any signing instructions.
105 raise MissingBoardInstructions(
106 self.board, image_type, input_insns
107 )
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500108 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -0600109 config.read_file(fp)
110
111 self.cfg = config
112
113 def GetInsnFile(self, image_type):
114 """Find the signer instruction files for this board/image type.
115
116 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000117 image_type: The type of instructions to load. It can be a common
118 file (like "DEFAULT"), or one of the --sign-types.
Alex Klein1699fab2022-09-08 08:46:06 -0600119
120 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000121 Full path to the instruction file using |image_type| and
122 |self.board|.
Alex Klein1699fab2022-09-08 08:46:06 -0600123 """
124 if image_type == image_type.upper():
125 name = image_type
126 elif image_type in (
127 constants.IMAGE_TYPE_RECOVERY,
128 constants.IMAGE_TYPE_BASE,
129 ):
130 name = self.board
131 else:
132 name = "%s.%s" % (self.board, image_type)
133
134 return os.path.join(
135 self.buildroot, signing.INPUT_INSN_DIR_REL, "%s.instructions" % name
136 )
137
138 @staticmethod
139 def SplitCfgField(val):
140 """Split a string into multiple elements.
141
142 This centralizes our convention for multiple elements in the input files
143 being delimited by either a space or comma.
144
145 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000146 val: The string to split.
Alex Klein1699fab2022-09-08 08:46:06 -0600147
148 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000149 The list of elements from having done split the string.
Alex Klein1699fab2022-09-08 08:46:06 -0600150 """
151 return val.replace(",", " ").split()
152
153 def GetChannels(self):
154 """Return the list of channels to sign for this board.
155
156 If the board-specific config doesn't specify a preference, we'll use the
157 common settings.
158 """
159 return self.SplitCfgField(self.cfg.get("insns", "channel"))
160
161 def GetKeysets(self, insns_merge=None):
162 """Return the list of keysets to sign for this board.
163
164 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000165 insns_merge: The additional section to look at over [insns].
Alex Klein1699fab2022-09-08 08:46:06 -0600166 """
167 # First load the default value from [insns.keyset] if available.
168 sections = ["insns"]
169 # Then overlay the [insns.xxx.keyset] if requested.
170 if insns_merge is not None:
171 sections += [insns_merge]
172
173 keyset = ""
174 for section in sections:
175 try:
176 keyset = self.cfg.get(section, "keyset")
177 except (configparser.NoSectionError, configparser.NoOptionError):
178 pass
179
180 # We do not perturb the order (e.g. using sorted() or making a set())
181 # because we want the behavior stable, and we want the input insns to
182 # explicitly control the order (since it has an impact on naming).
183 return self.SplitCfgField(keyset)
184
185 def GetAltInsnSets(self):
186 """Return the list of alternative insn sections."""
187 # We do not perturb the order (e.g. using sorted() or making a set())
188 # because we want the behavior stable, and we want the input insns to
189 # explicitly control the order (since it has an impact on naming).
190 ret = [x for x in self.cfg.sections() if x.startswith("insns.")]
191 return ret if ret else [None]
192
193 @staticmethod
194 def CopyConfigParser(config):
195 """Return a copy of a ConfigParser object.
196
197 The python folks broke the ability to use something like deepcopy:
198 https://bugs.python.org/issue16058
199 """
200 # Write the current config to a string io object.
201 data = io.StringIO()
202 config.write(data)
203 data.seek(0)
204
205 # Create a new ConfigParser from the serialized data.
206 ret = configparser.ConfigParser()
207 ret.read_file(data)
208
209 return ret
210
211 def OutputInsns(
212 self, output_file, sect_insns, sect_general, insns_merge=None
213 ):
214 """Generate the output instruction file for sending to the signer.
215
216 The override order is (later has precedence):
217 [insns]
218 [insns_merge] (should be named "insns.xxx")
219 sect_insns
220
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000221 Note: The format of the instruction file pushimage outputs (and the
222 signer reads) is not exactly the same as the instruction file pushimage
223 reads.
Alex Klein1699fab2022-09-08 08:46:06 -0600224
225 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000226 output_file: The file to write the new instruction file to.
227 sect_insns: Items to set/override in the [insns] section.
228 sect_general: Items to set/override in the [general] section.
229 insns_merge: The alternative insns.xxx section to merge.
Alex Klein1699fab2022-09-08 08:46:06 -0600230 """
231 # Create a copy so we can clobber certain fields.
232 config = self.CopyConfigParser(self.cfg)
233 sect_insns = sect_insns.copy()
234
235 # Merge in the alternative insns section if need be.
236 if insns_merge is not None:
237 for k, v in config.items(insns_merge):
238 sect_insns.setdefault(k, v)
239
240 # Clear channel entry in instructions file, ensuring we only get
241 # one channel for the signer to look at. Then provide all the
242 # other details for this signing request to avoid any ambiguity
243 # and to avoid relying on encoding data into filenames.
244 for sect, fields in zip(
245 ("insns", "general"), (sect_insns, sect_general)
246 ):
247 if not config.has_section(sect):
248 config.add_section(sect)
249 for k, v in fields.items():
250 config.set(sect, k, v)
251
252 # Now prune the alternative sections.
253 for alt in self.GetAltInsnSets():
254 config.remove_section(alt)
255
256 output = io.StringIO()
257 config.write(output)
258 data = output.getvalue()
259 osutils.WriteFile(output_file, data)
260 logging.debug("generated insns file for %s:\n%s", self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400261
262
263def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
Alex Klein1699fab2022-09-08 08:46:06 -0600264 """Mark an instructions file for signing.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400265
Alex Klein1699fab2022-09-08 08:46:06 -0600266 This will upload a file to the GS bucket flagging an image for signing by
267 the signers.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400268
Alex Klein1699fab2022-09-08 08:46:06 -0600269 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000270 ctx: A viable gs.GSContext.
271 tbs_base: The full path to where the tobesigned directory lives.
272 insns_path: The path (relative to |tbs_base|) of the file to sign.
273 priority: Set the signing priority (lower == higher prio).
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400274
Alex Klein1699fab2022-09-08 08:46:06 -0600275 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000276 The full path to the remote tobesigned file.
Alex Klein1699fab2022-09-08 08:46:06 -0600277 """
278 if priority < 0 or priority > 99:
279 raise ValueError("priority must be [0, 99] inclusive")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400280
Alex Klein1699fab2022-09-08 08:46:06 -0600281 if insns_path.startswith(tbs_base):
282 insns_path = insns_path[len(tbs_base) :].lstrip("/")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400283
Alex Klein1699fab2022-09-08 08:46:06 -0600284 tbs_path = "%s/tobesigned/%02i,%s" % (
285 tbs_base,
286 priority,
287 insns_path.replace("/", ","),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400288 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700289
Alex Klein1699fab2022-09-08 08:46:06 -0600290 # The caller will catch gs.GSContextException for us.
291 ctx.Copy("-", tbs_path, input=cros_build_lib.MachineDetails())
Amey Deshpandea936c622015-08-12 17:27:54 -0700292
Alex Klein1699fab2022-09-08 08:46:06 -0600293 return tbs_path
Amey Deshpandea936c622015-08-12 17:27:54 -0700294
David Rileya04d19d2015-09-04 16:11:50 -0700295
Alex Klein1699fab2022-09-08 08:46:06 -0600296def PushImage(
297 src_path,
298 board,
299 versionrev=None,
300 profile=None,
301 priority=50,
302 sign_types=None,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500303 dryrun=False,
Alex Klein1699fab2022-09-08 08:46:06 -0600304 mock=False,
305 force_keysets=(),
306 force_channels=None,
307 buildroot=constants.SOURCE_ROOT,
308 dest_bucket=constants.RELEASE_BUCKET,
309):
310 """Push the image from the archive bucket to the release bucket.
Evan Benn4d061102022-02-14 12:50:45 +1100311
Alex Klein1699fab2022-09-08 08:46:06 -0600312 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000313 src_path: Where to copy the files from; can be a local path or gs://
314 URL. Should be a full path to the artifacts in either case.
315 board: The board we're uploading artifacts for (e.g. $BOARD).
316 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
317 profile: The board profile in use (e.g. "asan").
318 priority: Set the signing priority (lower == higher prio).
319 sign_types: If set, a set of types which we'll restrict ourselves to
320 signing. See the --sign-types option for more details.
321 dryrun: Show what would be done, but do not upload anything.
322 mock: Upload to a testing bucket rather than the real one.
323 force_keysets: Set of keysets to use rather than what the inputs say.
324 force_channels: Set of channels to use rather than what the inputs say.
325 buildroot: Buildroot in which to look for signing instructions.
326 dest_bucket: Bucket to push results to.
Vincent Palatind599c662015-10-26 09:51:41 -0700327
Alex Klein1699fab2022-09-08 08:46:06 -0600328 Returns:
Alex Klein074f94f2023-06-22 10:32:06 -0600329 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
330 'gs://signer_instruction_uri2',
331 ...]
Alex Klein1699fab2022-09-08 08:46:06 -0600332 """
333 # Whether we hit an unknown error. If so, we'll throw an error, but only
334 # at the end (so that we still upload as many files as possible).
335 # It's implemented using a list to deal with variable scopes in nested
336 # functions below.
337 unknown_error = [False]
LaMont Jones7d6c98f2019-09-27 12:37:33 -0600338
Alex Klein1699fab2022-09-08 08:46:06 -0600339 if versionrev is None:
340 # Extract milestone/version from the directory name.
341 versionrev = os.path.basename(src_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700342
Alex Klein1699fab2022-09-08 08:46:06 -0600343 # We only support the latest format here. Older releases can use pushimage
344 # from the respective branch which deals with legacy cruft.
345 m = re.match(VERSION_REGEX, versionrev)
346 if not m:
347 raise ValueError(
348 "version %s does not match %s" % (versionrev, VERSION_REGEX)
349 )
350 milestone = m.group(1)
351 version = m.group(2)
Amey Deshpandea936c622015-08-12 17:27:54 -0700352
Alex Klein1699fab2022-09-08 08:46:06 -0600353 # Normalize board to always use dashes not underscores. This is mostly a
354 # historical artifact at this point, but we can't really break it since the
355 # value is used in URLs.
356 boardpath = board.replace("_", "-")
357 if profile is not None:
358 boardpath += "-%s" % profile.replace("_", "-")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400359
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500360 ctx = gs.GSContext(dry_run=dryrun)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400361
Alex Klein1699fab2022-09-08 08:46:06 -0600362 try:
363 input_insns = InputInsns(board, buildroot=buildroot)
364 except MissingBoardInstructions as e:
365 logging.warning("Missing base instruction file: %s", e)
366 logging.warning("not uploading anything for signing")
367 return
368
369 if force_channels is None:
370 channels = input_insns.GetChannels()
371 else:
372 # Filter out duplicates.
373 channels = sorted(set(force_channels))
374
375 # We want force_keysets as a set.
376 force_keysets = set(force_keysets)
377
378 if mock:
379 logging.info("Upload mode: mock; signers will not process anything")
380 tbs_base = gs_base = os.path.join(
381 constants.TRASH_BUCKET, "pushimage-tests", getpass.getuser()
382 )
Alex Klein041edd82023-04-17 12:23:23 -0600383 elif {f"{TEST_KEYSET_PREFIX}-{x}" for x in TEST_KEYSETS} & force_keysets:
Alex Klein1699fab2022-09-08 08:46:06 -0600384 logging.info("Upload mode: test; signers will process test keys")
385 # We need the tbs_base to be in the place the signer will actually scan.
386 tbs_base = TEST_SIGN_BUCKET_BASE
387 gs_base = os.path.join(tbs_base, getpass.getuser())
388 else:
389 logging.info("Upload mode: normal; signers will process the images")
390 tbs_base = gs_base = dest_bucket
391
392 sect_general = {
393 "config_board": board,
394 "board": boardpath,
395 "version": version,
396 "versionrev": versionrev,
397 "milestone": milestone,
398 }
399 sect_insns = {}
400
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500401 if dryrun:
Alex Klein1699fab2022-09-08 08:46:06 -0600402 logging.info("DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED")
403 logging.info("Signing for channels: %s", " ".join(channels))
404
405 instruction_urls = {}
406
407 def _ImageNameBase(image_type=None):
408 lmid = ("%s-" % image_type) if image_type else ""
409 return "ChromeOS-%s%s-%s" % (lmid, versionrev, boardpath)
410
411 # These variables are defined outside the loop so that the nested functions
412 # below can access them without 'cell-var-from-loop' linter warning.
413 dst_path = ""
Amey Deshpandea936c622015-08-12 17:27:54 -0700414 files_to_sign = []
Alex Klein1699fab2022-09-08 08:46:06 -0600415 for channel in channels:
416 logging.debug("\n\n#### CHANNEL: %s ####\n", channel)
417 sect_insns["channel"] = channel
418 sub_path = "%s-channel/%s/%s" % (channel, boardpath, version)
419 dst_path = "%s/%s" % (gs_base, sub_path)
420 logging.info("Copying images to %s", dst_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700421
Alex Klein1699fab2022-09-08 08:46:06 -0600422 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
423 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
424 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
425 hps_firmware_basename = _ImageNameBase(
426 constants.IMAGE_TYPE_HPS_FIRMWARE
427 )
428 acc_usbpd_basename = _ImageNameBase(
429 constants.IMAGE_TYPE_ACCESSORY_USBPD
430 )
431 acc_rwsig_basename = _ImageNameBase(
432 constants.IMAGE_TYPE_ACCESSORY_RWSIG
433 )
434 gsc_firmware_basename = _ImageNameBase(
435 constants.IMAGE_TYPE_GSC_FIRMWARE
436 )
437 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
438 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
439 hwqual_tarball = "chromeos-hwqual-%s-%s.tar.bz2" % (board, versionrev)
Amey Deshpandea936c622015-08-12 17:27:54 -0700440
Trent Apted4a0812b2023-05-15 15:33:55 +1000441 # The following build artifacts, if present, are always copied
442 # regardless of requested signing types.
Alex Klein1699fab2022-09-08 08:46:06 -0600443 files_to_copy_only = (
444 # (<src>, <dst>, <suffix>),
445 ("image.zip", _ImageNameBase(), "zip"),
446 (constants.TEST_IMAGE_TAR, test_basename, "tar.xz"),
447 ("debug.tgz", "debug-%s" % boardpath, "tgz"),
448 (hwqual_tarball, None, None),
449 ("stateful.tgz", None, None),
450 ("dlc", None, None),
451 (constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
452 (constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
453 (constants.QUICK_PROVISION_PAYLOAD_MINIOS, None, None),
454 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700455
Alex Klein1699fab2022-09-08 08:46:06 -0600456 # The following build artifacts, if present, are always copied.
457 # If |sign_types| is None, all of them are marked for signing, otherwise
458 # only the image types specified in |sign_types| are marked for signing.
459 files_to_copy_and_maybe_sign = (
460 # (<src>, <dst>, <suffix>, <signing type>),
461 (
462 constants.RECOVERY_IMAGE_TAR,
463 recovery_basename,
464 "tar.xz",
465 constants.IMAGE_TYPE_RECOVERY,
466 ),
467 (
468 "factory_image.zip",
469 factory_basename,
470 "zip",
471 constants.IMAGE_TYPE_FACTORY,
472 ),
473 (
474 "firmware_from_source.tar.bz2",
475 firmware_basename,
476 "tar.bz2",
477 constants.IMAGE_TYPE_FIRMWARE,
478 ),
Shuhei Takahashi7e05dbf2023-05-19 10:40:47 +0900479 )
480
481 # The following build artifacts are copied and marked for signing, if
482 # they are present *and* if the image type is specified via
483 # |sign_types|.
484 files_to_maybe_copy_and_sign = (
485 # (<src>, <dst>, <suffix>, <signing type>),
486 (
487 constants.BASE_IMAGE_TAR,
488 base_basename,
489 "tar.xz",
490 constants.IMAGE_TYPE_BASE,
491 ),
Alex Klein1699fab2022-09-08 08:46:06 -0600492 (
493 "firmware_from_source.tar.bz2",
494 hps_firmware_basename,
495 "tar.bz2",
496 constants.IMAGE_TYPE_HPS_FIRMWARE,
497 ),
498 (
499 "firmware_from_source.tar.bz2",
500 acc_usbpd_basename,
501 "tar.bz2",
502 constants.IMAGE_TYPE_ACCESSORY_USBPD,
503 ),
504 (
505 "firmware_from_source.tar.bz2",
506 acc_rwsig_basename,
507 "tar.bz2",
508 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
509 ),
510 (
511 "firmware_from_source.tar.bz2",
512 gsc_firmware_basename,
513 "tar.bz2",
514 constants.IMAGE_TYPE_GSC_FIRMWARE,
515 ),
516 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700517
Alex Klein1699fab2022-09-08 08:46:06 -0600518 def _CopyFileToGS(src, dst=None, suffix=None):
519 """Returns |dst| file name if the copying was successful."""
520 if dst is None:
521 dst = src
522 elif suffix is not None:
523 dst = "%s.%s" % (dst, suffix)
524 success = False
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500525 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600526 ctx.Copy(
527 os.path.join(src_path, src),
528 os.path.join(dst_path, dst),
529 recursive=True,
530 )
531 success = True
532 except gs.GSNoSuchKey:
533 logging.warning("Skipping %s as it does not exist", src)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500534 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600535 unknown_error[0] = True
536 logging.error(
537 "Skipping %s due to unknown GS error", src, exc_info=True
538 )
539 return dst if success else None
Mike Frysinger4495b032014-03-05 17:24:03 -0500540
Alex Klein1699fab2022-09-08 08:46:06 -0600541 for src, dst, suffix in files_to_copy_only:
542 _CopyFileToGS(src, dst, suffix)
543
544 # Clear the list of files to sign before adding new artifacts.
545 files_to_sign = []
546
547 def _AddToFilesToSign(image_type, dst, suffix):
548 assert dst.endswith("." + suffix), "dst: %s, suffix: %s" % (
549 dst,
550 suffix,
551 )
552 dst_base = dst[: -(len(suffix) + 1)]
553 files_to_sign.append([image_type, dst_base, suffix])
554
555 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
556 dst = _CopyFileToGS(src, dst, suffix)
557 if dst and (not sign_types or image_type in sign_types):
558 _AddToFilesToSign(image_type, dst, suffix)
559
560 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
561 if sign_types and image_type in sign_types:
562 dst = _CopyFileToGS(src, dst, suffix)
563 if dst:
564 _AddToFilesToSign(image_type, dst, suffix)
565
566 logging.debug("Files to sign: %s", files_to_sign)
567 unused_sign_types = set(sign_types or []) - set(
568 x for x, _, _ in files_to_sign
569 )
570 if unused_sign_types:
571 logging.warning(
572 "Some sign types were unused: %s", " ".join(unused_sign_types)
573 )
574
575 # Now go through the subset for signing.
576 for image_type, dst_name, suffix in files_to_sign:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500577 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600578 input_insns = InputInsns(
579 board, image_type=image_type, buildroot=buildroot
580 )
581 except MissingBoardInstructions as e:
582 logging.info("Nothing to sign: %s", e)
583 continue
584
585 dst_archive = "%s.%s" % (dst_name, suffix)
586 sect_general["archive"] = dst_archive
587 sect_general["type"] = image_type
588
589 # In the default/automatic mode, only flag files for signing if the
Trent Apted4a0812b2023-05-15 15:33:55 +1000590 # archives were actually uploaded in a previous stage. This
591 # additional check can be removed in future once |sign_types|
592 # becomes a required argument.
Alex Klein1699fab2022-09-08 08:46:06 -0600593 # TODO: Make |sign_types| a required argument.
594 gs_artifact_path = os.path.join(dst_path, dst_archive)
595 exists = False
596 try:
597 exists = ctx.Exists(gs_artifact_path)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500598 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600599 unknown_error[0] = True
600 logging.error(
601 "Unknown error while checking %s",
602 gs_artifact_path,
603 exc_info=True,
604 )
605 if not exists:
606 logging.info(
607 "%s does not exist. Nothing to sign.", gs_artifact_path
608 )
609 continue
Don Garrett9459c2f2014-01-22 18:20:24 -0800610
Alex Klein1699fab2022-09-08 08:46:06 -0600611 first_image = True
612 for alt_insn_set in input_insns.GetAltInsnSets():
613 # Figure out which keysets have been requested for this type.
614 # We sort the forced set so tests/runtime behavior is stable.
615 keysets = sorted(force_keysets)
616 if not keysets:
617 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
618 if not keysets:
619 logging.warning(
620 "Skipping %s image signing due to no keysets",
621 image_type,
622 )
Mike Frysinger4495b032014-03-05 17:24:03 -0500623
Alex Klein1699fab2022-09-08 08:46:06 -0600624 for keyset in keysets:
625 sect_insns["keyset"] = keyset
626
Trent Apted4a0812b2023-05-15 15:33:55 +1000627 # Generate the insn file for this artifact that the signer
628 # will use, and flag it for signing.
Alex Klein1699fab2022-09-08 08:46:06 -0600629 with cros_build_lib.UnbufferedNamedTemporaryFile(
630 prefix="pushimage.insns."
631 ) as insns_path:
632 input_insns.OutputInsns(
633 insns_path.name,
634 sect_insns,
635 sect_general,
636 insns_merge=alt_insn_set,
637 )
638
639 gs_insns_path = "%s/%s" % (dst_path, dst_name)
640 if not first_image:
641 gs_insns_path += "-%s" % keyset
642 first_image = False
643 gs_insns_path += ".instructions"
644
645 try:
646 ctx.Copy(insns_path.name, gs_insns_path)
647 except gs.GSContextException:
648 unknown_error[0] = True
649 logging.error(
650 "Unknown error while uploading insns %s",
651 gs_insns_path,
652 exc_info=True,
653 )
654 continue
655
656 try:
657 MarkImageToBeSigned(
658 ctx, tbs_base, gs_insns_path, priority
659 )
660 except gs.GSContextException:
661 unknown_error[0] = True
662 logging.error(
663 "Unknown error while marking for signing %s",
664 gs_insns_path,
665 exc_info=True,
666 )
667 continue
668 logging.info(
669 "Signing %s image with keyset %s at %s",
670 image_type,
671 keyset,
672 gs_insns_path,
673 )
674 instruction_urls.setdefault(channel, []).append(
675 gs_insns_path
676 )
677
678 if unknown_error[0]:
679 raise PushError("hit some unknown error(s)", instruction_urls)
680
681 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400682
683
Mike Frysinger26144192017-08-30 18:26:46 -0400684def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600685 """Creates the argparse parser."""
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500686 parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400687
Alex Klein1699fab2022-09-08 08:46:06 -0600688 # The type of image_dir will strip off trailing slashes (makes later
689 # processing simpler and the display prettier).
690 parser.add_argument(
691 "image_dir",
692 default=None,
693 type="local_or_gs_path",
694 help="full path of source artifacts to upload",
695 )
696 parser.add_argument(
697 "--board",
698 default=None,
699 required=True,
700 help="board to generate symbols for",
701 )
702 parser.add_argument(
703 "--profile", default=None, help='board profile in use (e.g. "asan")'
704 )
705 parser.add_argument(
706 "--version",
707 default=None,
708 help="version info (normally extracted from image_dir)",
709 )
710 parser.add_argument(
711 "--channels",
712 default=None,
713 action="split_extend",
714 help="override list of channels to process",
715 )
716 parser.add_argument(
Alex Klein1699fab2022-09-08 08:46:06 -0600717 "-M",
718 "--mock",
719 default=False,
720 action="store_true",
721 help="upload things to a testing bucket (dev testing)",
722 )
723 parser.add_argument(
724 "--test-sign",
725 default=[],
726 action="append",
727 choices=TEST_KEYSETS,
728 help="mung signing behavior to sign w/ test keys",
729 )
730 parser.add_argument(
731 "--priority",
732 type=int,
733 default=50,
734 help="set signing priority (lower == higher prio)",
735 )
736 parser.add_argument(
737 "--sign-types",
738 default=None,
739 nargs="+",
740 choices=_SUPPORTED_IMAGE_TYPES,
741 help="only sign specified image types",
742 )
743 parser.add_argument(
744 "--buildroot",
745 default=constants.SOURCE_ROOT,
746 type="path",
747 help="Buildroot to use. Defaults to current.",
748 )
749 parser.add_argument(
750 "--yes",
751 action="store_true",
752 default=False,
753 help="answer yes to all prompts",
754 )
755 parser.add_argument(
756 "--dest-bucket",
757 default=constants.RELEASE_BUCKET,
758 help="dest bucket. Default to %(default)s",
759 )
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400760
Alex Klein1699fab2022-09-08 08:46:06 -0600761 return parser
Mike Frysinger26144192017-08-30 18:26:46 -0400762
763
764def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600765 parser = GetParser()
766 opts = parser.parse_args(argv)
767 opts.Freeze()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400768
Alex Klein041edd82023-04-17 12:23:23 -0600769 force_keysets = {f"{TEST_KEYSET_PREFIX}-{x}" for x in opts.test_sign}
Mike Frysingerdad40d62014-02-09 02:18:02 -0500770
Trent Apted4a0812b2023-05-15 15:33:55 +1000771 # If we aren't using mock or test or dry run mode, then let's prompt the
772 # user to make sure they actually want to do this. It's rare that people
773 # want to run this directly and hit the release bucket.
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500774 if not (opts.mock or force_keysets or opts.dryrun) and not opts.yes:
Alex Klein1699fab2022-09-08 08:46:06 -0600775 prolog = "\n".join(
776 textwrap.wrap(
777 textwrap.dedent(
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000778 "Uploading images for signing to the *release* bucket is"
779 " not something you generally should be doing yourself."
Alex Klein1699fab2022-09-08 08:46:06 -0600780 ),
781 80,
782 )
783 ).strip()
784 if not cros_build_lib.BooleanPrompt(
785 prompt="Are you sure you want to sign these images",
786 default=False,
787 prolog=prolog,
788 ):
789 cros_build_lib.Die("better safe than sorry")
Mike Frysinger09fe0122014-02-09 02:44:05 -0500790
Alex Klein1699fab2022-09-08 08:46:06 -0600791 PushImage(
792 opts.image_dir,
793 opts.board,
794 versionrev=opts.version,
795 profile=opts.profile,
796 priority=opts.priority,
797 sign_types=opts.sign_types,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500798 dryrun=opts.dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600799 mock=opts.mock,
800 force_keysets=force_keysets,
801 force_channels=opts.channels,
802 buildroot=opts.buildroot,
803 dest_bucket=opts.dest_bucket,
804 )