blob: a2c121bc9edf107c3ad59cfe8a4e65f2fe74846e [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2013 The ChromiumOS Authors
Mike Frysingerd13faeb2013-09-05 16:00:46 -04002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysingere852b072021-05-21 12:39:03 -040011import configparser
Mike Frysingerd13faeb2013-09-05 16:00:46 -040012import getpass
Mike Frysingere852b072021-05-21 12:39:03 -040013import io
Chris McDonald59650c32021-07-20 15:29:28 -060014import logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import os
16import re
Mike Frysinger09fe0122014-02-09 02:44:05 -050017import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040018
Mike Frysingerd13faeb2013-09-05 16:00:46 -040019from chromite.lib import commandline
Chris McDonald59650c32021-07-20 15:29:28 -060020from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import gs
23from chromite.lib import osutils
24from chromite.lib import signing
25
26
27# This will split a fully qualified ChromeOS version string up.
28# R34-5126.0.0 will break into "34" and "5126.0.0".
Alex Klein1699fab2022-09-08 08:46:06 -060029VERSION_REGEX = r"^R([0-9]+)-([^-]+)"
Mike Frysingerd13faeb2013-09-05 16:00:46 -040030
Mike Frysingerdad40d62014-02-09 02:18:02 -050031# The test signers will scan this dir looking for test work.
32# Keep it in sync with the signer config files [gs_test_buckets].
Alex Klein1699fab2022-09-08 08:46:06 -060033TEST_SIGN_BUCKET_BASE = "gs://chromeos-throw-away-bucket/signer-tests"
Mike Frysingerdad40d62014-02-09 02:18:02 -050034
David Rileyf8205122015-09-04 13:46:36 -070035# Keysets that are only valid in the above test bucket.
Alex Klein1699fab2022-09-08 08:46:06 -060036TEST_KEYSET_PREFIX = "test-keys"
Alex Klein041edd82023-04-17 12:23:23 -060037TEST_KEYSETS = {"mp", "premp", "nvidia-premp"}
Mike Frysingerdad40d62014-02-09 02:18:02 -050038
Amey Deshpandea936c622015-08-12 17:27:54 -070039# Supported image types for signing.
40_SUPPORTED_IMAGE_TYPES = (
41 constants.IMAGE_TYPE_RECOVERY,
42 constants.IMAGE_TYPE_FACTORY,
43 constants.IMAGE_TYPE_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070044 constants.IMAGE_TYPE_ACCESSORY_USBPD,
Evan Benn4d061102022-02-14 12:50:45 +110045 constants.IMAGE_TYPE_HPS_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070046 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070047 constants.IMAGE_TYPE_BASE,
Vadim Bendeburyfe37f282020-11-04 19:05:49 -080048 constants.IMAGE_TYPE_GSC_FIRMWARE,
Amey Deshpandea936c622015-08-12 17:27:54 -070049)
50
Mike Frysingerd13faeb2013-09-05 16:00:46 -040051
Mike Frysinger4495b032014-03-05 17:24:03 -050052class PushError(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060053 """When an (unknown) error happened while trying to push artifacts."""
Mike Frysinger4495b032014-03-05 17:24:03 -050054
55
Mike Frysingerd13faeb2013-09-05 16:00:46 -040056class MissingBoardInstructions(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060057 """Raised when a board lacks any signer instructions."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -040058
Alex Klein1699fab2022-09-08 08:46:06 -060059 def __init__(self, board, image_type, input_insns):
60 Exception.__init__(
61 self,
62 "Board %s lacks insns for %s image: %s not found"
63 % (board, image_type, input_insns),
64 )
Mike Frysingerd84d91e2015-11-05 18:02:24 -050065
Mike Frysingerd13faeb2013-09-05 16:00:46 -040066
67class InputInsns(object):
Alex Klein1699fab2022-09-08 08:46:06 -060068 """Object to hold settings for a signable board.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050069
Mike Frysingerd13faeb2013-09-05 16:00:46 -040070 Note: The format of the instruction file pushimage outputs (and the signer
71 reads) is not exactly the same as the instruction file pushimage reads.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040072 """
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050073
Alex Klein1699fab2022-09-08 08:46:06 -060074 def __init__(self, board, image_type=None, buildroot=None):
75 """Initialization.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040076
Alex Klein1699fab2022-09-08 08:46:06 -060077 Args:
78 board: The board to look up details.
79 image_type: The type of image we will be signing (see --sign-types).
80 buildroot: Buildroot in which to look for signing instructions.
81 """
82 self.board = board
83 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040084
Alex Klein1699fab2022-09-08 08:46:06 -060085 config = configparser.ConfigParser()
Mike Frysinger31fdddd2023-02-24 15:50:55 -050086 with open(self.GetInsnFile("DEFAULT"), encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060087 config.read_file(fp)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050088
Alex Klein1699fab2022-09-08 08:46:06 -060089 # What pushimage internally refers to as 'recovery', are the basic signing
90 # instructions in practice, and other types are stacked on top.
91 if image_type is None:
92 image_type = constants.IMAGE_TYPE_RECOVERY
93 self.image_type = image_type
94 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
95 if not os.path.exists(input_insns):
96 # This board doesn't have any signing instructions.
97 raise MissingBoardInstructions(self.board, image_type, input_insns)
Mike Frysinger31fdddd2023-02-24 15:50:55 -050098 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060099 config.read_file(fp)
100
101 if image_type is not None:
102 input_insns = self.GetInsnFile(image_type)
103 if not os.path.exists(input_insns):
104 # This type doesn't have any signing instructions.
105 raise MissingBoardInstructions(
106 self.board, image_type, input_insns
107 )
108
109 self.image_type = image_type
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500110 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -0600111 config.read_file(fp)
112
113 self.cfg = config
114
115 def GetInsnFile(self, image_type):
116 """Find the signer instruction files for this board/image type.
117
118 Args:
119 image_type: The type of instructions to load. It can be a common file
120 (like "DEFAULT"), or one of the --sign-types.
121
122 Returns:
123 Full path to the instruction file using |image_type| and |self.board|.
124 """
125 if image_type == image_type.upper():
126 name = image_type
127 elif image_type in (
128 constants.IMAGE_TYPE_RECOVERY,
129 constants.IMAGE_TYPE_BASE,
130 ):
131 name = self.board
132 else:
133 name = "%s.%s" % (self.board, image_type)
134
135 return os.path.join(
136 self.buildroot, signing.INPUT_INSN_DIR_REL, "%s.instructions" % name
137 )
138
139 @staticmethod
140 def SplitCfgField(val):
141 """Split a string into multiple elements.
142
143 This centralizes our convention for multiple elements in the input files
144 being delimited by either a space or comma.
145
146 Args:
147 val: The string to split.
148
149 Returns:
150 The list of elements from having done split the string.
151 """
152 return val.replace(",", " ").split()
153
154 def GetChannels(self):
155 """Return the list of channels to sign for this board.
156
157 If the board-specific config doesn't specify a preference, we'll use the
158 common settings.
159 """
160 return self.SplitCfgField(self.cfg.get("insns", "channel"))
161
162 def GetKeysets(self, insns_merge=None):
163 """Return the list of keysets to sign for this board.
164
165 Args:
166 insns_merge: The additional section to look at over [insns].
167 """
168 # First load the default value from [insns.keyset] if available.
169 sections = ["insns"]
170 # Then overlay the [insns.xxx.keyset] if requested.
171 if insns_merge is not None:
172 sections += [insns_merge]
173
174 keyset = ""
175 for section in sections:
176 try:
177 keyset = self.cfg.get(section, "keyset")
178 except (configparser.NoSectionError, configparser.NoOptionError):
179 pass
180
181 # We do not perturb the order (e.g. using sorted() or making a set())
182 # because we want the behavior stable, and we want the input insns to
183 # explicitly control the order (since it has an impact on naming).
184 return self.SplitCfgField(keyset)
185
186 def GetAltInsnSets(self):
187 """Return the list of alternative insn sections."""
188 # We do not perturb the order (e.g. using sorted() or making a set())
189 # because we want the behavior stable, and we want the input insns to
190 # explicitly control the order (since it has an impact on naming).
191 ret = [x for x in self.cfg.sections() if x.startswith("insns.")]
192 return ret if ret else [None]
193
194 @staticmethod
195 def CopyConfigParser(config):
196 """Return a copy of a ConfigParser object.
197
198 The python folks broke the ability to use something like deepcopy:
199 https://bugs.python.org/issue16058
200 """
201 # Write the current config to a string io object.
202 data = io.StringIO()
203 config.write(data)
204 data.seek(0)
205
206 # Create a new ConfigParser from the serialized data.
207 ret = configparser.ConfigParser()
208 ret.read_file(data)
209
210 return ret
211
212 def OutputInsns(
213 self, output_file, sect_insns, sect_general, insns_merge=None
214 ):
215 """Generate the output instruction file for sending to the signer.
216
217 The override order is (later has precedence):
218 [insns]
219 [insns_merge] (should be named "insns.xxx")
220 sect_insns
221
222 Note: The format of the instruction file pushimage outputs (and the signer
223 reads) is not exactly the same as the instruction file pushimage reads.
224
225 Args:
226 output_file: The file to write the new instruction file to.
227 sect_insns: Items to set/override in the [insns] section.
228 sect_general: Items to set/override in the [general] section.
229 insns_merge: The alternative insns.xxx section to merge.
230 """
231 # Create a copy so we can clobber certain fields.
232 config = self.CopyConfigParser(self.cfg)
233 sect_insns = sect_insns.copy()
234
235 # Merge in the alternative insns section if need be.
236 if insns_merge is not None:
237 for k, v in config.items(insns_merge):
238 sect_insns.setdefault(k, v)
239
240 # Clear channel entry in instructions file, ensuring we only get
241 # one channel for the signer to look at. Then provide all the
242 # other details for this signing request to avoid any ambiguity
243 # and to avoid relying on encoding data into filenames.
244 for sect, fields in zip(
245 ("insns", "general"), (sect_insns, sect_general)
246 ):
247 if not config.has_section(sect):
248 config.add_section(sect)
249 for k, v in fields.items():
250 config.set(sect, k, v)
251
252 # Now prune the alternative sections.
253 for alt in self.GetAltInsnSets():
254 config.remove_section(alt)
255
256 output = io.StringIO()
257 config.write(output)
258 data = output.getvalue()
259 osutils.WriteFile(output_file, data)
260 logging.debug("generated insns file for %s:\n%s", self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400261
262
263def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
Alex Klein1699fab2022-09-08 08:46:06 -0600264 """Mark an instructions file for signing.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400265
Alex Klein1699fab2022-09-08 08:46:06 -0600266 This will upload a file to the GS bucket flagging an image for signing by
267 the signers.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400268
Alex Klein1699fab2022-09-08 08:46:06 -0600269 Args:
270 ctx: A viable gs.GSContext.
271 tbs_base: The full path to where the tobesigned directory lives.
272 insns_path: The path (relative to |tbs_base|) of the file to sign.
273 priority: Set the signing priority (lower == higher prio).
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400274
Alex Klein1699fab2022-09-08 08:46:06 -0600275 Returns:
276 The full path to the remote tobesigned file.
277 """
278 if priority < 0 or priority > 99:
279 raise ValueError("priority must be [0, 99] inclusive")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400280
Alex Klein1699fab2022-09-08 08:46:06 -0600281 if insns_path.startswith(tbs_base):
282 insns_path = insns_path[len(tbs_base) :].lstrip("/")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400283
Alex Klein1699fab2022-09-08 08:46:06 -0600284 tbs_path = "%s/tobesigned/%02i,%s" % (
285 tbs_base,
286 priority,
287 insns_path.replace("/", ","),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400288 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700289
Alex Klein1699fab2022-09-08 08:46:06 -0600290 # The caller will catch gs.GSContextException for us.
291 ctx.Copy("-", tbs_path, input=cros_build_lib.MachineDetails())
Amey Deshpandea936c622015-08-12 17:27:54 -0700292
Alex Klein1699fab2022-09-08 08:46:06 -0600293 return tbs_path
Amey Deshpandea936c622015-08-12 17:27:54 -0700294
David Rileya04d19d2015-09-04 16:11:50 -0700295
Alex Klein1699fab2022-09-08 08:46:06 -0600296def PushImage(
297 src_path,
298 board,
299 versionrev=None,
300 profile=None,
301 priority=50,
302 sign_types=None,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500303 dryrun=False,
Alex Klein1699fab2022-09-08 08:46:06 -0600304 mock=False,
305 force_keysets=(),
306 force_channels=None,
307 buildroot=constants.SOURCE_ROOT,
308 dest_bucket=constants.RELEASE_BUCKET,
309):
310 """Push the image from the archive bucket to the release bucket.
Evan Benn4d061102022-02-14 12:50:45 +1100311
Alex Klein1699fab2022-09-08 08:46:06 -0600312 Args:
313 src_path: Where to copy the files from; can be a local path or gs:// URL.
314 Should be a full path to the artifacts in either case.
315 board: The board we're uploading artifacts for (e.g. $BOARD).
316 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
317 profile: The board profile in use (e.g. "asan").
318 priority: Set the signing priority (lower == higher prio).
319 sign_types: If set, a set of types which we'll restrict ourselves to
320 signing. See the --sign-types option for more details.
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500321 dryrun: Show what would be done, but do not upload anything.
Alex Klein1699fab2022-09-08 08:46:06 -0600322 mock: Upload to a testing bucket rather than the real one.
323 force_keysets: Set of keysets to use rather than what the inputs say.
324 force_channels: Set of channels to use rather than what the inputs say.
325 buildroot: Buildroot in which to look for signing instructions.
326 dest_bucket: Bucket to push results to.
Vincent Palatind599c662015-10-26 09:51:41 -0700327
Alex Klein1699fab2022-09-08 08:46:06 -0600328 Returns:
329 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
330 'gs://signer_instruction_uri2',
331 ...]
332 """
333 # Whether we hit an unknown error. If so, we'll throw an error, but only
334 # at the end (so that we still upload as many files as possible).
335 # It's implemented using a list to deal with variable scopes in nested
336 # functions below.
337 unknown_error = [False]
LaMont Jones7d6c98f2019-09-27 12:37:33 -0600338
Alex Klein1699fab2022-09-08 08:46:06 -0600339 if versionrev is None:
340 # Extract milestone/version from the directory name.
341 versionrev = os.path.basename(src_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700342
Alex Klein1699fab2022-09-08 08:46:06 -0600343 # We only support the latest format here. Older releases can use pushimage
344 # from the respective branch which deals with legacy cruft.
345 m = re.match(VERSION_REGEX, versionrev)
346 if not m:
347 raise ValueError(
348 "version %s does not match %s" % (versionrev, VERSION_REGEX)
349 )
350 milestone = m.group(1)
351 version = m.group(2)
Amey Deshpandea936c622015-08-12 17:27:54 -0700352
Alex Klein1699fab2022-09-08 08:46:06 -0600353 # Normalize board to always use dashes not underscores. This is mostly a
354 # historical artifact at this point, but we can't really break it since the
355 # value is used in URLs.
356 boardpath = board.replace("_", "-")
357 if profile is not None:
358 boardpath += "-%s" % profile.replace("_", "-")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400359
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500360 ctx = gs.GSContext(dry_run=dryrun)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400361
Alex Klein1699fab2022-09-08 08:46:06 -0600362 try:
363 input_insns = InputInsns(board, buildroot=buildroot)
364 except MissingBoardInstructions as e:
365 logging.warning("Missing base instruction file: %s", e)
366 logging.warning("not uploading anything for signing")
367 return
368
369 if force_channels is None:
370 channels = input_insns.GetChannels()
371 else:
372 # Filter out duplicates.
373 channels = sorted(set(force_channels))
374
375 # We want force_keysets as a set.
376 force_keysets = set(force_keysets)
377
378 if mock:
379 logging.info("Upload mode: mock; signers will not process anything")
380 tbs_base = gs_base = os.path.join(
381 constants.TRASH_BUCKET, "pushimage-tests", getpass.getuser()
382 )
Alex Klein041edd82023-04-17 12:23:23 -0600383 elif {f"{TEST_KEYSET_PREFIX}-{x}" for x in TEST_KEYSETS} & force_keysets:
Alex Klein1699fab2022-09-08 08:46:06 -0600384 logging.info("Upload mode: test; signers will process test keys")
385 # We need the tbs_base to be in the place the signer will actually scan.
386 tbs_base = TEST_SIGN_BUCKET_BASE
387 gs_base = os.path.join(tbs_base, getpass.getuser())
388 else:
389 logging.info("Upload mode: normal; signers will process the images")
390 tbs_base = gs_base = dest_bucket
391
392 sect_general = {
393 "config_board": board,
394 "board": boardpath,
395 "version": version,
396 "versionrev": versionrev,
397 "milestone": milestone,
398 }
399 sect_insns = {}
400
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500401 if dryrun:
Alex Klein1699fab2022-09-08 08:46:06 -0600402 logging.info("DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED")
403 logging.info("Signing for channels: %s", " ".join(channels))
404
405 instruction_urls = {}
406
407 def _ImageNameBase(image_type=None):
408 lmid = ("%s-" % image_type) if image_type else ""
409 return "ChromeOS-%s%s-%s" % (lmid, versionrev, boardpath)
410
411 # These variables are defined outside the loop so that the nested functions
412 # below can access them without 'cell-var-from-loop' linter warning.
413 dst_path = ""
Amey Deshpandea936c622015-08-12 17:27:54 -0700414 files_to_sign = []
Alex Klein1699fab2022-09-08 08:46:06 -0600415 for channel in channels:
416 logging.debug("\n\n#### CHANNEL: %s ####\n", channel)
417 sect_insns["channel"] = channel
418 sub_path = "%s-channel/%s/%s" % (channel, boardpath, version)
419 dst_path = "%s/%s" % (gs_base, sub_path)
420 logging.info("Copying images to %s", dst_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700421
Alex Klein1699fab2022-09-08 08:46:06 -0600422 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
423 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
424 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
425 hps_firmware_basename = _ImageNameBase(
426 constants.IMAGE_TYPE_HPS_FIRMWARE
427 )
428 acc_usbpd_basename = _ImageNameBase(
429 constants.IMAGE_TYPE_ACCESSORY_USBPD
430 )
431 acc_rwsig_basename = _ImageNameBase(
432 constants.IMAGE_TYPE_ACCESSORY_RWSIG
433 )
434 gsc_firmware_basename = _ImageNameBase(
435 constants.IMAGE_TYPE_GSC_FIRMWARE
436 )
437 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
438 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
439 hwqual_tarball = "chromeos-hwqual-%s-%s.tar.bz2" % (board, versionrev)
Amey Deshpandea936c622015-08-12 17:27:54 -0700440
Alex Klein1699fab2022-09-08 08:46:06 -0600441 # The following build artifacts, if present, are always copied regardless of
442 # requested signing types.
443 files_to_copy_only = (
444 # (<src>, <dst>, <suffix>),
445 ("image.zip", _ImageNameBase(), "zip"),
446 (constants.TEST_IMAGE_TAR, test_basename, "tar.xz"),
447 ("debug.tgz", "debug-%s" % boardpath, "tgz"),
448 (hwqual_tarball, None, None),
449 ("stateful.tgz", None, None),
450 ("dlc", None, None),
451 (constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
452 (constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
453 (constants.QUICK_PROVISION_PAYLOAD_MINIOS, None, None),
454 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700455
Alex Klein1699fab2022-09-08 08:46:06 -0600456 # The following build artifacts, if present, are always copied.
457 # If |sign_types| is None, all of them are marked for signing, otherwise
458 # only the image types specified in |sign_types| are marked for signing.
459 files_to_copy_and_maybe_sign = (
460 # (<src>, <dst>, <suffix>, <signing type>),
461 (
462 constants.RECOVERY_IMAGE_TAR,
463 recovery_basename,
464 "tar.xz",
465 constants.IMAGE_TYPE_RECOVERY,
466 ),
467 (
468 "factory_image.zip",
469 factory_basename,
470 "zip",
471 constants.IMAGE_TYPE_FACTORY,
472 ),
473 (
474 "firmware_from_source.tar.bz2",
475 firmware_basename,
476 "tar.bz2",
477 constants.IMAGE_TYPE_FIRMWARE,
478 ),
479 (
480 "firmware_from_source.tar.bz2",
481 hps_firmware_basename,
482 "tar.bz2",
483 constants.IMAGE_TYPE_HPS_FIRMWARE,
484 ),
485 (
486 "firmware_from_source.tar.bz2",
487 acc_usbpd_basename,
488 "tar.bz2",
489 constants.IMAGE_TYPE_ACCESSORY_USBPD,
490 ),
491 (
492 "firmware_from_source.tar.bz2",
493 acc_rwsig_basename,
494 "tar.bz2",
495 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
496 ),
497 (
498 "firmware_from_source.tar.bz2",
499 gsc_firmware_basename,
500 "tar.bz2",
501 constants.IMAGE_TYPE_GSC_FIRMWARE,
502 ),
503 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700504
Alex Klein1699fab2022-09-08 08:46:06 -0600505 # The following build artifacts are copied and marked for signing, if
506 # they are present *and* if the image type is specified via |sign_types|.
507 files_to_maybe_copy_and_sign = (
508 # (<src>, <dst>, <suffix>, <signing type>),
509 (
510 constants.BASE_IMAGE_TAR,
511 base_basename,
512 "tar.xz",
513 constants.IMAGE_TYPE_BASE,
514 ),
515 )
Evan Bennb9dfaf42022-02-14 17:38:26 +1100516
Alex Klein1699fab2022-09-08 08:46:06 -0600517 def _CopyFileToGS(src, dst=None, suffix=None):
518 """Returns |dst| file name if the copying was successful."""
519 if dst is None:
520 dst = src
521 elif suffix is not None:
522 dst = "%s.%s" % (dst, suffix)
523 success = False
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500524 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600525 ctx.Copy(
526 os.path.join(src_path, src),
527 os.path.join(dst_path, dst),
528 recursive=True,
529 )
530 success = True
531 except gs.GSNoSuchKey:
532 logging.warning("Skipping %s as it does not exist", src)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500533 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600534 unknown_error[0] = True
535 logging.error(
536 "Skipping %s due to unknown GS error", src, exc_info=True
537 )
538 return dst if success else None
Mike Frysinger4495b032014-03-05 17:24:03 -0500539
Alex Klein1699fab2022-09-08 08:46:06 -0600540 for src, dst, suffix in files_to_copy_only:
541 _CopyFileToGS(src, dst, suffix)
542
543 # Clear the list of files to sign before adding new artifacts.
544 files_to_sign = []
545
546 def _AddToFilesToSign(image_type, dst, suffix):
547 assert dst.endswith("." + suffix), "dst: %s, suffix: %s" % (
548 dst,
549 suffix,
550 )
551 dst_base = dst[: -(len(suffix) + 1)]
552 files_to_sign.append([image_type, dst_base, suffix])
553
554 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
555 dst = _CopyFileToGS(src, dst, suffix)
556 if dst and (not sign_types or image_type in sign_types):
557 _AddToFilesToSign(image_type, dst, suffix)
558
559 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
560 if sign_types and image_type in sign_types:
561 dst = _CopyFileToGS(src, dst, suffix)
562 if dst:
563 _AddToFilesToSign(image_type, dst, suffix)
564
565 logging.debug("Files to sign: %s", files_to_sign)
566 unused_sign_types = set(sign_types or []) - set(
567 x for x, _, _ in files_to_sign
568 )
569 if unused_sign_types:
570 logging.warning(
571 "Some sign types were unused: %s", " ".join(unused_sign_types)
572 )
573
574 # Now go through the subset for signing.
575 for image_type, dst_name, suffix in files_to_sign:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500576 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600577 input_insns = InputInsns(
578 board, image_type=image_type, buildroot=buildroot
579 )
580 except MissingBoardInstructions as e:
581 logging.info("Nothing to sign: %s", e)
582 continue
583
584 dst_archive = "%s.%s" % (dst_name, suffix)
585 sect_general["archive"] = dst_archive
586 sect_general["type"] = image_type
587
588 # In the default/automatic mode, only flag files for signing if the
589 # archives were actually uploaded in a previous stage. This additional
590 # check can be removed in future once |sign_types| becomes a required
591 # argument.
592 # TODO: Make |sign_types| a required argument.
593 gs_artifact_path = os.path.join(dst_path, dst_archive)
594 exists = False
595 try:
596 exists = ctx.Exists(gs_artifact_path)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500597 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600598 unknown_error[0] = True
599 logging.error(
600 "Unknown error while checking %s",
601 gs_artifact_path,
602 exc_info=True,
603 )
604 if not exists:
605 logging.info(
606 "%s does not exist. Nothing to sign.", gs_artifact_path
607 )
608 continue
Don Garrett9459c2f2014-01-22 18:20:24 -0800609
Alex Klein1699fab2022-09-08 08:46:06 -0600610 first_image = True
611 for alt_insn_set in input_insns.GetAltInsnSets():
612 # Figure out which keysets have been requested for this type.
613 # We sort the forced set so tests/runtime behavior is stable.
614 keysets = sorted(force_keysets)
615 if not keysets:
616 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
617 if not keysets:
618 logging.warning(
619 "Skipping %s image signing due to no keysets",
620 image_type,
621 )
Mike Frysinger4495b032014-03-05 17:24:03 -0500622
Alex Klein1699fab2022-09-08 08:46:06 -0600623 for keyset in keysets:
624 sect_insns["keyset"] = keyset
625
626 # Generate the insn file for this artifact that the signer will use,
627 # and flag it for signing.
628 with cros_build_lib.UnbufferedNamedTemporaryFile(
629 prefix="pushimage.insns."
630 ) as insns_path:
631 input_insns.OutputInsns(
632 insns_path.name,
633 sect_insns,
634 sect_general,
635 insns_merge=alt_insn_set,
636 )
637
638 gs_insns_path = "%s/%s" % (dst_path, dst_name)
639 if not first_image:
640 gs_insns_path += "-%s" % keyset
641 first_image = False
642 gs_insns_path += ".instructions"
643
644 try:
645 ctx.Copy(insns_path.name, gs_insns_path)
646 except gs.GSContextException:
647 unknown_error[0] = True
648 logging.error(
649 "Unknown error while uploading insns %s",
650 gs_insns_path,
651 exc_info=True,
652 )
653 continue
654
655 try:
656 MarkImageToBeSigned(
657 ctx, tbs_base, gs_insns_path, priority
658 )
659 except gs.GSContextException:
660 unknown_error[0] = True
661 logging.error(
662 "Unknown error while marking for signing %s",
663 gs_insns_path,
664 exc_info=True,
665 )
666 continue
667 logging.info(
668 "Signing %s image with keyset %s at %s",
669 image_type,
670 keyset,
671 gs_insns_path,
672 )
673 instruction_urls.setdefault(channel, []).append(
674 gs_insns_path
675 )
676
677 if unknown_error[0]:
678 raise PushError("hit some unknown error(s)", instruction_urls)
679
680 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400681
682
Mike Frysinger26144192017-08-30 18:26:46 -0400683def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600684 """Creates the argparse parser."""
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500685 parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400686
Alex Klein1699fab2022-09-08 08:46:06 -0600687 # The type of image_dir will strip off trailing slashes (makes later
688 # processing simpler and the display prettier).
689 parser.add_argument(
690 "image_dir",
691 default=None,
692 type="local_or_gs_path",
693 help="full path of source artifacts to upload",
694 )
695 parser.add_argument(
696 "--board",
697 default=None,
698 required=True,
699 help="board to generate symbols for",
700 )
701 parser.add_argument(
702 "--profile", default=None, help='board profile in use (e.g. "asan")'
703 )
704 parser.add_argument(
705 "--version",
706 default=None,
707 help="version info (normally extracted from image_dir)",
708 )
709 parser.add_argument(
710 "--channels",
711 default=None,
712 action="split_extend",
713 help="override list of channels to process",
714 )
715 parser.add_argument(
Alex Klein1699fab2022-09-08 08:46:06 -0600716 "-M",
717 "--mock",
718 default=False,
719 action="store_true",
720 help="upload things to a testing bucket (dev testing)",
721 )
722 parser.add_argument(
723 "--test-sign",
724 default=[],
725 action="append",
726 choices=TEST_KEYSETS,
727 help="mung signing behavior to sign w/ test keys",
728 )
729 parser.add_argument(
730 "--priority",
731 type=int,
732 default=50,
733 help="set signing priority (lower == higher prio)",
734 )
735 parser.add_argument(
736 "--sign-types",
737 default=None,
738 nargs="+",
739 choices=_SUPPORTED_IMAGE_TYPES,
740 help="only sign specified image types",
741 )
742 parser.add_argument(
743 "--buildroot",
744 default=constants.SOURCE_ROOT,
745 type="path",
746 help="Buildroot to use. Defaults to current.",
747 )
748 parser.add_argument(
749 "--yes",
750 action="store_true",
751 default=False,
752 help="answer yes to all prompts",
753 )
754 parser.add_argument(
755 "--dest-bucket",
756 default=constants.RELEASE_BUCKET,
757 help="dest bucket. Default to %(default)s",
758 )
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400759
Alex Klein1699fab2022-09-08 08:46:06 -0600760 return parser
Mike Frysinger26144192017-08-30 18:26:46 -0400761
762
763def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600764 parser = GetParser()
765 opts = parser.parse_args(argv)
766 opts.Freeze()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400767
Alex Klein041edd82023-04-17 12:23:23 -0600768 force_keysets = {f"{TEST_KEYSET_PREFIX}-{x}" for x in opts.test_sign}
Mike Frysingerdad40d62014-02-09 02:18:02 -0500769
Alex Klein1699fab2022-09-08 08:46:06 -0600770 # If we aren't using mock or test or dry run mode, then let's prompt the user
771 # to make sure they actually want to do this. It's rare that people want to
772 # run this directly and hit the release bucket.
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500773 if not (opts.mock or force_keysets or opts.dryrun) and not opts.yes:
Alex Klein1699fab2022-09-08 08:46:06 -0600774 prolog = "\n".join(
775 textwrap.wrap(
776 textwrap.dedent(
777 "Uploading images for signing to the *release* bucket is not something "
778 "you generally should be doing yourself."
779 ),
780 80,
781 )
782 ).strip()
783 if not cros_build_lib.BooleanPrompt(
784 prompt="Are you sure you want to sign these images",
785 default=False,
786 prolog=prolog,
787 ):
788 cros_build_lib.Die("better safe than sorry")
Mike Frysinger09fe0122014-02-09 02:44:05 -0500789
Alex Klein1699fab2022-09-08 08:46:06 -0600790 PushImage(
791 opts.image_dir,
792 opts.board,
793 versionrev=opts.version,
794 profile=opts.profile,
795 priority=opts.priority,
796 sign_types=opts.sign_types,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500797 dryrun=opts.dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600798 mock=opts.mock,
799 force_keysets=force_keysets,
800 force_channels=opts.channels,
801 buildroot=opts.buildroot,
802 dest_bucket=opts.dest_bucket,
803 )