blob: d07aca69cf2a22d622b1c5577a536aadaba62f32 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2013 The ChromiumOS Authors
Mike Frysingerd13faeb2013-09-05 16:00:46 -04002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysingere852b072021-05-21 12:39:03 -040011import configparser
Mike Frysingerd13faeb2013-09-05 16:00:46 -040012import getpass
Mike Frysingere852b072021-05-21 12:39:03 -040013import io
Chris McDonald59650c32021-07-20 15:29:28 -060014import logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import os
16import re
Mike Frysinger09fe0122014-02-09 02:44:05 -050017import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040018
Mike Frysingerd13faeb2013-09-05 16:00:46 -040019from chromite.lib import commandline
Chris McDonald59650c32021-07-20 15:29:28 -060020from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import gs
23from chromite.lib import osutils
24from chromite.lib import signing
25
26
27# This will split a fully qualified ChromeOS version string up.
28# R34-5126.0.0 will break into "34" and "5126.0.0".
Alex Klein1699fab2022-09-08 08:46:06 -060029VERSION_REGEX = r"^R([0-9]+)-([^-]+)"
Mike Frysingerd13faeb2013-09-05 16:00:46 -040030
Mike Frysingerdad40d62014-02-09 02:18:02 -050031# The test signers will scan this dir looking for test work.
32# Keep it in sync with the signer config files [gs_test_buckets].
Alex Klein1699fab2022-09-08 08:46:06 -060033TEST_SIGN_BUCKET_BASE = "gs://chromeos-throw-away-bucket/signer-tests"
Mike Frysingerdad40d62014-02-09 02:18:02 -050034
David Rileyf8205122015-09-04 13:46:36 -070035# Keysets that are only valid in the above test bucket.
Alex Klein1699fab2022-09-08 08:46:06 -060036TEST_KEYSET_PREFIX = "test-keys"
Alex Klein041edd82023-04-17 12:23:23 -060037TEST_KEYSETS = {"mp", "premp", "nvidia-premp"}
Mike Frysingerdad40d62014-02-09 02:18:02 -050038
Amey Deshpandea936c622015-08-12 17:27:54 -070039# Supported image types for signing.
40_SUPPORTED_IMAGE_TYPES = (
41 constants.IMAGE_TYPE_RECOVERY,
42 constants.IMAGE_TYPE_FACTORY,
43 constants.IMAGE_TYPE_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070044 constants.IMAGE_TYPE_ACCESSORY_USBPD,
Evan Benn4d061102022-02-14 12:50:45 +110045 constants.IMAGE_TYPE_HPS_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070046 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070047 constants.IMAGE_TYPE_BASE,
Vadim Bendeburyfe37f282020-11-04 19:05:49 -080048 constants.IMAGE_TYPE_GSC_FIRMWARE,
Amey Deshpandea936c622015-08-12 17:27:54 -070049)
50
Mike Frysingerd13faeb2013-09-05 16:00:46 -040051
Mike Frysinger4495b032014-03-05 17:24:03 -050052class PushError(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060053 """When an (unknown) error happened while trying to push artifacts."""
Mike Frysinger4495b032014-03-05 17:24:03 -050054
55
Mike Frysingerd13faeb2013-09-05 16:00:46 -040056class MissingBoardInstructions(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060057 """Raised when a board lacks any signer instructions."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -040058
Alex Klein1699fab2022-09-08 08:46:06 -060059 def __init__(self, board, image_type, input_insns):
60 Exception.__init__(
61 self,
62 "Board %s lacks insns for %s image: %s not found"
63 % (board, image_type, input_insns),
64 )
Mike Frysingerd84d91e2015-11-05 18:02:24 -050065
Mike Frysingerd13faeb2013-09-05 16:00:46 -040066
67class InputInsns(object):
Alex Klein1699fab2022-09-08 08:46:06 -060068 """Object to hold settings for a signable board.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050069
Mike Frysingerd13faeb2013-09-05 16:00:46 -040070 Note: The format of the instruction file pushimage outputs (and the signer
71 reads) is not exactly the same as the instruction file pushimage reads.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040072 """
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050073
Alex Klein1699fab2022-09-08 08:46:06 -060074 def __init__(self, board, image_type=None, buildroot=None):
75 """Initialization.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040076
Alex Klein1699fab2022-09-08 08:46:06 -060077 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +100078 board: The board to look up details.
79 image_type: The type of image we will be signing (see --sign-types).
80 buildroot: Buildroot in which to look for signing instructions.
Alex Klein1699fab2022-09-08 08:46:06 -060081 """
82 self.board = board
83 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040084
Alex Klein1699fab2022-09-08 08:46:06 -060085 config = configparser.ConfigParser()
Mike Frysinger31fdddd2023-02-24 15:50:55 -050086 with open(self.GetInsnFile("DEFAULT"), encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060087 config.read_file(fp)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050088
Alex Klein1699fab2022-09-08 08:46:06 -060089 # What pushimage internally refers to as 'recovery', are the basic signing
90 # instructions in practice, and other types are stacked on top.
91 if image_type is None:
92 image_type = constants.IMAGE_TYPE_RECOVERY
93 self.image_type = image_type
94 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
95 if not os.path.exists(input_insns):
96 # This board doesn't have any signing instructions.
97 raise MissingBoardInstructions(self.board, image_type, input_insns)
Mike Frysinger31fdddd2023-02-24 15:50:55 -050098 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060099 config.read_file(fp)
100
101 if image_type is not None:
102 input_insns = self.GetInsnFile(image_type)
103 if not os.path.exists(input_insns):
104 # This type doesn't have any signing instructions.
105 raise MissingBoardInstructions(
106 self.board, image_type, input_insns
107 )
108
109 self.image_type = image_type
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500110 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -0600111 config.read_file(fp)
112
113 self.cfg = config
114
115 def GetInsnFile(self, image_type):
116 """Find the signer instruction files for this board/image type.
117
118 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000119 image_type: The type of instructions to load. It can be a common
120 file (like "DEFAULT"), or one of the --sign-types.
Alex Klein1699fab2022-09-08 08:46:06 -0600121
122 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000123 Full path to the instruction file using |image_type| and
124 |self.board|.
Alex Klein1699fab2022-09-08 08:46:06 -0600125 """
126 if image_type == image_type.upper():
127 name = image_type
128 elif image_type in (
129 constants.IMAGE_TYPE_RECOVERY,
130 constants.IMAGE_TYPE_BASE,
131 ):
132 name = self.board
133 else:
134 name = "%s.%s" % (self.board, image_type)
135
136 return os.path.join(
137 self.buildroot, signing.INPUT_INSN_DIR_REL, "%s.instructions" % name
138 )
139
140 @staticmethod
141 def SplitCfgField(val):
142 """Split a string into multiple elements.
143
144 This centralizes our convention for multiple elements in the input files
145 being delimited by either a space or comma.
146
147 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000148 val: The string to split.
Alex Klein1699fab2022-09-08 08:46:06 -0600149
150 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000151 The list of elements from having done split the string.
Alex Klein1699fab2022-09-08 08:46:06 -0600152 """
153 return val.replace(",", " ").split()
154
155 def GetChannels(self):
156 """Return the list of channels to sign for this board.
157
158 If the board-specific config doesn't specify a preference, we'll use the
159 common settings.
160 """
161 return self.SplitCfgField(self.cfg.get("insns", "channel"))
162
163 def GetKeysets(self, insns_merge=None):
164 """Return the list of keysets to sign for this board.
165
166 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000167 insns_merge: The additional section to look at over [insns].
Alex Klein1699fab2022-09-08 08:46:06 -0600168 """
169 # First load the default value from [insns.keyset] if available.
170 sections = ["insns"]
171 # Then overlay the [insns.xxx.keyset] if requested.
172 if insns_merge is not None:
173 sections += [insns_merge]
174
175 keyset = ""
176 for section in sections:
177 try:
178 keyset = self.cfg.get(section, "keyset")
179 except (configparser.NoSectionError, configparser.NoOptionError):
180 pass
181
182 # We do not perturb the order (e.g. using sorted() or making a set())
183 # because we want the behavior stable, and we want the input insns to
184 # explicitly control the order (since it has an impact on naming).
185 return self.SplitCfgField(keyset)
186
187 def GetAltInsnSets(self):
188 """Return the list of alternative insn sections."""
189 # We do not perturb the order (e.g. using sorted() or making a set())
190 # because we want the behavior stable, and we want the input insns to
191 # explicitly control the order (since it has an impact on naming).
192 ret = [x for x in self.cfg.sections() if x.startswith("insns.")]
193 return ret if ret else [None]
194
195 @staticmethod
196 def CopyConfigParser(config):
197 """Return a copy of a ConfigParser object.
198
199 The python folks broke the ability to use something like deepcopy:
200 https://bugs.python.org/issue16058
201 """
202 # Write the current config to a string io object.
203 data = io.StringIO()
204 config.write(data)
205 data.seek(0)
206
207 # Create a new ConfigParser from the serialized data.
208 ret = configparser.ConfigParser()
209 ret.read_file(data)
210
211 return ret
212
213 def OutputInsns(
214 self, output_file, sect_insns, sect_general, insns_merge=None
215 ):
216 """Generate the output instruction file for sending to the signer.
217
218 The override order is (later has precedence):
219 [insns]
220 [insns_merge] (should be named "insns.xxx")
221 sect_insns
222
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000223 Note: The format of the instruction file pushimage outputs (and the
224 signer reads) is not exactly the same as the instruction file pushimage
225 reads.
Alex Klein1699fab2022-09-08 08:46:06 -0600226
227 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000228 output_file: The file to write the new instruction file to.
229 sect_insns: Items to set/override in the [insns] section.
230 sect_general: Items to set/override in the [general] section.
231 insns_merge: The alternative insns.xxx section to merge.
Alex Klein1699fab2022-09-08 08:46:06 -0600232 """
233 # Create a copy so we can clobber certain fields.
234 config = self.CopyConfigParser(self.cfg)
235 sect_insns = sect_insns.copy()
236
237 # Merge in the alternative insns section if need be.
238 if insns_merge is not None:
239 for k, v in config.items(insns_merge):
240 sect_insns.setdefault(k, v)
241
242 # Clear channel entry in instructions file, ensuring we only get
243 # one channel for the signer to look at. Then provide all the
244 # other details for this signing request to avoid any ambiguity
245 # and to avoid relying on encoding data into filenames.
246 for sect, fields in zip(
247 ("insns", "general"), (sect_insns, sect_general)
248 ):
249 if not config.has_section(sect):
250 config.add_section(sect)
251 for k, v in fields.items():
252 config.set(sect, k, v)
253
254 # Now prune the alternative sections.
255 for alt in self.GetAltInsnSets():
256 config.remove_section(alt)
257
258 output = io.StringIO()
259 config.write(output)
260 data = output.getvalue()
261 osutils.WriteFile(output_file, data)
262 logging.debug("generated insns file for %s:\n%s", self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400263
264
265def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
Alex Klein1699fab2022-09-08 08:46:06 -0600266 """Mark an instructions file for signing.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400267
Alex Klein1699fab2022-09-08 08:46:06 -0600268 This will upload a file to the GS bucket flagging an image for signing by
269 the signers.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400270
Alex Klein1699fab2022-09-08 08:46:06 -0600271 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000272 ctx: A viable gs.GSContext.
273 tbs_base: The full path to where the tobesigned directory lives.
274 insns_path: The path (relative to |tbs_base|) of the file to sign.
275 priority: Set the signing priority (lower == higher prio).
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400276
Alex Klein1699fab2022-09-08 08:46:06 -0600277 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000278 The full path to the remote tobesigned file.
Alex Klein1699fab2022-09-08 08:46:06 -0600279 """
280 if priority < 0 or priority > 99:
281 raise ValueError("priority must be [0, 99] inclusive")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400282
Alex Klein1699fab2022-09-08 08:46:06 -0600283 if insns_path.startswith(tbs_base):
284 insns_path = insns_path[len(tbs_base) :].lstrip("/")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400285
Alex Klein1699fab2022-09-08 08:46:06 -0600286 tbs_path = "%s/tobesigned/%02i,%s" % (
287 tbs_base,
288 priority,
289 insns_path.replace("/", ","),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400290 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700291
Alex Klein1699fab2022-09-08 08:46:06 -0600292 # The caller will catch gs.GSContextException for us.
293 ctx.Copy("-", tbs_path, input=cros_build_lib.MachineDetails())
Amey Deshpandea936c622015-08-12 17:27:54 -0700294
Alex Klein1699fab2022-09-08 08:46:06 -0600295 return tbs_path
Amey Deshpandea936c622015-08-12 17:27:54 -0700296
David Rileya04d19d2015-09-04 16:11:50 -0700297
Alex Klein1699fab2022-09-08 08:46:06 -0600298def PushImage(
299 src_path,
300 board,
301 versionrev=None,
302 profile=None,
303 priority=50,
304 sign_types=None,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500305 dryrun=False,
Alex Klein1699fab2022-09-08 08:46:06 -0600306 mock=False,
307 force_keysets=(),
308 force_channels=None,
309 buildroot=constants.SOURCE_ROOT,
310 dest_bucket=constants.RELEASE_BUCKET,
311):
312 """Push the image from the archive bucket to the release bucket.
Evan Benn4d061102022-02-14 12:50:45 +1100313
Alex Klein1699fab2022-09-08 08:46:06 -0600314 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000315 src_path: Where to copy the files from; can be a local path or gs://
316 URL. Should be a full path to the artifacts in either case.
317 board: The board we're uploading artifacts for (e.g. $BOARD).
318 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
319 profile: The board profile in use (e.g. "asan").
320 priority: Set the signing priority (lower == higher prio).
321 sign_types: If set, a set of types which we'll restrict ourselves to
322 signing. See the --sign-types option for more details.
323 dryrun: Show what would be done, but do not upload anything.
324 mock: Upload to a testing bucket rather than the real one.
325 force_keysets: Set of keysets to use rather than what the inputs say.
326 force_channels: Set of channels to use rather than what the inputs say.
327 buildroot: Buildroot in which to look for signing instructions.
328 dest_bucket: Bucket to push results to.
Vincent Palatind599c662015-10-26 09:51:41 -0700329
Alex Klein1699fab2022-09-08 08:46:06 -0600330 Returns:
331 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
332 'gs://signer_instruction_uri2',
333 ...]
334 """
335 # Whether we hit an unknown error. If so, we'll throw an error, but only
336 # at the end (so that we still upload as many files as possible).
337 # It's implemented using a list to deal with variable scopes in nested
338 # functions below.
339 unknown_error = [False]
LaMont Jones7d6c98f2019-09-27 12:37:33 -0600340
Alex Klein1699fab2022-09-08 08:46:06 -0600341 if versionrev is None:
342 # Extract milestone/version from the directory name.
343 versionrev = os.path.basename(src_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700344
Alex Klein1699fab2022-09-08 08:46:06 -0600345 # We only support the latest format here. Older releases can use pushimage
346 # from the respective branch which deals with legacy cruft.
347 m = re.match(VERSION_REGEX, versionrev)
348 if not m:
349 raise ValueError(
350 "version %s does not match %s" % (versionrev, VERSION_REGEX)
351 )
352 milestone = m.group(1)
353 version = m.group(2)
Amey Deshpandea936c622015-08-12 17:27:54 -0700354
Alex Klein1699fab2022-09-08 08:46:06 -0600355 # Normalize board to always use dashes not underscores. This is mostly a
356 # historical artifact at this point, but we can't really break it since the
357 # value is used in URLs.
358 boardpath = board.replace("_", "-")
359 if profile is not None:
360 boardpath += "-%s" % profile.replace("_", "-")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400361
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500362 ctx = gs.GSContext(dry_run=dryrun)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400363
Alex Klein1699fab2022-09-08 08:46:06 -0600364 try:
365 input_insns = InputInsns(board, buildroot=buildroot)
366 except MissingBoardInstructions as e:
367 logging.warning("Missing base instruction file: %s", e)
368 logging.warning("not uploading anything for signing")
369 return
370
371 if force_channels is None:
372 channels = input_insns.GetChannels()
373 else:
374 # Filter out duplicates.
375 channels = sorted(set(force_channels))
376
377 # We want force_keysets as a set.
378 force_keysets = set(force_keysets)
379
380 if mock:
381 logging.info("Upload mode: mock; signers will not process anything")
382 tbs_base = gs_base = os.path.join(
383 constants.TRASH_BUCKET, "pushimage-tests", getpass.getuser()
384 )
Alex Klein041edd82023-04-17 12:23:23 -0600385 elif {f"{TEST_KEYSET_PREFIX}-{x}" for x in TEST_KEYSETS} & force_keysets:
Alex Klein1699fab2022-09-08 08:46:06 -0600386 logging.info("Upload mode: test; signers will process test keys")
387 # We need the tbs_base to be in the place the signer will actually scan.
388 tbs_base = TEST_SIGN_BUCKET_BASE
389 gs_base = os.path.join(tbs_base, getpass.getuser())
390 else:
391 logging.info("Upload mode: normal; signers will process the images")
392 tbs_base = gs_base = dest_bucket
393
394 sect_general = {
395 "config_board": board,
396 "board": boardpath,
397 "version": version,
398 "versionrev": versionrev,
399 "milestone": milestone,
400 }
401 sect_insns = {}
402
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500403 if dryrun:
Alex Klein1699fab2022-09-08 08:46:06 -0600404 logging.info("DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED")
405 logging.info("Signing for channels: %s", " ".join(channels))
406
407 instruction_urls = {}
408
409 def _ImageNameBase(image_type=None):
410 lmid = ("%s-" % image_type) if image_type else ""
411 return "ChromeOS-%s%s-%s" % (lmid, versionrev, boardpath)
412
413 # These variables are defined outside the loop so that the nested functions
414 # below can access them without 'cell-var-from-loop' linter warning.
415 dst_path = ""
Amey Deshpandea936c622015-08-12 17:27:54 -0700416 files_to_sign = []
Alex Klein1699fab2022-09-08 08:46:06 -0600417 for channel in channels:
418 logging.debug("\n\n#### CHANNEL: %s ####\n", channel)
419 sect_insns["channel"] = channel
420 sub_path = "%s-channel/%s/%s" % (channel, boardpath, version)
421 dst_path = "%s/%s" % (gs_base, sub_path)
422 logging.info("Copying images to %s", dst_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700423
Alex Klein1699fab2022-09-08 08:46:06 -0600424 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
425 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
426 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
427 hps_firmware_basename = _ImageNameBase(
428 constants.IMAGE_TYPE_HPS_FIRMWARE
429 )
430 acc_usbpd_basename = _ImageNameBase(
431 constants.IMAGE_TYPE_ACCESSORY_USBPD
432 )
433 acc_rwsig_basename = _ImageNameBase(
434 constants.IMAGE_TYPE_ACCESSORY_RWSIG
435 )
436 gsc_firmware_basename = _ImageNameBase(
437 constants.IMAGE_TYPE_GSC_FIRMWARE
438 )
439 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
440 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
441 hwqual_tarball = "chromeos-hwqual-%s-%s.tar.bz2" % (board, versionrev)
Amey Deshpandea936c622015-08-12 17:27:54 -0700442
Alex Klein1699fab2022-09-08 08:46:06 -0600443 # The following build artifacts, if present, are always copied regardless of
444 # requested signing types.
445 files_to_copy_only = (
446 # (<src>, <dst>, <suffix>),
447 ("image.zip", _ImageNameBase(), "zip"),
448 (constants.TEST_IMAGE_TAR, test_basename, "tar.xz"),
449 ("debug.tgz", "debug-%s" % boardpath, "tgz"),
450 (hwqual_tarball, None, None),
451 ("stateful.tgz", None, None),
452 ("dlc", None, None),
453 (constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
454 (constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
455 (constants.QUICK_PROVISION_PAYLOAD_MINIOS, None, None),
456 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700457
Alex Klein1699fab2022-09-08 08:46:06 -0600458 # The following build artifacts, if present, are always copied.
459 # If |sign_types| is None, all of them are marked for signing, otherwise
460 # only the image types specified in |sign_types| are marked for signing.
461 files_to_copy_and_maybe_sign = (
462 # (<src>, <dst>, <suffix>, <signing type>),
463 (
464 constants.RECOVERY_IMAGE_TAR,
465 recovery_basename,
466 "tar.xz",
467 constants.IMAGE_TYPE_RECOVERY,
468 ),
469 (
470 "factory_image.zip",
471 factory_basename,
472 "zip",
473 constants.IMAGE_TYPE_FACTORY,
474 ),
475 (
476 "firmware_from_source.tar.bz2",
477 firmware_basename,
478 "tar.bz2",
479 constants.IMAGE_TYPE_FIRMWARE,
480 ),
481 (
482 "firmware_from_source.tar.bz2",
483 hps_firmware_basename,
484 "tar.bz2",
485 constants.IMAGE_TYPE_HPS_FIRMWARE,
486 ),
487 (
488 "firmware_from_source.tar.bz2",
489 acc_usbpd_basename,
490 "tar.bz2",
491 constants.IMAGE_TYPE_ACCESSORY_USBPD,
492 ),
493 (
494 "firmware_from_source.tar.bz2",
495 acc_rwsig_basename,
496 "tar.bz2",
497 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
498 ),
499 (
500 "firmware_from_source.tar.bz2",
501 gsc_firmware_basename,
502 "tar.bz2",
503 constants.IMAGE_TYPE_GSC_FIRMWARE,
504 ),
505 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700506
Alex Klein1699fab2022-09-08 08:46:06 -0600507 # The following build artifacts are copied and marked for signing, if
508 # they are present *and* if the image type is specified via |sign_types|.
509 files_to_maybe_copy_and_sign = (
510 # (<src>, <dst>, <suffix>, <signing type>),
511 (
512 constants.BASE_IMAGE_TAR,
513 base_basename,
514 "tar.xz",
515 constants.IMAGE_TYPE_BASE,
516 ),
517 )
Evan Bennb9dfaf42022-02-14 17:38:26 +1100518
Alex Klein1699fab2022-09-08 08:46:06 -0600519 def _CopyFileToGS(src, dst=None, suffix=None):
520 """Returns |dst| file name if the copying was successful."""
521 if dst is None:
522 dst = src
523 elif suffix is not None:
524 dst = "%s.%s" % (dst, suffix)
525 success = False
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500526 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600527 ctx.Copy(
528 os.path.join(src_path, src),
529 os.path.join(dst_path, dst),
530 recursive=True,
531 )
532 success = True
533 except gs.GSNoSuchKey:
534 logging.warning("Skipping %s as it does not exist", src)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500535 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600536 unknown_error[0] = True
537 logging.error(
538 "Skipping %s due to unknown GS error", src, exc_info=True
539 )
540 return dst if success else None
Mike Frysinger4495b032014-03-05 17:24:03 -0500541
Alex Klein1699fab2022-09-08 08:46:06 -0600542 for src, dst, suffix in files_to_copy_only:
543 _CopyFileToGS(src, dst, suffix)
544
545 # Clear the list of files to sign before adding new artifacts.
546 files_to_sign = []
547
548 def _AddToFilesToSign(image_type, dst, suffix):
549 assert dst.endswith("." + suffix), "dst: %s, suffix: %s" % (
550 dst,
551 suffix,
552 )
553 dst_base = dst[: -(len(suffix) + 1)]
554 files_to_sign.append([image_type, dst_base, suffix])
555
556 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
557 dst = _CopyFileToGS(src, dst, suffix)
558 if dst and (not sign_types or image_type in sign_types):
559 _AddToFilesToSign(image_type, dst, suffix)
560
561 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
562 if sign_types and image_type in sign_types:
563 dst = _CopyFileToGS(src, dst, suffix)
564 if dst:
565 _AddToFilesToSign(image_type, dst, suffix)
566
567 logging.debug("Files to sign: %s", files_to_sign)
568 unused_sign_types = set(sign_types or []) - set(
569 x for x, _, _ in files_to_sign
570 )
571 if unused_sign_types:
572 logging.warning(
573 "Some sign types were unused: %s", " ".join(unused_sign_types)
574 )
575
576 # Now go through the subset for signing.
577 for image_type, dst_name, suffix in files_to_sign:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500578 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600579 input_insns = InputInsns(
580 board, image_type=image_type, buildroot=buildroot
581 )
582 except MissingBoardInstructions as e:
583 logging.info("Nothing to sign: %s", e)
584 continue
585
586 dst_archive = "%s.%s" % (dst_name, suffix)
587 sect_general["archive"] = dst_archive
588 sect_general["type"] = image_type
589
590 # In the default/automatic mode, only flag files for signing if the
591 # archives were actually uploaded in a previous stage. This additional
592 # check can be removed in future once |sign_types| becomes a required
593 # argument.
594 # TODO: Make |sign_types| a required argument.
595 gs_artifact_path = os.path.join(dst_path, dst_archive)
596 exists = False
597 try:
598 exists = ctx.Exists(gs_artifact_path)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500599 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600600 unknown_error[0] = True
601 logging.error(
602 "Unknown error while checking %s",
603 gs_artifact_path,
604 exc_info=True,
605 )
606 if not exists:
607 logging.info(
608 "%s does not exist. Nothing to sign.", gs_artifact_path
609 )
610 continue
Don Garrett9459c2f2014-01-22 18:20:24 -0800611
Alex Klein1699fab2022-09-08 08:46:06 -0600612 first_image = True
613 for alt_insn_set in input_insns.GetAltInsnSets():
614 # Figure out which keysets have been requested for this type.
615 # We sort the forced set so tests/runtime behavior is stable.
616 keysets = sorted(force_keysets)
617 if not keysets:
618 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
619 if not keysets:
620 logging.warning(
621 "Skipping %s image signing due to no keysets",
622 image_type,
623 )
Mike Frysinger4495b032014-03-05 17:24:03 -0500624
Alex Klein1699fab2022-09-08 08:46:06 -0600625 for keyset in keysets:
626 sect_insns["keyset"] = keyset
627
628 # Generate the insn file for this artifact that the signer will use,
629 # and flag it for signing.
630 with cros_build_lib.UnbufferedNamedTemporaryFile(
631 prefix="pushimage.insns."
632 ) as insns_path:
633 input_insns.OutputInsns(
634 insns_path.name,
635 sect_insns,
636 sect_general,
637 insns_merge=alt_insn_set,
638 )
639
640 gs_insns_path = "%s/%s" % (dst_path, dst_name)
641 if not first_image:
642 gs_insns_path += "-%s" % keyset
643 first_image = False
644 gs_insns_path += ".instructions"
645
646 try:
647 ctx.Copy(insns_path.name, gs_insns_path)
648 except gs.GSContextException:
649 unknown_error[0] = True
650 logging.error(
651 "Unknown error while uploading insns %s",
652 gs_insns_path,
653 exc_info=True,
654 )
655 continue
656
657 try:
658 MarkImageToBeSigned(
659 ctx, tbs_base, gs_insns_path, priority
660 )
661 except gs.GSContextException:
662 unknown_error[0] = True
663 logging.error(
664 "Unknown error while marking for signing %s",
665 gs_insns_path,
666 exc_info=True,
667 )
668 continue
669 logging.info(
670 "Signing %s image with keyset %s at %s",
671 image_type,
672 keyset,
673 gs_insns_path,
674 )
675 instruction_urls.setdefault(channel, []).append(
676 gs_insns_path
677 )
678
679 if unknown_error[0]:
680 raise PushError("hit some unknown error(s)", instruction_urls)
681
682 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400683
684
Mike Frysinger26144192017-08-30 18:26:46 -0400685def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600686 """Creates the argparse parser."""
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500687 parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400688
Alex Klein1699fab2022-09-08 08:46:06 -0600689 # The type of image_dir will strip off trailing slashes (makes later
690 # processing simpler and the display prettier).
691 parser.add_argument(
692 "image_dir",
693 default=None,
694 type="local_or_gs_path",
695 help="full path of source artifacts to upload",
696 )
697 parser.add_argument(
698 "--board",
699 default=None,
700 required=True,
701 help="board to generate symbols for",
702 )
703 parser.add_argument(
704 "--profile", default=None, help='board profile in use (e.g. "asan")'
705 )
706 parser.add_argument(
707 "--version",
708 default=None,
709 help="version info (normally extracted from image_dir)",
710 )
711 parser.add_argument(
712 "--channels",
713 default=None,
714 action="split_extend",
715 help="override list of channels to process",
716 )
717 parser.add_argument(
Alex Klein1699fab2022-09-08 08:46:06 -0600718 "-M",
719 "--mock",
720 default=False,
721 action="store_true",
722 help="upload things to a testing bucket (dev testing)",
723 )
724 parser.add_argument(
725 "--test-sign",
726 default=[],
727 action="append",
728 choices=TEST_KEYSETS,
729 help="mung signing behavior to sign w/ test keys",
730 )
731 parser.add_argument(
732 "--priority",
733 type=int,
734 default=50,
735 help="set signing priority (lower == higher prio)",
736 )
737 parser.add_argument(
738 "--sign-types",
739 default=None,
740 nargs="+",
741 choices=_SUPPORTED_IMAGE_TYPES,
742 help="only sign specified image types",
743 )
744 parser.add_argument(
745 "--buildroot",
746 default=constants.SOURCE_ROOT,
747 type="path",
748 help="Buildroot to use. Defaults to current.",
749 )
750 parser.add_argument(
751 "--yes",
752 action="store_true",
753 default=False,
754 help="answer yes to all prompts",
755 )
756 parser.add_argument(
757 "--dest-bucket",
758 default=constants.RELEASE_BUCKET,
759 help="dest bucket. Default to %(default)s",
760 )
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400761
Alex Klein1699fab2022-09-08 08:46:06 -0600762 return parser
Mike Frysinger26144192017-08-30 18:26:46 -0400763
764
765def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600766 parser = GetParser()
767 opts = parser.parse_args(argv)
768 opts.Freeze()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400769
Alex Klein041edd82023-04-17 12:23:23 -0600770 force_keysets = {f"{TEST_KEYSET_PREFIX}-{x}" for x in opts.test_sign}
Mike Frysingerdad40d62014-02-09 02:18:02 -0500771
Alex Klein1699fab2022-09-08 08:46:06 -0600772 # If we aren't using mock or test or dry run mode, then let's prompt the user
773 # to make sure they actually want to do this. It's rare that people want to
774 # run this directly and hit the release bucket.
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500775 if not (opts.mock or force_keysets or opts.dryrun) and not opts.yes:
Alex Klein1699fab2022-09-08 08:46:06 -0600776 prolog = "\n".join(
777 textwrap.wrap(
778 textwrap.dedent(
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000779 "Uploading images for signing to the *release* bucket is"
780 " not something you generally should be doing yourself."
Alex Klein1699fab2022-09-08 08:46:06 -0600781 ),
782 80,
783 )
784 ).strip()
785 if not cros_build_lib.BooleanPrompt(
786 prompt="Are you sure you want to sign these images",
787 default=False,
788 prolog=prolog,
789 ):
790 cros_build_lib.Die("better safe than sorry")
Mike Frysinger09fe0122014-02-09 02:44:05 -0500791
Alex Klein1699fab2022-09-08 08:46:06 -0600792 PushImage(
793 opts.image_dir,
794 opts.board,
795 versionrev=opts.version,
796 profile=opts.profile,
797 priority=opts.priority,
798 sign_types=opts.sign_types,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500799 dryrun=opts.dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600800 mock=opts.mock,
801 force_keysets=force_keysets,
802 force_channels=opts.channels,
803 buildroot=opts.buildroot,
804 dest_bucket=opts.dest_bucket,
805 )