blob: 5bd0288517769a9db7f2300b7edff2ba1a3280f0 [file] [log] [blame]
Mike Frysingerf1ba7ad2022-09-12 05:42:57 -04001# Copyright 2013 The ChromiumOS Authors
Mike Frysingerd13faeb2013-09-05 16:00:46 -04002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysingere852b072021-05-21 12:39:03 -040011import configparser
Mike Frysingerd13faeb2013-09-05 16:00:46 -040012import getpass
Mike Frysingere852b072021-05-21 12:39:03 -040013import io
Chris McDonald59650c32021-07-20 15:29:28 -060014import logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import os
16import re
Mike Frysinger09fe0122014-02-09 02:44:05 -050017import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040018
Mike Frysingerd13faeb2013-09-05 16:00:46 -040019from chromite.lib import commandline
Chris McDonald59650c32021-07-20 15:29:28 -060020from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import gs
23from chromite.lib import osutils
24from chromite.lib import signing
25
26
27# This will split a fully qualified ChromeOS version string up.
28# R34-5126.0.0 will break into "34" and "5126.0.0".
Alex Klein1699fab2022-09-08 08:46:06 -060029VERSION_REGEX = r"^R([0-9]+)-([^-]+)"
Mike Frysingerd13faeb2013-09-05 16:00:46 -040030
Mike Frysingerdad40d62014-02-09 02:18:02 -050031# The test signers will scan this dir looking for test work.
32# Keep it in sync with the signer config files [gs_test_buckets].
Alex Klein1699fab2022-09-08 08:46:06 -060033TEST_SIGN_BUCKET_BASE = "gs://chromeos-throw-away-bucket/signer-tests"
Mike Frysingerdad40d62014-02-09 02:18:02 -050034
David Rileyf8205122015-09-04 13:46:36 -070035# Keysets that are only valid in the above test bucket.
Alex Klein1699fab2022-09-08 08:46:06 -060036TEST_KEYSET_PREFIX = "test-keys"
Alex Klein041edd82023-04-17 12:23:23 -060037TEST_KEYSETS = {"mp", "premp", "nvidia-premp"}
Mike Frysingerdad40d62014-02-09 02:18:02 -050038
Amey Deshpandea936c622015-08-12 17:27:54 -070039# Supported image types for signing.
40_SUPPORTED_IMAGE_TYPES = (
41 constants.IMAGE_TYPE_RECOVERY,
42 constants.IMAGE_TYPE_FACTORY,
43 constants.IMAGE_TYPE_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070044 constants.IMAGE_TYPE_ACCESSORY_USBPD,
Evan Benn4d061102022-02-14 12:50:45 +110045 constants.IMAGE_TYPE_HPS_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070046 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070047 constants.IMAGE_TYPE_BASE,
Vadim Bendeburyfe37f282020-11-04 19:05:49 -080048 constants.IMAGE_TYPE_GSC_FIRMWARE,
Amey Deshpandea936c622015-08-12 17:27:54 -070049)
50
Mike Frysingerd13faeb2013-09-05 16:00:46 -040051
Mike Frysinger4495b032014-03-05 17:24:03 -050052class PushError(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060053 """When an (unknown) error happened while trying to push artifacts."""
Mike Frysinger4495b032014-03-05 17:24:03 -050054
55
Mike Frysingerd13faeb2013-09-05 16:00:46 -040056class MissingBoardInstructions(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060057 """Raised when a board lacks any signer instructions."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -040058
Alex Klein1699fab2022-09-08 08:46:06 -060059 def __init__(self, board, image_type, input_insns):
60 Exception.__init__(
61 self,
62 "Board %s lacks insns for %s image: %s not found"
63 % (board, image_type, input_insns),
64 )
Mike Frysingerd84d91e2015-11-05 18:02:24 -050065
Mike Frysingerd13faeb2013-09-05 16:00:46 -040066
67class InputInsns(object):
Alex Klein1699fab2022-09-08 08:46:06 -060068 """Object to hold settings for a signable board.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050069
Mike Frysingerd13faeb2013-09-05 16:00:46 -040070 Note: The format of the instruction file pushimage outputs (and the signer
71 reads) is not exactly the same as the instruction file pushimage reads.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040072 """
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050073
Alex Klein1699fab2022-09-08 08:46:06 -060074 def __init__(self, board, image_type=None, buildroot=None):
75 """Initialization.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040076
Alex Klein1699fab2022-09-08 08:46:06 -060077 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +100078 board: The board to look up details.
79 image_type: The type of image we will be signing (see --sign-types).
80 buildroot: Buildroot in which to look for signing instructions.
Alex Klein1699fab2022-09-08 08:46:06 -060081 """
82 self.board = board
83 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040084
Alex Klein1699fab2022-09-08 08:46:06 -060085 config = configparser.ConfigParser()
Mike Frysinger31fdddd2023-02-24 15:50:55 -050086 with open(self.GetInsnFile("DEFAULT"), encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060087 config.read_file(fp)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050088
Alex Klein1699fab2022-09-08 08:46:06 -060089 # What pushimage internally refers to as 'recovery', are the basic signing
90 # instructions in practice, and other types are stacked on top.
91 if image_type is None:
92 image_type = constants.IMAGE_TYPE_RECOVERY
93 self.image_type = image_type
94 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
95 if not os.path.exists(input_insns):
96 # This board doesn't have any signing instructions.
97 raise MissingBoardInstructions(self.board, image_type, input_insns)
Mike Frysinger31fdddd2023-02-24 15:50:55 -050098 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -060099 config.read_file(fp)
100
101 if image_type is not None:
102 input_insns = self.GetInsnFile(image_type)
103 if not os.path.exists(input_insns):
104 # This type doesn't have any signing instructions.
105 raise MissingBoardInstructions(
106 self.board, image_type, input_insns
107 )
108
109 self.image_type = image_type
Mike Frysinger31fdddd2023-02-24 15:50:55 -0500110 with open(input_insns, encoding="utf-8") as fp:
Alex Klein1699fab2022-09-08 08:46:06 -0600111 config.read_file(fp)
112
113 self.cfg = config
114
115 def GetInsnFile(self, image_type):
116 """Find the signer instruction files for this board/image type.
117
118 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000119 image_type: The type of instructions to load. It can be a common
120 file (like "DEFAULT"), or one of the --sign-types.
Alex Klein1699fab2022-09-08 08:46:06 -0600121
122 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000123 Full path to the instruction file using |image_type| and
124 |self.board|.
Alex Klein1699fab2022-09-08 08:46:06 -0600125 """
126 if image_type == image_type.upper():
127 name = image_type
128 elif image_type in (
129 constants.IMAGE_TYPE_RECOVERY,
130 constants.IMAGE_TYPE_BASE,
131 ):
132 name = self.board
133 else:
134 name = "%s.%s" % (self.board, image_type)
135
136 return os.path.join(
137 self.buildroot, signing.INPUT_INSN_DIR_REL, "%s.instructions" % name
138 )
139
140 @staticmethod
141 def SplitCfgField(val):
142 """Split a string into multiple elements.
143
144 This centralizes our convention for multiple elements in the input files
145 being delimited by either a space or comma.
146
147 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000148 val: The string to split.
Alex Klein1699fab2022-09-08 08:46:06 -0600149
150 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000151 The list of elements from having done split the string.
Alex Klein1699fab2022-09-08 08:46:06 -0600152 """
153 return val.replace(",", " ").split()
154
155 def GetChannels(self):
156 """Return the list of channels to sign for this board.
157
158 If the board-specific config doesn't specify a preference, we'll use the
159 common settings.
160 """
161 return self.SplitCfgField(self.cfg.get("insns", "channel"))
162
163 def GetKeysets(self, insns_merge=None):
164 """Return the list of keysets to sign for this board.
165
166 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000167 insns_merge: The additional section to look at over [insns].
Alex Klein1699fab2022-09-08 08:46:06 -0600168 """
169 # First load the default value from [insns.keyset] if available.
170 sections = ["insns"]
171 # Then overlay the [insns.xxx.keyset] if requested.
172 if insns_merge is not None:
173 sections += [insns_merge]
174
175 keyset = ""
176 for section in sections:
177 try:
178 keyset = self.cfg.get(section, "keyset")
179 except (configparser.NoSectionError, configparser.NoOptionError):
180 pass
181
182 # We do not perturb the order (e.g. using sorted() or making a set())
183 # because we want the behavior stable, and we want the input insns to
184 # explicitly control the order (since it has an impact on naming).
185 return self.SplitCfgField(keyset)
186
187 def GetAltInsnSets(self):
188 """Return the list of alternative insn sections."""
189 # We do not perturb the order (e.g. using sorted() or making a set())
190 # because we want the behavior stable, and we want the input insns to
191 # explicitly control the order (since it has an impact on naming).
192 ret = [x for x in self.cfg.sections() if x.startswith("insns.")]
193 return ret if ret else [None]
194
195 @staticmethod
196 def CopyConfigParser(config):
197 """Return a copy of a ConfigParser object.
198
199 The python folks broke the ability to use something like deepcopy:
200 https://bugs.python.org/issue16058
201 """
202 # Write the current config to a string io object.
203 data = io.StringIO()
204 config.write(data)
205 data.seek(0)
206
207 # Create a new ConfigParser from the serialized data.
208 ret = configparser.ConfigParser()
209 ret.read_file(data)
210
211 return ret
212
213 def OutputInsns(
214 self, output_file, sect_insns, sect_general, insns_merge=None
215 ):
216 """Generate the output instruction file for sending to the signer.
217
218 The override order is (later has precedence):
219 [insns]
220 [insns_merge] (should be named "insns.xxx")
221 sect_insns
222
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000223 Note: The format of the instruction file pushimage outputs (and the
224 signer reads) is not exactly the same as the instruction file pushimage
225 reads.
Alex Klein1699fab2022-09-08 08:46:06 -0600226
227 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000228 output_file: The file to write the new instruction file to.
229 sect_insns: Items to set/override in the [insns] section.
230 sect_general: Items to set/override in the [general] section.
231 insns_merge: The alternative insns.xxx section to merge.
Alex Klein1699fab2022-09-08 08:46:06 -0600232 """
233 # Create a copy so we can clobber certain fields.
234 config = self.CopyConfigParser(self.cfg)
235 sect_insns = sect_insns.copy()
236
237 # Merge in the alternative insns section if need be.
238 if insns_merge is not None:
239 for k, v in config.items(insns_merge):
240 sect_insns.setdefault(k, v)
241
242 # Clear channel entry in instructions file, ensuring we only get
243 # one channel for the signer to look at. Then provide all the
244 # other details for this signing request to avoid any ambiguity
245 # and to avoid relying on encoding data into filenames.
246 for sect, fields in zip(
247 ("insns", "general"), (sect_insns, sect_general)
248 ):
249 if not config.has_section(sect):
250 config.add_section(sect)
251 for k, v in fields.items():
252 config.set(sect, k, v)
253
254 # Now prune the alternative sections.
255 for alt in self.GetAltInsnSets():
256 config.remove_section(alt)
257
258 output = io.StringIO()
259 config.write(output)
260 data = output.getvalue()
261 osutils.WriteFile(output_file, data)
262 logging.debug("generated insns file for %s:\n%s", self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400263
264
265def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
Alex Klein1699fab2022-09-08 08:46:06 -0600266 """Mark an instructions file for signing.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400267
Alex Klein1699fab2022-09-08 08:46:06 -0600268 This will upload a file to the GS bucket flagging an image for signing by
269 the signers.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400270
Alex Klein1699fab2022-09-08 08:46:06 -0600271 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000272 ctx: A viable gs.GSContext.
273 tbs_base: The full path to where the tobesigned directory lives.
274 insns_path: The path (relative to |tbs_base|) of the file to sign.
275 priority: Set the signing priority (lower == higher prio).
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400276
Alex Klein1699fab2022-09-08 08:46:06 -0600277 Returns:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000278 The full path to the remote tobesigned file.
Alex Klein1699fab2022-09-08 08:46:06 -0600279 """
280 if priority < 0 or priority > 99:
281 raise ValueError("priority must be [0, 99] inclusive")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400282
Alex Klein1699fab2022-09-08 08:46:06 -0600283 if insns_path.startswith(tbs_base):
284 insns_path = insns_path[len(tbs_base) :].lstrip("/")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400285
Alex Klein1699fab2022-09-08 08:46:06 -0600286 tbs_path = "%s/tobesigned/%02i,%s" % (
287 tbs_base,
288 priority,
289 insns_path.replace("/", ","),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400290 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700291
Alex Klein1699fab2022-09-08 08:46:06 -0600292 # The caller will catch gs.GSContextException for us.
293 ctx.Copy("-", tbs_path, input=cros_build_lib.MachineDetails())
Amey Deshpandea936c622015-08-12 17:27:54 -0700294
Alex Klein1699fab2022-09-08 08:46:06 -0600295 return tbs_path
Amey Deshpandea936c622015-08-12 17:27:54 -0700296
David Rileya04d19d2015-09-04 16:11:50 -0700297
Alex Klein1699fab2022-09-08 08:46:06 -0600298def PushImage(
299 src_path,
300 board,
301 versionrev=None,
302 profile=None,
303 priority=50,
304 sign_types=None,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500305 dryrun=False,
Alex Klein1699fab2022-09-08 08:46:06 -0600306 mock=False,
307 force_keysets=(),
308 force_channels=None,
309 buildroot=constants.SOURCE_ROOT,
310 dest_bucket=constants.RELEASE_BUCKET,
311):
312 """Push the image from the archive bucket to the release bucket.
Evan Benn4d061102022-02-14 12:50:45 +1100313
Alex Klein1699fab2022-09-08 08:46:06 -0600314 Args:
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000315 src_path: Where to copy the files from; can be a local path or gs://
316 URL. Should be a full path to the artifacts in either case.
317 board: The board we're uploading artifacts for (e.g. $BOARD).
318 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
319 profile: The board profile in use (e.g. "asan").
320 priority: Set the signing priority (lower == higher prio).
321 sign_types: If set, a set of types which we'll restrict ourselves to
322 signing. See the --sign-types option for more details.
323 dryrun: Show what would be done, but do not upload anything.
324 mock: Upload to a testing bucket rather than the real one.
325 force_keysets: Set of keysets to use rather than what the inputs say.
326 force_channels: Set of channels to use rather than what the inputs say.
327 buildroot: Buildroot in which to look for signing instructions.
328 dest_bucket: Bucket to push results to.
Vincent Palatind599c662015-10-26 09:51:41 -0700329
Alex Klein1699fab2022-09-08 08:46:06 -0600330 Returns:
331 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
332 'gs://signer_instruction_uri2',
333 ...]
334 """
335 # Whether we hit an unknown error. If so, we'll throw an error, but only
336 # at the end (so that we still upload as many files as possible).
337 # It's implemented using a list to deal with variable scopes in nested
338 # functions below.
339 unknown_error = [False]
LaMont Jones7d6c98f2019-09-27 12:37:33 -0600340
Alex Klein1699fab2022-09-08 08:46:06 -0600341 if versionrev is None:
342 # Extract milestone/version from the directory name.
343 versionrev = os.path.basename(src_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700344
Alex Klein1699fab2022-09-08 08:46:06 -0600345 # We only support the latest format here. Older releases can use pushimage
346 # from the respective branch which deals with legacy cruft.
347 m = re.match(VERSION_REGEX, versionrev)
348 if not m:
349 raise ValueError(
350 "version %s does not match %s" % (versionrev, VERSION_REGEX)
351 )
352 milestone = m.group(1)
353 version = m.group(2)
Amey Deshpandea936c622015-08-12 17:27:54 -0700354
Alex Klein1699fab2022-09-08 08:46:06 -0600355 # Normalize board to always use dashes not underscores. This is mostly a
356 # historical artifact at this point, but we can't really break it since the
357 # value is used in URLs.
358 boardpath = board.replace("_", "-")
359 if profile is not None:
360 boardpath += "-%s" % profile.replace("_", "-")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400361
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500362 ctx = gs.GSContext(dry_run=dryrun)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400363
Alex Klein1699fab2022-09-08 08:46:06 -0600364 try:
365 input_insns = InputInsns(board, buildroot=buildroot)
366 except MissingBoardInstructions as e:
367 logging.warning("Missing base instruction file: %s", e)
368 logging.warning("not uploading anything for signing")
369 return
370
371 if force_channels is None:
372 channels = input_insns.GetChannels()
373 else:
374 # Filter out duplicates.
375 channels = sorted(set(force_channels))
376
377 # We want force_keysets as a set.
378 force_keysets = set(force_keysets)
379
380 if mock:
381 logging.info("Upload mode: mock; signers will not process anything")
382 tbs_base = gs_base = os.path.join(
383 constants.TRASH_BUCKET, "pushimage-tests", getpass.getuser()
384 )
Alex Klein041edd82023-04-17 12:23:23 -0600385 elif {f"{TEST_KEYSET_PREFIX}-{x}" for x in TEST_KEYSETS} & force_keysets:
Alex Klein1699fab2022-09-08 08:46:06 -0600386 logging.info("Upload mode: test; signers will process test keys")
387 # We need the tbs_base to be in the place the signer will actually scan.
388 tbs_base = TEST_SIGN_BUCKET_BASE
389 gs_base = os.path.join(tbs_base, getpass.getuser())
390 else:
391 logging.info("Upload mode: normal; signers will process the images")
392 tbs_base = gs_base = dest_bucket
393
394 sect_general = {
395 "config_board": board,
396 "board": boardpath,
397 "version": version,
398 "versionrev": versionrev,
399 "milestone": milestone,
400 }
401 sect_insns = {}
402
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500403 if dryrun:
Alex Klein1699fab2022-09-08 08:46:06 -0600404 logging.info("DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED")
405 logging.info("Signing for channels: %s", " ".join(channels))
406
407 instruction_urls = {}
408
409 def _ImageNameBase(image_type=None):
410 lmid = ("%s-" % image_type) if image_type else ""
411 return "ChromeOS-%s%s-%s" % (lmid, versionrev, boardpath)
412
413 # These variables are defined outside the loop so that the nested functions
414 # below can access them without 'cell-var-from-loop' linter warning.
415 dst_path = ""
Amey Deshpandea936c622015-08-12 17:27:54 -0700416 files_to_sign = []
Alex Klein1699fab2022-09-08 08:46:06 -0600417 for channel in channels:
418 logging.debug("\n\n#### CHANNEL: %s ####\n", channel)
419 sect_insns["channel"] = channel
420 sub_path = "%s-channel/%s/%s" % (channel, boardpath, version)
421 dst_path = "%s/%s" % (gs_base, sub_path)
422 logging.info("Copying images to %s", dst_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700423
Alex Klein1699fab2022-09-08 08:46:06 -0600424 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
425 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
426 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
427 hps_firmware_basename = _ImageNameBase(
428 constants.IMAGE_TYPE_HPS_FIRMWARE
429 )
430 acc_usbpd_basename = _ImageNameBase(
431 constants.IMAGE_TYPE_ACCESSORY_USBPD
432 )
433 acc_rwsig_basename = _ImageNameBase(
434 constants.IMAGE_TYPE_ACCESSORY_RWSIG
435 )
436 gsc_firmware_basename = _ImageNameBase(
437 constants.IMAGE_TYPE_GSC_FIRMWARE
438 )
439 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
440 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
441 hwqual_tarball = "chromeos-hwqual-%s-%s.tar.bz2" % (board, versionrev)
Amey Deshpandea936c622015-08-12 17:27:54 -0700442
Alex Klein1699fab2022-09-08 08:46:06 -0600443 # The following build artifacts, if present, are always copied regardless of
444 # requested signing types.
445 files_to_copy_only = (
446 # (<src>, <dst>, <suffix>),
447 ("image.zip", _ImageNameBase(), "zip"),
448 (constants.TEST_IMAGE_TAR, test_basename, "tar.xz"),
449 ("debug.tgz", "debug-%s" % boardpath, "tgz"),
450 (hwqual_tarball, None, None),
451 ("stateful.tgz", None, None),
452 ("dlc", None, None),
453 (constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
454 (constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
455 (constants.QUICK_PROVISION_PAYLOAD_MINIOS, None, None),
456 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700457
Alex Klein1699fab2022-09-08 08:46:06 -0600458 # The following build artifacts, if present, are always copied.
459 # If |sign_types| is None, all of them are marked for signing, otherwise
460 # only the image types specified in |sign_types| are marked for signing.
461 files_to_copy_and_maybe_sign = (
462 # (<src>, <dst>, <suffix>, <signing type>),
463 (
464 constants.RECOVERY_IMAGE_TAR,
465 recovery_basename,
466 "tar.xz",
467 constants.IMAGE_TYPE_RECOVERY,
468 ),
469 (
470 "factory_image.zip",
471 factory_basename,
472 "zip",
473 constants.IMAGE_TYPE_FACTORY,
474 ),
475 (
476 "firmware_from_source.tar.bz2",
477 firmware_basename,
478 "tar.bz2",
479 constants.IMAGE_TYPE_FIRMWARE,
480 ),
Shuhei Takahashi7e05dbf2023-05-19 10:40:47 +0900481 )
482
483 # The following build artifacts are copied and marked for signing, if
484 # they are present *and* if the image type is specified via
485 # |sign_types|.
486 files_to_maybe_copy_and_sign = (
487 # (<src>, <dst>, <suffix>, <signing type>),
488 (
489 constants.BASE_IMAGE_TAR,
490 base_basename,
491 "tar.xz",
492 constants.IMAGE_TYPE_BASE,
493 ),
Alex Klein1699fab2022-09-08 08:46:06 -0600494 (
495 "firmware_from_source.tar.bz2",
496 hps_firmware_basename,
497 "tar.bz2",
498 constants.IMAGE_TYPE_HPS_FIRMWARE,
499 ),
500 (
501 "firmware_from_source.tar.bz2",
502 acc_usbpd_basename,
503 "tar.bz2",
504 constants.IMAGE_TYPE_ACCESSORY_USBPD,
505 ),
506 (
507 "firmware_from_source.tar.bz2",
508 acc_rwsig_basename,
509 "tar.bz2",
510 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
511 ),
512 (
513 "firmware_from_source.tar.bz2",
514 gsc_firmware_basename,
515 "tar.bz2",
516 constants.IMAGE_TYPE_GSC_FIRMWARE,
517 ),
518 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700519
Alex Klein1699fab2022-09-08 08:46:06 -0600520 def _CopyFileToGS(src, dst=None, suffix=None):
521 """Returns |dst| file name if the copying was successful."""
522 if dst is None:
523 dst = src
524 elif suffix is not None:
525 dst = "%s.%s" % (dst, suffix)
526 success = False
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500527 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600528 ctx.Copy(
529 os.path.join(src_path, src),
530 os.path.join(dst_path, dst),
531 recursive=True,
532 )
533 success = True
534 except gs.GSNoSuchKey:
535 logging.warning("Skipping %s as it does not exist", src)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500536 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600537 unknown_error[0] = True
538 logging.error(
539 "Skipping %s due to unknown GS error", src, exc_info=True
540 )
541 return dst if success else None
Mike Frysinger4495b032014-03-05 17:24:03 -0500542
Alex Klein1699fab2022-09-08 08:46:06 -0600543 for src, dst, suffix in files_to_copy_only:
544 _CopyFileToGS(src, dst, suffix)
545
546 # Clear the list of files to sign before adding new artifacts.
547 files_to_sign = []
548
549 def _AddToFilesToSign(image_type, dst, suffix):
550 assert dst.endswith("." + suffix), "dst: %s, suffix: %s" % (
551 dst,
552 suffix,
553 )
554 dst_base = dst[: -(len(suffix) + 1)]
555 files_to_sign.append([image_type, dst_base, suffix])
556
557 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
558 dst = _CopyFileToGS(src, dst, suffix)
559 if dst and (not sign_types or image_type in sign_types):
560 _AddToFilesToSign(image_type, dst, suffix)
561
562 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
563 if sign_types and image_type in sign_types:
564 dst = _CopyFileToGS(src, dst, suffix)
565 if dst:
566 _AddToFilesToSign(image_type, dst, suffix)
567
568 logging.debug("Files to sign: %s", files_to_sign)
569 unused_sign_types = set(sign_types or []) - set(
570 x for x, _, _ in files_to_sign
571 )
572 if unused_sign_types:
573 logging.warning(
574 "Some sign types were unused: %s", " ".join(unused_sign_types)
575 )
576
577 # Now go through the subset for signing.
578 for image_type, dst_name, suffix in files_to_sign:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500579 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600580 input_insns = InputInsns(
581 board, image_type=image_type, buildroot=buildroot
582 )
583 except MissingBoardInstructions as e:
584 logging.info("Nothing to sign: %s", e)
585 continue
586
587 dst_archive = "%s.%s" % (dst_name, suffix)
588 sect_general["archive"] = dst_archive
589 sect_general["type"] = image_type
590
591 # In the default/automatic mode, only flag files for signing if the
592 # archives were actually uploaded in a previous stage. This additional
593 # check can be removed in future once |sign_types| becomes a required
594 # argument.
595 # TODO: Make |sign_types| a required argument.
596 gs_artifact_path = os.path.join(dst_path, dst_archive)
597 exists = False
598 try:
599 exists = ctx.Exists(gs_artifact_path)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500600 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600601 unknown_error[0] = True
602 logging.error(
603 "Unknown error while checking %s",
604 gs_artifact_path,
605 exc_info=True,
606 )
607 if not exists:
608 logging.info(
609 "%s does not exist. Nothing to sign.", gs_artifact_path
610 )
611 continue
Don Garrett9459c2f2014-01-22 18:20:24 -0800612
Alex Klein1699fab2022-09-08 08:46:06 -0600613 first_image = True
614 for alt_insn_set in input_insns.GetAltInsnSets():
615 # Figure out which keysets have been requested for this type.
616 # We sort the forced set so tests/runtime behavior is stable.
617 keysets = sorted(force_keysets)
618 if not keysets:
619 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
620 if not keysets:
621 logging.warning(
622 "Skipping %s image signing due to no keysets",
623 image_type,
624 )
Mike Frysinger4495b032014-03-05 17:24:03 -0500625
Alex Klein1699fab2022-09-08 08:46:06 -0600626 for keyset in keysets:
627 sect_insns["keyset"] = keyset
628
629 # Generate the insn file for this artifact that the signer will use,
630 # and flag it for signing.
631 with cros_build_lib.UnbufferedNamedTemporaryFile(
632 prefix="pushimage.insns."
633 ) as insns_path:
634 input_insns.OutputInsns(
635 insns_path.name,
636 sect_insns,
637 sect_general,
638 insns_merge=alt_insn_set,
639 )
640
641 gs_insns_path = "%s/%s" % (dst_path, dst_name)
642 if not first_image:
643 gs_insns_path += "-%s" % keyset
644 first_image = False
645 gs_insns_path += ".instructions"
646
647 try:
648 ctx.Copy(insns_path.name, gs_insns_path)
649 except gs.GSContextException:
650 unknown_error[0] = True
651 logging.error(
652 "Unknown error while uploading insns %s",
653 gs_insns_path,
654 exc_info=True,
655 )
656 continue
657
658 try:
659 MarkImageToBeSigned(
660 ctx, tbs_base, gs_insns_path, priority
661 )
662 except gs.GSContextException:
663 unknown_error[0] = True
664 logging.error(
665 "Unknown error while marking for signing %s",
666 gs_insns_path,
667 exc_info=True,
668 )
669 continue
670 logging.info(
671 "Signing %s image with keyset %s at %s",
672 image_type,
673 keyset,
674 gs_insns_path,
675 )
676 instruction_urls.setdefault(channel, []).append(
677 gs_insns_path
678 )
679
680 if unknown_error[0]:
681 raise PushError("hit some unknown error(s)", instruction_urls)
682
683 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400684
685
Mike Frysinger26144192017-08-30 18:26:46 -0400686def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600687 """Creates the argparse parser."""
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500688 parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400689
Alex Klein1699fab2022-09-08 08:46:06 -0600690 # The type of image_dir will strip off trailing slashes (makes later
691 # processing simpler and the display prettier).
692 parser.add_argument(
693 "image_dir",
694 default=None,
695 type="local_or_gs_path",
696 help="full path of source artifacts to upload",
697 )
698 parser.add_argument(
699 "--board",
700 default=None,
701 required=True,
702 help="board to generate symbols for",
703 )
704 parser.add_argument(
705 "--profile", default=None, help='board profile in use (e.g. "asan")'
706 )
707 parser.add_argument(
708 "--version",
709 default=None,
710 help="version info (normally extracted from image_dir)",
711 )
712 parser.add_argument(
713 "--channels",
714 default=None,
715 action="split_extend",
716 help="override list of channels to process",
717 )
718 parser.add_argument(
Alex Klein1699fab2022-09-08 08:46:06 -0600719 "-M",
720 "--mock",
721 default=False,
722 action="store_true",
723 help="upload things to a testing bucket (dev testing)",
724 )
725 parser.add_argument(
726 "--test-sign",
727 default=[],
728 action="append",
729 choices=TEST_KEYSETS,
730 help="mung signing behavior to sign w/ test keys",
731 )
732 parser.add_argument(
733 "--priority",
734 type=int,
735 default=50,
736 help="set signing priority (lower == higher prio)",
737 )
738 parser.add_argument(
739 "--sign-types",
740 default=None,
741 nargs="+",
742 choices=_SUPPORTED_IMAGE_TYPES,
743 help="only sign specified image types",
744 )
745 parser.add_argument(
746 "--buildroot",
747 default=constants.SOURCE_ROOT,
748 type="path",
749 help="Buildroot to use. Defaults to current.",
750 )
751 parser.add_argument(
752 "--yes",
753 action="store_true",
754 default=False,
755 help="answer yes to all prompts",
756 )
757 parser.add_argument(
758 "--dest-bucket",
759 default=constants.RELEASE_BUCKET,
760 help="dest bucket. Default to %(default)s",
761 )
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400762
Alex Klein1699fab2022-09-08 08:46:06 -0600763 return parser
Mike Frysinger26144192017-08-30 18:26:46 -0400764
765
766def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600767 parser = GetParser()
768 opts = parser.parse_args(argv)
769 opts.Freeze()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400770
Alex Klein041edd82023-04-17 12:23:23 -0600771 force_keysets = {f"{TEST_KEYSET_PREFIX}-{x}" for x in opts.test_sign}
Mike Frysingerdad40d62014-02-09 02:18:02 -0500772
Alex Klein1699fab2022-09-08 08:46:06 -0600773 # If we aren't using mock or test or dry run mode, then let's prompt the user
774 # to make sure they actually want to do this. It's rare that people want to
775 # run this directly and hit the release bucket.
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500776 if not (opts.mock or force_keysets or opts.dryrun) and not opts.yes:
Alex Klein1699fab2022-09-08 08:46:06 -0600777 prolog = "\n".join(
778 textwrap.wrap(
779 textwrap.dedent(
Trent Aptedc20bb6d2023-05-10 15:00:03 +1000780 "Uploading images for signing to the *release* bucket is"
781 " not something you generally should be doing yourself."
Alex Klein1699fab2022-09-08 08:46:06 -0600782 ),
783 80,
784 )
785 ).strip()
786 if not cros_build_lib.BooleanPrompt(
787 prompt="Are you sure you want to sign these images",
788 default=False,
789 prolog=prolog,
790 ):
791 cros_build_lib.Die("better safe than sorry")
Mike Frysinger09fe0122014-02-09 02:44:05 -0500792
Alex Klein1699fab2022-09-08 08:46:06 -0600793 PushImage(
794 opts.image_dir,
795 opts.board,
796 versionrev=opts.version,
797 profile=opts.profile,
798 priority=opts.priority,
799 sign_types=opts.sign_types,
Mike Frysinger9d1253c2023-02-02 09:00:07 -0500800 dryrun=opts.dryrun,
Alex Klein1699fab2022-09-08 08:46:06 -0600801 mock=opts.mock,
802 force_keysets=force_keysets,
803 force_channels=opts.channels,
804 buildroot=opts.buildroot,
805 dest_bucket=opts.dest_bucket,
806 )