blob: f076f9c12b199517d5c1e7518f10edb924e31822 [file] [log] [blame]
Mike Frysingerd13faeb2013-09-05 16:00:46 -04001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysingere852b072021-05-21 12:39:03 -040011import configparser
Mike Frysingerd13faeb2013-09-05 16:00:46 -040012import getpass
Mike Frysingere852b072021-05-21 12:39:03 -040013import io
Chris McDonald59650c32021-07-20 15:29:28 -060014import logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import os
16import re
Mike Frysinger09fe0122014-02-09 02:44:05 -050017import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040018
Mike Frysingerd13faeb2013-09-05 16:00:46 -040019from chromite.lib import commandline
Chris McDonald59650c32021-07-20 15:29:28 -060020from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import gs
23from chromite.lib import osutils
24from chromite.lib import signing
25
26
27# This will split a fully qualified ChromeOS version string up.
28# R34-5126.0.0 will break into "34" and "5126.0.0".
Alex Klein1699fab2022-09-08 08:46:06 -060029VERSION_REGEX = r"^R([0-9]+)-([^-]+)"
Mike Frysingerd13faeb2013-09-05 16:00:46 -040030
Mike Frysingerdad40d62014-02-09 02:18:02 -050031# The test signers will scan this dir looking for test work.
32# Keep it in sync with the signer config files [gs_test_buckets].
Alex Klein1699fab2022-09-08 08:46:06 -060033TEST_SIGN_BUCKET_BASE = "gs://chromeos-throw-away-bucket/signer-tests"
Mike Frysingerdad40d62014-02-09 02:18:02 -050034
David Rileyf8205122015-09-04 13:46:36 -070035# Keysets that are only valid in the above test bucket.
Alex Klein1699fab2022-09-08 08:46:06 -060036TEST_KEYSET_PREFIX = "test-keys"
37TEST_KEYSETS = set(
38 (
39 "mp",
40 "premp",
41 "nvidia-premp",
42 )
43)
Mike Frysingerdad40d62014-02-09 02:18:02 -050044
Amey Deshpandea936c622015-08-12 17:27:54 -070045# Supported image types for signing.
46_SUPPORTED_IMAGE_TYPES = (
47 constants.IMAGE_TYPE_RECOVERY,
48 constants.IMAGE_TYPE_FACTORY,
49 constants.IMAGE_TYPE_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070050 constants.IMAGE_TYPE_ACCESSORY_USBPD,
Evan Benn4d061102022-02-14 12:50:45 +110051 constants.IMAGE_TYPE_HPS_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070052 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070053 constants.IMAGE_TYPE_BASE,
Vadim Bendeburyfe37f282020-11-04 19:05:49 -080054 constants.IMAGE_TYPE_GSC_FIRMWARE,
Amey Deshpandea936c622015-08-12 17:27:54 -070055)
56
Mike Frysingerd13faeb2013-09-05 16:00:46 -040057
Mike Frysinger4495b032014-03-05 17:24:03 -050058class PushError(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060059 """When an (unknown) error happened while trying to push artifacts."""
Mike Frysinger4495b032014-03-05 17:24:03 -050060
61
Mike Frysingerd13faeb2013-09-05 16:00:46 -040062class MissingBoardInstructions(Exception):
Alex Klein1699fab2022-09-08 08:46:06 -060063 """Raised when a board lacks any signer instructions."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -040064
Alex Klein1699fab2022-09-08 08:46:06 -060065 def __init__(self, board, image_type, input_insns):
66 Exception.__init__(
67 self,
68 "Board %s lacks insns for %s image: %s not found"
69 % (board, image_type, input_insns),
70 )
Mike Frysingerd84d91e2015-11-05 18:02:24 -050071
Mike Frysingerd13faeb2013-09-05 16:00:46 -040072
73class InputInsns(object):
Alex Klein1699fab2022-09-08 08:46:06 -060074 """Object to hold settings for a signable board.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050075
Mike Frysingerd13faeb2013-09-05 16:00:46 -040076 Note: The format of the instruction file pushimage outputs (and the signer
77 reads) is not exactly the same as the instruction file pushimage reads.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040078 """
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050079
Alex Klein1699fab2022-09-08 08:46:06 -060080 def __init__(self, board, image_type=None, buildroot=None):
81 """Initialization.
Mike Frysingerd13faeb2013-09-05 16:00:46 -040082
Alex Klein1699fab2022-09-08 08:46:06 -060083 Args:
84 board: The board to look up details.
85 image_type: The type of image we will be signing (see --sign-types).
86 buildroot: Buildroot in which to look for signing instructions.
87 """
88 self.board = board
89 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040090
Alex Klein1699fab2022-09-08 08:46:06 -060091 config = configparser.ConfigParser()
92 with open(self.GetInsnFile("DEFAULT")) as fp:
93 config.read_file(fp)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -050094
Alex Klein1699fab2022-09-08 08:46:06 -060095 # What pushimage internally refers to as 'recovery', are the basic signing
96 # instructions in practice, and other types are stacked on top.
97 if image_type is None:
98 image_type = constants.IMAGE_TYPE_RECOVERY
99 self.image_type = image_type
100 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
101 if not os.path.exists(input_insns):
102 # This board doesn't have any signing instructions.
103 raise MissingBoardInstructions(self.board, image_type, input_insns)
104 with open(input_insns) as fp:
105 config.read_file(fp)
106
107 if image_type is not None:
108 input_insns = self.GetInsnFile(image_type)
109 if not os.path.exists(input_insns):
110 # This type doesn't have any signing instructions.
111 raise MissingBoardInstructions(
112 self.board, image_type, input_insns
113 )
114
115 self.image_type = image_type
116 with open(input_insns) as fp:
117 config.read_file(fp)
118
119 self.cfg = config
120
121 def GetInsnFile(self, image_type):
122 """Find the signer instruction files for this board/image type.
123
124 Args:
125 image_type: The type of instructions to load. It can be a common file
126 (like "DEFAULT"), or one of the --sign-types.
127
128 Returns:
129 Full path to the instruction file using |image_type| and |self.board|.
130 """
131 if image_type == image_type.upper():
132 name = image_type
133 elif image_type in (
134 constants.IMAGE_TYPE_RECOVERY,
135 constants.IMAGE_TYPE_BASE,
136 ):
137 name = self.board
138 else:
139 name = "%s.%s" % (self.board, image_type)
140
141 return os.path.join(
142 self.buildroot, signing.INPUT_INSN_DIR_REL, "%s.instructions" % name
143 )
144
145 @staticmethod
146 def SplitCfgField(val):
147 """Split a string into multiple elements.
148
149 This centralizes our convention for multiple elements in the input files
150 being delimited by either a space or comma.
151
152 Args:
153 val: The string to split.
154
155 Returns:
156 The list of elements from having done split the string.
157 """
158 return val.replace(",", " ").split()
159
160 def GetChannels(self):
161 """Return the list of channels to sign for this board.
162
163 If the board-specific config doesn't specify a preference, we'll use the
164 common settings.
165 """
166 return self.SplitCfgField(self.cfg.get("insns", "channel"))
167
168 def GetKeysets(self, insns_merge=None):
169 """Return the list of keysets to sign for this board.
170
171 Args:
172 insns_merge: The additional section to look at over [insns].
173 """
174 # First load the default value from [insns.keyset] if available.
175 sections = ["insns"]
176 # Then overlay the [insns.xxx.keyset] if requested.
177 if insns_merge is not None:
178 sections += [insns_merge]
179
180 keyset = ""
181 for section in sections:
182 try:
183 keyset = self.cfg.get(section, "keyset")
184 except (configparser.NoSectionError, configparser.NoOptionError):
185 pass
186
187 # We do not perturb the order (e.g. using sorted() or making a set())
188 # because we want the behavior stable, and we want the input insns to
189 # explicitly control the order (since it has an impact on naming).
190 return self.SplitCfgField(keyset)
191
192 def GetAltInsnSets(self):
193 """Return the list of alternative insn sections."""
194 # We do not perturb the order (e.g. using sorted() or making a set())
195 # because we want the behavior stable, and we want the input insns to
196 # explicitly control the order (since it has an impact on naming).
197 ret = [x for x in self.cfg.sections() if x.startswith("insns.")]
198 return ret if ret else [None]
199
200 @staticmethod
201 def CopyConfigParser(config):
202 """Return a copy of a ConfigParser object.
203
204 The python folks broke the ability to use something like deepcopy:
205 https://bugs.python.org/issue16058
206 """
207 # Write the current config to a string io object.
208 data = io.StringIO()
209 config.write(data)
210 data.seek(0)
211
212 # Create a new ConfigParser from the serialized data.
213 ret = configparser.ConfigParser()
214 ret.read_file(data)
215
216 return ret
217
218 def OutputInsns(
219 self, output_file, sect_insns, sect_general, insns_merge=None
220 ):
221 """Generate the output instruction file for sending to the signer.
222
223 The override order is (later has precedence):
224 [insns]
225 [insns_merge] (should be named "insns.xxx")
226 sect_insns
227
228 Note: The format of the instruction file pushimage outputs (and the signer
229 reads) is not exactly the same as the instruction file pushimage reads.
230
231 Args:
232 output_file: The file to write the new instruction file to.
233 sect_insns: Items to set/override in the [insns] section.
234 sect_general: Items to set/override in the [general] section.
235 insns_merge: The alternative insns.xxx section to merge.
236 """
237 # Create a copy so we can clobber certain fields.
238 config = self.CopyConfigParser(self.cfg)
239 sect_insns = sect_insns.copy()
240
241 # Merge in the alternative insns section if need be.
242 if insns_merge is not None:
243 for k, v in config.items(insns_merge):
244 sect_insns.setdefault(k, v)
245
246 # Clear channel entry in instructions file, ensuring we only get
247 # one channel for the signer to look at. Then provide all the
248 # other details for this signing request to avoid any ambiguity
249 # and to avoid relying on encoding data into filenames.
250 for sect, fields in zip(
251 ("insns", "general"), (sect_insns, sect_general)
252 ):
253 if not config.has_section(sect):
254 config.add_section(sect)
255 for k, v in fields.items():
256 config.set(sect, k, v)
257
258 # Now prune the alternative sections.
259 for alt in self.GetAltInsnSets():
260 config.remove_section(alt)
261
262 output = io.StringIO()
263 config.write(output)
264 data = output.getvalue()
265 osutils.WriteFile(output_file, data)
266 logging.debug("generated insns file for %s:\n%s", self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400267
268
269def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
Alex Klein1699fab2022-09-08 08:46:06 -0600270 """Mark an instructions file for signing.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400271
Alex Klein1699fab2022-09-08 08:46:06 -0600272 This will upload a file to the GS bucket flagging an image for signing by
273 the signers.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400274
Alex Klein1699fab2022-09-08 08:46:06 -0600275 Args:
276 ctx: A viable gs.GSContext.
277 tbs_base: The full path to where the tobesigned directory lives.
278 insns_path: The path (relative to |tbs_base|) of the file to sign.
279 priority: Set the signing priority (lower == higher prio).
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400280
Alex Klein1699fab2022-09-08 08:46:06 -0600281 Returns:
282 The full path to the remote tobesigned file.
283 """
284 if priority < 0 or priority > 99:
285 raise ValueError("priority must be [0, 99] inclusive")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400286
Alex Klein1699fab2022-09-08 08:46:06 -0600287 if insns_path.startswith(tbs_base):
288 insns_path = insns_path[len(tbs_base) :].lstrip("/")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400289
Alex Klein1699fab2022-09-08 08:46:06 -0600290 tbs_path = "%s/tobesigned/%02i,%s" % (
291 tbs_base,
292 priority,
293 insns_path.replace("/", ","),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400294 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700295
Alex Klein1699fab2022-09-08 08:46:06 -0600296 # The caller will catch gs.GSContextException for us.
297 ctx.Copy("-", tbs_path, input=cros_build_lib.MachineDetails())
Amey Deshpandea936c622015-08-12 17:27:54 -0700298
Alex Klein1699fab2022-09-08 08:46:06 -0600299 return tbs_path
Amey Deshpandea936c622015-08-12 17:27:54 -0700300
David Rileya04d19d2015-09-04 16:11:50 -0700301
Alex Klein1699fab2022-09-08 08:46:06 -0600302def PushImage(
303 src_path,
304 board,
305 versionrev=None,
306 profile=None,
307 priority=50,
308 sign_types=None,
309 dry_run=False,
310 mock=False,
311 force_keysets=(),
312 force_channels=None,
313 buildroot=constants.SOURCE_ROOT,
314 dest_bucket=constants.RELEASE_BUCKET,
315):
316 """Push the image from the archive bucket to the release bucket.
Evan Benn4d061102022-02-14 12:50:45 +1100317
Alex Klein1699fab2022-09-08 08:46:06 -0600318 Args:
319 src_path: Where to copy the files from; can be a local path or gs:// URL.
320 Should be a full path to the artifacts in either case.
321 board: The board we're uploading artifacts for (e.g. $BOARD).
322 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
323 profile: The board profile in use (e.g. "asan").
324 priority: Set the signing priority (lower == higher prio).
325 sign_types: If set, a set of types which we'll restrict ourselves to
326 signing. See the --sign-types option for more details.
327 dry_run: Show what would be done, but do not upload anything.
328 mock: Upload to a testing bucket rather than the real one.
329 force_keysets: Set of keysets to use rather than what the inputs say.
330 force_channels: Set of channels to use rather than what the inputs say.
331 buildroot: Buildroot in which to look for signing instructions.
332 dest_bucket: Bucket to push results to.
Vincent Palatind599c662015-10-26 09:51:41 -0700333
Alex Klein1699fab2022-09-08 08:46:06 -0600334 Returns:
335 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
336 'gs://signer_instruction_uri2',
337 ...]
338 """
339 # Whether we hit an unknown error. If so, we'll throw an error, but only
340 # at the end (so that we still upload as many files as possible).
341 # It's implemented using a list to deal with variable scopes in nested
342 # functions below.
343 unknown_error = [False]
LaMont Jones7d6c98f2019-09-27 12:37:33 -0600344
Alex Klein1699fab2022-09-08 08:46:06 -0600345 if versionrev is None:
346 # Extract milestone/version from the directory name.
347 versionrev = os.path.basename(src_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700348
Alex Klein1699fab2022-09-08 08:46:06 -0600349 # We only support the latest format here. Older releases can use pushimage
350 # from the respective branch which deals with legacy cruft.
351 m = re.match(VERSION_REGEX, versionrev)
352 if not m:
353 raise ValueError(
354 "version %s does not match %s" % (versionrev, VERSION_REGEX)
355 )
356 milestone = m.group(1)
357 version = m.group(2)
Amey Deshpandea936c622015-08-12 17:27:54 -0700358
Alex Klein1699fab2022-09-08 08:46:06 -0600359 # Normalize board to always use dashes not underscores. This is mostly a
360 # historical artifact at this point, but we can't really break it since the
361 # value is used in URLs.
362 boardpath = board.replace("_", "-")
363 if profile is not None:
364 boardpath += "-%s" % profile.replace("_", "-")
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400365
Alex Klein1699fab2022-09-08 08:46:06 -0600366 ctx = gs.GSContext(dry_run=dry_run)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400367
Alex Klein1699fab2022-09-08 08:46:06 -0600368 try:
369 input_insns = InputInsns(board, buildroot=buildroot)
370 except MissingBoardInstructions as e:
371 logging.warning("Missing base instruction file: %s", e)
372 logging.warning("not uploading anything for signing")
373 return
374
375 if force_channels is None:
376 channels = input_insns.GetChannels()
377 else:
378 # Filter out duplicates.
379 channels = sorted(set(force_channels))
380
381 # We want force_keysets as a set.
382 force_keysets = set(force_keysets)
383
384 if mock:
385 logging.info("Upload mode: mock; signers will not process anything")
386 tbs_base = gs_base = os.path.join(
387 constants.TRASH_BUCKET, "pushimage-tests", getpass.getuser()
388 )
389 elif (
390 set(["%s-%s" % (TEST_KEYSET_PREFIX, x) for x in TEST_KEYSETS])
391 & force_keysets
392 ):
393 logging.info("Upload mode: test; signers will process test keys")
394 # We need the tbs_base to be in the place the signer will actually scan.
395 tbs_base = TEST_SIGN_BUCKET_BASE
396 gs_base = os.path.join(tbs_base, getpass.getuser())
397 else:
398 logging.info("Upload mode: normal; signers will process the images")
399 tbs_base = gs_base = dest_bucket
400
401 sect_general = {
402 "config_board": board,
403 "board": boardpath,
404 "version": version,
405 "versionrev": versionrev,
406 "milestone": milestone,
407 }
408 sect_insns = {}
409
410 if dry_run:
411 logging.info("DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED")
412 logging.info("Signing for channels: %s", " ".join(channels))
413
414 instruction_urls = {}
415
416 def _ImageNameBase(image_type=None):
417 lmid = ("%s-" % image_type) if image_type else ""
418 return "ChromeOS-%s%s-%s" % (lmid, versionrev, boardpath)
419
420 # These variables are defined outside the loop so that the nested functions
421 # below can access them without 'cell-var-from-loop' linter warning.
422 dst_path = ""
Amey Deshpandea936c622015-08-12 17:27:54 -0700423 files_to_sign = []
Alex Klein1699fab2022-09-08 08:46:06 -0600424 for channel in channels:
425 logging.debug("\n\n#### CHANNEL: %s ####\n", channel)
426 sect_insns["channel"] = channel
427 sub_path = "%s-channel/%s/%s" % (channel, boardpath, version)
428 dst_path = "%s/%s" % (gs_base, sub_path)
429 logging.info("Copying images to %s", dst_path)
Amey Deshpandea936c622015-08-12 17:27:54 -0700430
Alex Klein1699fab2022-09-08 08:46:06 -0600431 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
432 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
433 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
434 hps_firmware_basename = _ImageNameBase(
435 constants.IMAGE_TYPE_HPS_FIRMWARE
436 )
437 acc_usbpd_basename = _ImageNameBase(
438 constants.IMAGE_TYPE_ACCESSORY_USBPD
439 )
440 acc_rwsig_basename = _ImageNameBase(
441 constants.IMAGE_TYPE_ACCESSORY_RWSIG
442 )
443 gsc_firmware_basename = _ImageNameBase(
444 constants.IMAGE_TYPE_GSC_FIRMWARE
445 )
446 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
447 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
448 hwqual_tarball = "chromeos-hwqual-%s-%s.tar.bz2" % (board, versionrev)
Amey Deshpandea936c622015-08-12 17:27:54 -0700449
Alex Klein1699fab2022-09-08 08:46:06 -0600450 # The following build artifacts, if present, are always copied regardless of
451 # requested signing types.
452 files_to_copy_only = (
453 # (<src>, <dst>, <suffix>),
454 ("image.zip", _ImageNameBase(), "zip"),
455 (constants.TEST_IMAGE_TAR, test_basename, "tar.xz"),
456 ("debug.tgz", "debug-%s" % boardpath, "tgz"),
457 (hwqual_tarball, None, None),
458 ("stateful.tgz", None, None),
459 ("dlc", None, None),
460 (constants.QUICK_PROVISION_PAYLOAD_KERNEL, None, None),
461 (constants.QUICK_PROVISION_PAYLOAD_ROOTFS, None, None),
462 (constants.QUICK_PROVISION_PAYLOAD_MINIOS, None, None),
463 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700464
Alex Klein1699fab2022-09-08 08:46:06 -0600465 # The following build artifacts, if present, are always copied.
466 # If |sign_types| is None, all of them are marked for signing, otherwise
467 # only the image types specified in |sign_types| are marked for signing.
468 files_to_copy_and_maybe_sign = (
469 # (<src>, <dst>, <suffix>, <signing type>),
470 (
471 constants.RECOVERY_IMAGE_TAR,
472 recovery_basename,
473 "tar.xz",
474 constants.IMAGE_TYPE_RECOVERY,
475 ),
476 (
477 "factory_image.zip",
478 factory_basename,
479 "zip",
480 constants.IMAGE_TYPE_FACTORY,
481 ),
482 (
483 "firmware_from_source.tar.bz2",
484 firmware_basename,
485 "tar.bz2",
486 constants.IMAGE_TYPE_FIRMWARE,
487 ),
488 (
489 "firmware_from_source.tar.bz2",
490 hps_firmware_basename,
491 "tar.bz2",
492 constants.IMAGE_TYPE_HPS_FIRMWARE,
493 ),
494 (
495 "firmware_from_source.tar.bz2",
496 acc_usbpd_basename,
497 "tar.bz2",
498 constants.IMAGE_TYPE_ACCESSORY_USBPD,
499 ),
500 (
501 "firmware_from_source.tar.bz2",
502 acc_rwsig_basename,
503 "tar.bz2",
504 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
505 ),
506 (
507 "firmware_from_source.tar.bz2",
508 gsc_firmware_basename,
509 "tar.bz2",
510 constants.IMAGE_TYPE_GSC_FIRMWARE,
511 ),
512 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700513
Alex Klein1699fab2022-09-08 08:46:06 -0600514 # The following build artifacts are copied and marked for signing, if
515 # they are present *and* if the image type is specified via |sign_types|.
516 files_to_maybe_copy_and_sign = (
517 # (<src>, <dst>, <suffix>, <signing type>),
518 (
519 constants.BASE_IMAGE_TAR,
520 base_basename,
521 "tar.xz",
522 constants.IMAGE_TYPE_BASE,
523 ),
524 )
Evan Bennb9dfaf42022-02-14 17:38:26 +1100525
Alex Klein1699fab2022-09-08 08:46:06 -0600526 def _CopyFileToGS(src, dst=None, suffix=None):
527 """Returns |dst| file name if the copying was successful."""
528 if dst is None:
529 dst = src
530 elif suffix is not None:
531 dst = "%s.%s" % (dst, suffix)
532 success = False
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500533 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600534 ctx.Copy(
535 os.path.join(src_path, src),
536 os.path.join(dst_path, dst),
537 recursive=True,
538 )
539 success = True
540 except gs.GSNoSuchKey:
541 logging.warning("Skipping %s as it does not exist", src)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500542 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600543 unknown_error[0] = True
544 logging.error(
545 "Skipping %s due to unknown GS error", src, exc_info=True
546 )
547 return dst if success else None
Mike Frysinger4495b032014-03-05 17:24:03 -0500548
Alex Klein1699fab2022-09-08 08:46:06 -0600549 for src, dst, suffix in files_to_copy_only:
550 _CopyFileToGS(src, dst, suffix)
551
552 # Clear the list of files to sign before adding new artifacts.
553 files_to_sign = []
554
555 def _AddToFilesToSign(image_type, dst, suffix):
556 assert dst.endswith("." + suffix), "dst: %s, suffix: %s" % (
557 dst,
558 suffix,
559 )
560 dst_base = dst[: -(len(suffix) + 1)]
561 files_to_sign.append([image_type, dst_base, suffix])
562
563 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
564 dst = _CopyFileToGS(src, dst, suffix)
565 if dst and (not sign_types or image_type in sign_types):
566 _AddToFilesToSign(image_type, dst, suffix)
567
568 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
569 if sign_types and image_type in sign_types:
570 dst = _CopyFileToGS(src, dst, suffix)
571 if dst:
572 _AddToFilesToSign(image_type, dst, suffix)
573
574 logging.debug("Files to sign: %s", files_to_sign)
575 unused_sign_types = set(sign_types or []) - set(
576 x for x, _, _ in files_to_sign
577 )
578 if unused_sign_types:
579 logging.warning(
580 "Some sign types were unused: %s", " ".join(unused_sign_types)
581 )
582
583 # Now go through the subset for signing.
584 for image_type, dst_name, suffix in files_to_sign:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500585 try:
Alex Klein1699fab2022-09-08 08:46:06 -0600586 input_insns = InputInsns(
587 board, image_type=image_type, buildroot=buildroot
588 )
589 except MissingBoardInstructions as e:
590 logging.info("Nothing to sign: %s", e)
591 continue
592
593 dst_archive = "%s.%s" % (dst_name, suffix)
594 sect_general["archive"] = dst_archive
595 sect_general["type"] = image_type
596
597 # In the default/automatic mode, only flag files for signing if the
598 # archives were actually uploaded in a previous stage. This additional
599 # check can be removed in future once |sign_types| becomes a required
600 # argument.
601 # TODO: Make |sign_types| a required argument.
602 gs_artifact_path = os.path.join(dst_path, dst_archive)
603 exists = False
604 try:
605 exists = ctx.Exists(gs_artifact_path)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500606 except gs.GSContextException:
Alex Klein1699fab2022-09-08 08:46:06 -0600607 unknown_error[0] = True
608 logging.error(
609 "Unknown error while checking %s",
610 gs_artifact_path,
611 exc_info=True,
612 )
613 if not exists:
614 logging.info(
615 "%s does not exist. Nothing to sign.", gs_artifact_path
616 )
617 continue
Don Garrett9459c2f2014-01-22 18:20:24 -0800618
Alex Klein1699fab2022-09-08 08:46:06 -0600619 first_image = True
620 for alt_insn_set in input_insns.GetAltInsnSets():
621 # Figure out which keysets have been requested for this type.
622 # We sort the forced set so tests/runtime behavior is stable.
623 keysets = sorted(force_keysets)
624 if not keysets:
625 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
626 if not keysets:
627 logging.warning(
628 "Skipping %s image signing due to no keysets",
629 image_type,
630 )
Mike Frysinger4495b032014-03-05 17:24:03 -0500631
Alex Klein1699fab2022-09-08 08:46:06 -0600632 for keyset in keysets:
633 sect_insns["keyset"] = keyset
634
635 # Generate the insn file for this artifact that the signer will use,
636 # and flag it for signing.
637 with cros_build_lib.UnbufferedNamedTemporaryFile(
638 prefix="pushimage.insns."
639 ) as insns_path:
640 input_insns.OutputInsns(
641 insns_path.name,
642 sect_insns,
643 sect_general,
644 insns_merge=alt_insn_set,
645 )
646
647 gs_insns_path = "%s/%s" % (dst_path, dst_name)
648 if not first_image:
649 gs_insns_path += "-%s" % keyset
650 first_image = False
651 gs_insns_path += ".instructions"
652
653 try:
654 ctx.Copy(insns_path.name, gs_insns_path)
655 except gs.GSContextException:
656 unknown_error[0] = True
657 logging.error(
658 "Unknown error while uploading insns %s",
659 gs_insns_path,
660 exc_info=True,
661 )
662 continue
663
664 try:
665 MarkImageToBeSigned(
666 ctx, tbs_base, gs_insns_path, priority
667 )
668 except gs.GSContextException:
669 unknown_error[0] = True
670 logging.error(
671 "Unknown error while marking for signing %s",
672 gs_insns_path,
673 exc_info=True,
674 )
675 continue
676 logging.info(
677 "Signing %s image with keyset %s at %s",
678 image_type,
679 keyset,
680 gs_insns_path,
681 )
682 instruction_urls.setdefault(channel, []).append(
683 gs_insns_path
684 )
685
686 if unknown_error[0]:
687 raise PushError("hit some unknown error(s)", instruction_urls)
688
689 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400690
691
Mike Frysinger26144192017-08-30 18:26:46 -0400692def GetParser():
Alex Klein1699fab2022-09-08 08:46:06 -0600693 """Creates the argparse parser."""
694 parser = commandline.ArgumentParser(description=__doc__)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400695
Alex Klein1699fab2022-09-08 08:46:06 -0600696 # The type of image_dir will strip off trailing slashes (makes later
697 # processing simpler and the display prettier).
698 parser.add_argument(
699 "image_dir",
700 default=None,
701 type="local_or_gs_path",
702 help="full path of source artifacts to upload",
703 )
704 parser.add_argument(
705 "--board",
706 default=None,
707 required=True,
708 help="board to generate symbols for",
709 )
710 parser.add_argument(
711 "--profile", default=None, help='board profile in use (e.g. "asan")'
712 )
713 parser.add_argument(
714 "--version",
715 default=None,
716 help="version info (normally extracted from image_dir)",
717 )
718 parser.add_argument(
719 "--channels",
720 default=None,
721 action="split_extend",
722 help="override list of channels to process",
723 )
724 parser.add_argument(
725 "-n",
726 "--dry-run",
727 default=False,
728 action="store_true",
729 help="show what would be done, but do not upload",
730 )
731 parser.add_argument(
732 "-M",
733 "--mock",
734 default=False,
735 action="store_true",
736 help="upload things to a testing bucket (dev testing)",
737 )
738 parser.add_argument(
739 "--test-sign",
740 default=[],
741 action="append",
742 choices=TEST_KEYSETS,
743 help="mung signing behavior to sign w/ test keys",
744 )
745 parser.add_argument(
746 "--priority",
747 type=int,
748 default=50,
749 help="set signing priority (lower == higher prio)",
750 )
751 parser.add_argument(
752 "--sign-types",
753 default=None,
754 nargs="+",
755 choices=_SUPPORTED_IMAGE_TYPES,
756 help="only sign specified image types",
757 )
758 parser.add_argument(
759 "--buildroot",
760 default=constants.SOURCE_ROOT,
761 type="path",
762 help="Buildroot to use. Defaults to current.",
763 )
764 parser.add_argument(
765 "--yes",
766 action="store_true",
767 default=False,
768 help="answer yes to all prompts",
769 )
770 parser.add_argument(
771 "--dest-bucket",
772 default=constants.RELEASE_BUCKET,
773 help="dest bucket. Default to %(default)s",
774 )
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400775
Alex Klein1699fab2022-09-08 08:46:06 -0600776 return parser
Mike Frysinger26144192017-08-30 18:26:46 -0400777
778
779def main(argv):
Alex Klein1699fab2022-09-08 08:46:06 -0600780 parser = GetParser()
781 opts = parser.parse_args(argv)
782 opts.Freeze()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400783
Alex Klein1699fab2022-09-08 08:46:06 -0600784 force_keysets = set(
785 ["%s-%s" % (TEST_KEYSET_PREFIX, x) for x in opts.test_sign]
786 )
Mike Frysingerdad40d62014-02-09 02:18:02 -0500787
Alex Klein1699fab2022-09-08 08:46:06 -0600788 # If we aren't using mock or test or dry run mode, then let's prompt the user
789 # to make sure they actually want to do this. It's rare that people want to
790 # run this directly and hit the release bucket.
791 if not (opts.mock or force_keysets or opts.dry_run) and not opts.yes:
792 prolog = "\n".join(
793 textwrap.wrap(
794 textwrap.dedent(
795 "Uploading images for signing to the *release* bucket is not something "
796 "you generally should be doing yourself."
797 ),
798 80,
799 )
800 ).strip()
801 if not cros_build_lib.BooleanPrompt(
802 prompt="Are you sure you want to sign these images",
803 default=False,
804 prolog=prolog,
805 ):
806 cros_build_lib.Die("better safe than sorry")
Mike Frysinger09fe0122014-02-09 02:44:05 -0500807
Alex Klein1699fab2022-09-08 08:46:06 -0600808 PushImage(
809 opts.image_dir,
810 opts.board,
811 versionrev=opts.version,
812 profile=opts.profile,
813 priority=opts.priority,
814 sign_types=opts.sign_types,
815 dry_run=opts.dry_run,
816 mock=opts.mock,
817 force_keysets=force_keysets,
818 force_channels=opts.channels,
819 buildroot=opts.buildroot,
820 dest_bucket=opts.dest_bucket,
821 )