blob: 26e8d89f6c1bad2fa911df97a6dd1ee26bb30886 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Mike Frysingerd13faeb2013-09-05 16:00:46 -04002# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""ChromeOS image pusher (from cbuildbot to signer).
7
8This pushes files from the archive bucket to the signer bucket and marks
9artifacts for signing (which a signing process will look for).
10"""
11
12from __future__ import print_function
13
14import ConfigParser
15import cStringIO
Mike Frysingerd13faeb2013-09-05 16:00:46 -040016import getpass
17import os
18import re
19import tempfile
Mike Frysinger09fe0122014-02-09 02:44:05 -050020import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040021
Aviv Keshetb7519e12016-10-04 00:50:00 -070022from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040023from chromite.lib import commandline
24from chromite.lib import cros_build_lib
Ralph Nathan5a582ff2015-03-20 18:18:30 -070025from chromite.lib import cros_logging as logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040026from chromite.lib import gs
27from chromite.lib import osutils
28from chromite.lib import signing
29
30
31# This will split a fully qualified ChromeOS version string up.
32# R34-5126.0.0 will break into "34" and "5126.0.0".
33VERSION_REGEX = r'^R([0-9]+)-([^-]+)'
34
Mike Frysingerdad40d62014-02-09 02:18:02 -050035# The test signers will scan this dir looking for test work.
36# Keep it in sync with the signer config files [gs_test_buckets].
37TEST_SIGN_BUCKET_BASE = 'gs://chromeos-throw-away-bucket/signer-tests'
38
David Rileyf8205122015-09-04 13:46:36 -070039# Keysets that are only valid in the above test bucket.
40TEST_KEYSET_PREFIX = 'test-keys'
41TEST_KEYSETS = set((
42 'mp',
43 'premp',
44 'nvidia-premp',
45))
Mike Frysingerdad40d62014-02-09 02:18:02 -050046
Amey Deshpandea936c622015-08-12 17:27:54 -070047# Supported image types for signing.
48_SUPPORTED_IMAGE_TYPES = (
49 constants.IMAGE_TYPE_RECOVERY,
50 constants.IMAGE_TYPE_FACTORY,
51 constants.IMAGE_TYPE_FIRMWARE,
David Rileya04d19d2015-09-04 16:11:50 -070052 constants.IMAGE_TYPE_NV_LP0_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070053 constants.IMAGE_TYPE_ACCESSORY_USBPD,
54 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070055 constants.IMAGE_TYPE_BASE,
56)
57
Mike Frysingerd13faeb2013-09-05 16:00:46 -040058
Mike Frysinger4495b032014-03-05 17:24:03 -050059class PushError(Exception):
60 """When an (unknown) error happened while trying to push artifacts."""
61
62
Mike Frysingerd13faeb2013-09-05 16:00:46 -040063class MissingBoardInstructions(Exception):
64 """Raised when a board lacks any signer instructions."""
65
Mike Frysingerd84d91e2015-11-05 18:02:24 -050066 def __init__(self, board, image_type, input_insns):
67 Exception.__init__(self, 'Board %s lacks insns for %s image: %s not found' %
68 (board, image_type, input_insns))
69
Mike Frysingerd13faeb2013-09-05 16:00:46 -040070
71class InputInsns(object):
72 """Object to hold settings for a signable board.
73
74 Note: The format of the instruction file pushimage outputs (and the signer
75 reads) is not exactly the same as the instruction file pushimage reads.
76 """
77
Don Garrett3cf5f9a2018-08-14 13:14:47 -070078 def __init__(self, board, image_type=None, buildroot=None):
Mike Frysingerd84d91e2015-11-05 18:02:24 -050079 """Initialization.
80
81 Args:
82 board: The board to look up details.
83 image_type: The type of image we will be signing (see --sign-types).
Don Garrett3cf5f9a2018-08-14 13:14:47 -070084 buildroot: Buildroot in which to look for signing instructions.
Mike Frysingerd84d91e2015-11-05 18:02:24 -050085 """
Mike Frysingerd13faeb2013-09-05 16:00:46 -040086 self.board = board
Don Garrett3cf5f9a2018-08-14 13:14:47 -070087 self.buildroot = buildroot or constants.SOURCE_ROOT
Mike Frysingerd13faeb2013-09-05 16:00:46 -040088
89 config = ConfigParser.ConfigParser()
90 config.readfp(open(self.GetInsnFile('DEFAULT')))
Mike Frysingerd84d91e2015-11-05 18:02:24 -050091
Amey Deshpandea936c622015-08-12 17:27:54 -070092 # What pushimage internally refers to as 'recovery', are the basic signing
93 # instructions in practice, and other types are stacked on top.
Mike Frysingerd84d91e2015-11-05 18:02:24 -050094 if image_type is None:
95 image_type = constants.IMAGE_TYPE_RECOVERY
96 self.image_type = image_type
Amey Deshpandea936c622015-08-12 17:27:54 -070097 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
98 if not os.path.exists(input_insns):
99 # This board doesn't have any signing instructions.
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500100 raise MissingBoardInstructions(self.board, image_type, input_insns)
Amey Deshpandea936c622015-08-12 17:27:54 -0700101 config.readfp(open(input_insns))
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500102
103 if image_type is not None:
104 input_insns = self.GetInsnFile(image_type)
105 if not os.path.exists(input_insns):
106 # This type doesn't have any signing instructions.
107 raise MissingBoardInstructions(self.board, image_type, input_insns)
108
109 self.image_type = image_type
110 config.readfp(open(input_insns))
111
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400112 self.cfg = config
113
114 def GetInsnFile(self, image_type):
115 """Find the signer instruction files for this board/image type.
116
117 Args:
118 image_type: The type of instructions to load. It can be a common file
119 (like "DEFAULT"), or one of the --sign-types.
120
121 Returns:
122 Full path to the instruction file using |image_type| and |self.board|.
123 """
124 if image_type == image_type.upper():
125 name = image_type
Amey Deshpandea936c622015-08-12 17:27:54 -0700126 elif image_type in (constants.IMAGE_TYPE_RECOVERY,
127 constants.IMAGE_TYPE_BASE):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400128 name = self.board
129 else:
130 name = '%s.%s' % (self.board, image_type)
131
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700132 return os.path.join(
133 self.buildroot, signing.INPUT_INSN_DIR_REL, '%s.instructions' % name)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400134
135 @staticmethod
136 def SplitCfgField(val):
137 """Split a string into multiple elements.
138
139 This centralizes our convention for multiple elements in the input files
140 being delimited by either a space or comma.
141
142 Args:
143 val: The string to split.
144
145 Returns:
146 The list of elements from having done split the string.
147 """
148 return val.replace(',', ' ').split()
149
150 def GetChannels(self):
151 """Return the list of channels to sign for this board.
152
153 If the board-specific config doesn't specify a preference, we'll use the
154 common settings.
155 """
156 return self.SplitCfgField(self.cfg.get('insns', 'channel'))
157
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500158 def GetKeysets(self, insns_merge=None):
159 """Return the list of keysets to sign for this board.
160
161 Args:
162 insns_merge: The additional section to look at over [insns].
163 """
164 # First load the default value from [insns.keyset] if available.
165 sections = ['insns']
166 # Then overlay the [insns.xxx.keyset] if requested.
167 if insns_merge is not None:
168 sections += [insns_merge]
169
170 keyset = ''
171 for section in sections:
172 try:
173 keyset = self.cfg.get(section, 'keyset')
174 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
175 pass
176
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500177 # We do not perturb the order (e.g. using sorted() or making a set())
178 # because we want the behavior stable, and we want the input insns to
179 # explicitly control the order (since it has an impact on naming).
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500180 return self.SplitCfgField(keyset)
181
182 def GetAltInsnSets(self):
183 """Return the list of alternative insn sections."""
184 # We do not perturb the order (e.g. using sorted() or making a set())
185 # because we want the behavior stable, and we want the input insns to
186 # explicitly control the order (since it has an impact on naming).
187 ret = [x for x in self.cfg.sections() if x.startswith('insns.')]
188 return ret if ret else [None]
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400189
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500190 @staticmethod
191 def CopyConfigParser(config):
192 """Return a copy of a ConfigParser object.
193
Thiemo Nagel9fb99722017-05-26 16:26:40 +0200194 The python folks broke the ability to use something like deepcopy:
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500195 https://bugs.python.org/issue16058
196 """
197 # Write the current config to a string io object.
198 data = cStringIO.StringIO()
199 config.write(data)
200 data.seek(0)
201
202 # Create a new ConfigParser from the serialized data.
203 ret = ConfigParser.ConfigParser()
204 ret.readfp(data)
205
206 return ret
207
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500208 def OutputInsns(self, output_file, sect_insns, sect_general,
209 insns_merge=None):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400210 """Generate the output instruction file for sending to the signer.
211
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500212 The override order is (later has precedence):
213 [insns]
214 [insns_merge] (should be named "insns.xxx")
215 sect_insns
216
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400217 Note: The format of the instruction file pushimage outputs (and the signer
218 reads) is not exactly the same as the instruction file pushimage reads.
219
220 Args:
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400221 output_file: The file to write the new instruction file to.
222 sect_insns: Items to set/override in the [insns] section.
223 sect_general: Items to set/override in the [general] section.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500224 insns_merge: The alternative insns.xxx section to merge.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400225 """
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500226 # Create a copy so we can clobber certain fields.
227 config = self.CopyConfigParser(self.cfg)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500228 sect_insns = sect_insns.copy()
229
230 # Merge in the alternative insns section if need be.
231 if insns_merge is not None:
232 for k, v in config.items(insns_merge):
233 sect_insns.setdefault(k, v)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400234
235 # Clear channel entry in instructions file, ensuring we only get
236 # one channel for the signer to look at. Then provide all the
237 # other details for this signing request to avoid any ambiguity
238 # and to avoid relying on encoding data into filenames.
239 for sect, fields in zip(('insns', 'general'), (sect_insns, sect_general)):
240 if not config.has_section(sect):
241 config.add_section(sect)
242 for k, v in fields.iteritems():
243 config.set(sect, k, v)
244
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500245 # Now prune the alternative sections.
246 for alt in self.GetAltInsnSets():
247 config.remove_section(alt)
248
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400249 output = cStringIO.StringIO()
250 config.write(output)
251 data = output.getvalue()
252 osutils.WriteFile(output_file, data)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500253 logging.debug('generated insns file for %s:\n%s', self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400254
255
256def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
257 """Mark an instructions file for signing.
258
259 This will upload a file to the GS bucket flagging an image for signing by
260 the signers.
261
262 Args:
263 ctx: A viable gs.GSContext.
264 tbs_base: The full path to where the tobesigned directory lives.
265 insns_path: The path (relative to |tbs_base|) of the file to sign.
266 priority: Set the signing priority (lower == higher prio).
267
268 Returns:
269 The full path to the remote tobesigned file.
270 """
271 if priority < 0 or priority > 99:
272 raise ValueError('priority must be [0, 99] inclusive')
273
274 if insns_path.startswith(tbs_base):
275 insns_path = insns_path[len(tbs_base):].lstrip('/')
276
277 tbs_path = '%s/tobesigned/%02i,%s' % (tbs_base, priority,
278 insns_path.replace('/', ','))
279
Mike Frysinger6430d132014-10-27 23:43:30 -0400280 # The caller will catch gs.GSContextException for us.
281 ctx.Copy('-', tbs_path, input=cros_build_lib.MachineDetails())
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400282
283 return tbs_path
284
285
286def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
Mike Frysinger77912102017-08-30 18:35:46 -0400287 sign_types=None, dry_run=False, mock=False, force_keysets=(),
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700288 force_channels=None, buildroot=constants.SOURCE_ROOT):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400289 """Push the image from the archive bucket to the release bucket.
290
291 Args:
292 src_path: Where to copy the files from; can be a local path or gs:// URL.
293 Should be a full path to the artifacts in either case.
294 board: The board we're uploading artifacts for (e.g. $BOARD).
295 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
296 profile: The board profile in use (e.g. "asan").
297 priority: Set the signing priority (lower == higher prio).
298 sign_types: If set, a set of types which we'll restrict ourselves to
299 signing. See the --sign-types option for more details.
300 dry_run: Show what would be done, but do not upload anything.
301 mock: Upload to a testing bucket rather than the real one.
Mike Frysingerdad40d62014-02-09 02:18:02 -0500302 force_keysets: Set of keysets to use rather than what the inputs say.
Mike Frysinger77912102017-08-30 18:35:46 -0400303 force_channels: Set of channels to use rather than what the inputs say.
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700304 buildroot: Buildroot in which to look for signing instructions.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400305
306 Returns:
Don Garrett9459c2f2014-01-22 18:20:24 -0800307 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
308 'gs://signer_instruction_uri2',
309 ...]
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400310 """
Mike Frysinger4495b032014-03-05 17:24:03 -0500311 # Whether we hit an unknown error. If so, we'll throw an error, but only
312 # at the end (so that we still upload as many files as possible).
Amey Deshpandea936c622015-08-12 17:27:54 -0700313 # It's implemented using a list to deal with variable scopes in nested
314 # functions below.
315 unknown_error = [False]
Mike Frysinger4495b032014-03-05 17:24:03 -0500316
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400317 if versionrev is None:
318 # Extract milestone/version from the directory name.
319 versionrev = os.path.basename(src_path)
320
321 # We only support the latest format here. Older releases can use pushimage
322 # from the respective branch which deals with legacy cruft.
323 m = re.match(VERSION_REGEX, versionrev)
324 if not m:
325 raise ValueError('version %s does not match %s' %
326 (versionrev, VERSION_REGEX))
327 milestone = m.group(1)
328 version = m.group(2)
329
330 # Normalize board to always use dashes not underscores. This is mostly a
331 # historical artifact at this point, but we can't really break it since the
332 # value is used in URLs.
333 boardpath = board.replace('_', '-')
334 if profile is not None:
335 boardpath += '-%s' % profile.replace('_', '-')
336
337 ctx = gs.GSContext(dry_run=dry_run)
338
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400339 try:
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700340 input_insns = InputInsns(board, buildroot=buildroot)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400341 except MissingBoardInstructions as e:
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500342 logging.warning('Missing base instruction file: %s', e)
Ralph Nathan446aee92015-03-23 14:44:56 -0700343 logging.warning('not uploading anything for signing')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400344 return
Mike Frysinger77912102017-08-30 18:35:46 -0400345
346 if force_channels is None:
347 channels = input_insns.GetChannels()
348 else:
349 # Filter out duplicates.
350 channels = sorted(set(force_channels))
Mike Frysingerdad40d62014-02-09 02:18:02 -0500351
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500352 # We want force_keysets as a set.
Mike Frysingerdad40d62014-02-09 02:18:02 -0500353 force_keysets = set(force_keysets)
Mike Frysingerdad40d62014-02-09 02:18:02 -0500354
355 if mock:
Ralph Nathan03047282015-03-23 11:09:32 -0700356 logging.info('Upload mode: mock; signers will not process anything')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500357 tbs_base = gs_base = os.path.join(constants.TRASH_BUCKET, 'pushimage-tests',
358 getpass.getuser())
David Rileyf8205122015-09-04 13:46:36 -0700359 elif set(['%s-%s' % (TEST_KEYSET_PREFIX, x)
360 for x in TEST_KEYSETS]) & force_keysets:
Ralph Nathan03047282015-03-23 11:09:32 -0700361 logging.info('Upload mode: test; signers will process test keys')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500362 # We need the tbs_base to be in the place the signer will actually scan.
363 tbs_base = TEST_SIGN_BUCKET_BASE
364 gs_base = os.path.join(tbs_base, getpass.getuser())
365 else:
Ralph Nathan03047282015-03-23 11:09:32 -0700366 logging.info('Upload mode: normal; signers will process the images')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500367 tbs_base = gs_base = constants.RELEASE_BUCKET
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400368
369 sect_general = {
370 'config_board': board,
371 'board': boardpath,
372 'version': version,
373 'versionrev': versionrev,
374 'milestone': milestone,
375 }
376 sect_insns = {}
377
378 if dry_run:
Ralph Nathan03047282015-03-23 11:09:32 -0700379 logging.info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED')
380 logging.info('Signing for channels: %s', ' '.join(channels))
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400381
Don Garrett9459c2f2014-01-22 18:20:24 -0800382 instruction_urls = {}
383
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400384 def _ImageNameBase(image_type=None):
385 lmid = ('%s-' % image_type) if image_type else ''
386 return 'ChromeOS-%s%s-%s' % (lmid, versionrev, boardpath)
387
Amey Deshpandea936c622015-08-12 17:27:54 -0700388 # These variables are defined outside the loop so that the nested functions
389 # below can access them without 'cell-var-from-loop' linter warning.
390 dst_path = ""
391 files_to_sign = []
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400392 for channel in channels:
Ralph Nathan5a582ff2015-03-20 18:18:30 -0700393 logging.debug('\n\n#### CHANNEL: %s ####\n', channel)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400394 sect_insns['channel'] = channel
395 sub_path = '%s-channel/%s/%s' % (channel, boardpath, version)
396 dst_path = '%s/%s' % (gs_base, sub_path)
Ralph Nathan03047282015-03-23 11:09:32 -0700397 logging.info('Copying images to %s', dst_path)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400398
Amey Deshpandea936c622015-08-12 17:27:54 -0700399 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
400 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
401 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
David Rileya04d19d2015-09-04 16:11:50 -0700402 nv_lp0_firmware_basename = _ImageNameBase(
403 constants.IMAGE_TYPE_NV_LP0_FIRMWARE)
Vincent Palatind599c662015-10-26 09:51:41 -0700404 acc_usbpd_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_USBPD)
405 acc_rwsig_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_RWSIG)
Amey Deshpandea936c622015-08-12 17:27:54 -0700406 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
407 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400408 hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev)
409
Amey Deshpandea936c622015-08-12 17:27:54 -0700410 # The following build artifacts, if present, are always copied regardless of
411 # requested signing types.
412 files_to_copy_only = (
413 # (<src>, <dst>, <suffix>),
414 ('image.zip', _ImageNameBase(), 'zip'),
415 (constants.TEST_IMAGE_TAR, test_basename, 'tar.xz'),
416 ('debug.tgz', 'debug-%s' % boardpath, 'tgz'),
417 (hwqual_tarball, '', ''),
Amey Deshpandea936c622015-08-12 17:27:54 -0700418 ('stateful.tgz', '', ''),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400419 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700420
421 # The following build artifacts, if present, are always copied.
422 # If |sign_types| is None, all of them are marked for signing, otherwise
423 # only the image types specified in |sign_types| are marked for signing.
424 files_to_copy_and_maybe_sign = (
425 # (<src>, <dst>, <suffix>, <signing type>),
426 (constants.RECOVERY_IMAGE_TAR, recovery_basename, 'tar.xz',
427 constants.IMAGE_TYPE_RECOVERY),
428
429 ('factory_image.zip', factory_basename, 'zip',
430 constants.IMAGE_TYPE_FACTORY),
431
432 ('firmware_from_source.tar.bz2', firmware_basename, 'tar.bz2',
433 constants.IMAGE_TYPE_FIRMWARE),
David Rileya04d19d2015-09-04 16:11:50 -0700434
435 ('firmware_from_source.tar.bz2', nv_lp0_firmware_basename, 'tar.bz2',
436 constants.IMAGE_TYPE_NV_LP0_FIRMWARE),
Vincent Palatind599c662015-10-26 09:51:41 -0700437
438 ('firmware_from_source.tar.bz2', acc_usbpd_basename, 'tar.bz2',
439 constants.IMAGE_TYPE_ACCESSORY_USBPD),
440
441 ('firmware_from_source.tar.bz2', acc_rwsig_basename, 'tar.bz2',
442 constants.IMAGE_TYPE_ACCESSORY_RWSIG),
Amey Deshpandea936c622015-08-12 17:27:54 -0700443 )
444
445 # The following build artifacts are copied and marked for signing, if
446 # they are present *and* if the image type is specified via |sign_types|.
447 files_to_maybe_copy_and_sign = (
448 # (<src>, <dst>, <suffix>, <signing type>),
449 (constants.BASE_IMAGE_TAR, base_basename, 'tar.xz',
450 constants.IMAGE_TYPE_BASE),
451 )
452
453 def _CopyFileToGS(src, dst, suffix):
454 """Returns |dst| file name if the copying was successful."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400455 if not dst:
456 dst = src
Amey Deshpandea936c622015-08-12 17:27:54 -0700457 elif suffix:
458 dst = '%s.%s' % (dst, suffix)
459 success = False
Mike Frysingere51a2652014-01-18 02:36:16 -0500460 try:
461 ctx.Copy(os.path.join(src_path, src), os.path.join(dst_path, dst))
Amey Deshpandea936c622015-08-12 17:27:54 -0700462 success = True
Mike Frysingere51a2652014-01-18 02:36:16 -0500463 except gs.GSNoSuchKey:
Ralph Nathan446aee92015-03-23 14:44:56 -0700464 logging.warning('Skipping %s as it does not exist', src)
Mike Frysinger4495b032014-03-05 17:24:03 -0500465 except gs.GSContextException:
Amey Deshpandea936c622015-08-12 17:27:54 -0700466 unknown_error[0] = True
Ralph Nathan59900422015-03-24 10:41:17 -0700467 logging.error('Skipping %s due to unknown GS error', src, exc_info=True)
Amey Deshpandea936c622015-08-12 17:27:54 -0700468 return dst if success else None
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400469
Amey Deshpandea936c622015-08-12 17:27:54 -0700470 for src, dst, suffix in files_to_copy_only:
471 _CopyFileToGS(src, dst, suffix)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400472
Amey Deshpandea936c622015-08-12 17:27:54 -0700473 # Clear the list of files to sign before adding new artifacts.
474 files_to_sign = []
475
476 def _AddToFilesToSign(image_type, dst, suffix):
477 assert dst.endswith('.' + suffix), (
478 'dst: %s, suffix: %s' % (dst, suffix))
479 dst_base = dst[:-(len(suffix) + 1)]
480 files_to_sign.append([image_type, dst_base, suffix])
481
482 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
483 dst = _CopyFileToGS(src, dst, suffix)
484 if dst and (not sign_types or image_type in sign_types):
485 _AddToFilesToSign(image_type, dst, suffix)
486
487 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
488 if sign_types and image_type in sign_types:
489 dst = _CopyFileToGS(src, dst, suffix)
490 if dst:
491 _AddToFilesToSign(image_type, dst, suffix)
492
493 logging.debug('Files to sign: %s', files_to_sign)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400494 # Now go through the subset for signing.
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500495 for image_type, dst_name, suffix in files_to_sign:
496 try:
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700497 input_insns = InputInsns(board, image_type=image_type,
498 buildroot=buildroot)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500499 except MissingBoardInstructions as e:
500 logging.info('Nothing to sign: %s', e)
501 continue
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400502
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500503 dst_archive = '%s.%s' % (dst_name, suffix)
504 sect_general['archive'] = dst_archive
505 sect_general['type'] = image_type
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400506
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500507 # In the default/automatic mode, only flag files for signing if the
508 # archives were actually uploaded in a previous stage. This additional
509 # check can be removed in future once |sign_types| becomes a required
510 # argument.
511 # TODO: Make |sign_types| a required argument.
512 gs_artifact_path = os.path.join(dst_path, dst_archive)
513 exists = False
514 try:
515 exists = ctx.Exists(gs_artifact_path)
516 except gs.GSContextException:
517 unknown_error[0] = True
518 logging.error('Unknown error while checking %s', gs_artifact_path,
519 exc_info=True)
520 if not exists:
521 logging.info('%s does not exist. Nothing to sign.',
522 gs_artifact_path)
523 continue
524
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500525 first_image = True
526 for alt_insn_set in input_insns.GetAltInsnSets():
527 # Figure out which keysets have been requested for this type.
528 # We sort the forced set so tests/runtime behavior is stable.
529 keysets = sorted(force_keysets)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500530 if not keysets:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500531 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
532 if not keysets:
533 logging.warning('Skipping %s image signing due to no keysets',
534 image_type)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500535
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500536 for keyset in keysets:
537 sect_insns['keyset'] = keyset
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400538
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500539 # Generate the insn file for this artifact that the signer will use,
540 # and flag it for signing.
541 with tempfile.NamedTemporaryFile(
542 bufsize=0, prefix='pushimage.insns.') as insns_path:
543 input_insns.OutputInsns(insns_path.name, sect_insns, sect_general,
544 insns_merge=alt_insn_set)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400545
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500546 gs_insns_path = '%s/%s' % (dst_path, dst_name)
547 if not first_image:
548 gs_insns_path += '-%s' % keyset
549 first_image = False
550 gs_insns_path += '.instructions'
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400551
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500552 try:
553 ctx.Copy(insns_path.name, gs_insns_path)
554 except gs.GSContextException:
555 unknown_error[0] = True
556 logging.error('Unknown error while uploading insns %s',
557 gs_insns_path, exc_info=True)
558 continue
Mike Frysinger4495b032014-03-05 17:24:03 -0500559
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500560 try:
561 MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
562 except gs.GSContextException:
563 unknown_error[0] = True
564 logging.error('Unknown error while marking for signing %s',
565 gs_insns_path, exc_info=True)
566 continue
567 logging.info('Signing %s image with keyset %s at %s', image_type,
568 keyset, gs_insns_path)
569 instruction_urls.setdefault(channel, []).append(gs_insns_path)
Don Garrett9459c2f2014-01-22 18:20:24 -0800570
Amey Deshpandea936c622015-08-12 17:27:54 -0700571 if unknown_error[0]:
Mike Frysinger4495b032014-03-05 17:24:03 -0500572 raise PushError('hit some unknown error(s)', instruction_urls)
573
Don Garrett9459c2f2014-01-22 18:20:24 -0800574 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400575
576
Mike Frysinger26144192017-08-30 18:26:46 -0400577def GetParser():
578 """Creates the argparse parser."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400579 parser = commandline.ArgumentParser(description=__doc__)
580
581 # The type of image_dir will strip off trailing slashes (makes later
582 # processing simpler and the display prettier).
583 parser.add_argument('image_dir', default=None, type='local_or_gs_path',
584 help='full path of source artifacts to upload')
585 parser.add_argument('--board', default=None, required=True,
586 help='board to generate symbols for')
587 parser.add_argument('--profile', default=None,
588 help='board profile in use (e.g. "asan")')
589 parser.add_argument('--version', default=None,
590 help='version info (normally extracted from image_dir)')
Mike Frysinger77912102017-08-30 18:35:46 -0400591 parser.add_argument('--channels', default=None, action='split_extend',
592 help='override list of channels to process')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400593 parser.add_argument('-n', '--dry-run', default=False, action='store_true',
594 help='show what would be done, but do not upload')
595 parser.add_argument('-M', '--mock', default=False, action='store_true',
596 help='upload things to a testing bucket (dev testing)')
David Rileyf8205122015-09-04 13:46:36 -0700597 parser.add_argument('--test-sign', default=[], action='append',
598 choices=TEST_KEYSETS,
599 help='mung signing behavior to sign w/ test keys')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400600 parser.add_argument('--priority', type=int, default=50,
601 help='set signing priority (lower == higher prio)')
602 parser.add_argument('--sign-types', default=None, nargs='+',
Amey Deshpandea936c622015-08-12 17:27:54 -0700603 choices=_SUPPORTED_IMAGE_TYPES,
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400604 help='only sign specified image types')
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700605 parser.add_argument('--buildroot', default=constants.SOURCE_ROOT, type='path',
606 help='Buildroot to use. Defaults to current.')
Mike Frysinger09fe0122014-02-09 02:44:05 -0500607 parser.add_argument('--yes', action='store_true', default=False,
608 help='answer yes to all prompts')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400609
Mike Frysinger26144192017-08-30 18:26:46 -0400610 return parser
611
612
613def main(argv):
614 parser = GetParser()
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400615 opts = parser.parse_args(argv)
616 opts.Freeze()
617
David Rileyf8205122015-09-04 13:46:36 -0700618 force_keysets = set(['%s-%s' % (TEST_KEYSET_PREFIX, x)
619 for x in opts.test_sign])
Mike Frysingerdad40d62014-02-09 02:18:02 -0500620
Mike Frysinger09fe0122014-02-09 02:44:05 -0500621 # If we aren't using mock or test or dry run mode, then let's prompt the user
622 # to make sure they actually want to do this. It's rare that people want to
623 # run this directly and hit the release bucket.
624 if not (opts.mock or force_keysets or opts.dry_run) and not opts.yes:
625 prolog = '\n'.join(textwrap.wrap(textwrap.dedent(
626 'Uploading images for signing to the *release* bucket is not something '
627 'you generally should be doing yourself.'), 80)).strip()
628 if not cros_build_lib.BooleanPrompt(
629 prompt='Are you sure you want to sign these images',
630 default=False, prolog=prolog):
631 cros_build_lib.Die('better safe than sorry')
632
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400633 PushImage(opts.image_dir, opts.board, versionrev=opts.version,
634 profile=opts.profile, priority=opts.priority,
Mike Frysingerdad40d62014-02-09 02:18:02 -0500635 sign_types=opts.sign_types, dry_run=opts.dry_run, mock=opts.mock,
Don Garrett3cf5f9a2018-08-14 13:14:47 -0700636 force_keysets=force_keysets, force_channels=opts.channels,
637 buildroot=opts.buildroot)