blob: ba30b3d588bccd3cbeb2062a06493fec13a98bb7 [file] [log] [blame]
Mike Frysingerd13faeb2013-09-05 16:00:46 -04001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
11from __future__ import print_function
12
13import ConfigParser
14import cStringIO
Mike Frysingerd13faeb2013-09-05 16:00:46 -040015import getpass
16import os
17import re
18import tempfile
Mike Frysinger09fe0122014-02-09 02:44:05 -050019import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040020
Aviv Keshetb7519e12016-10-04 00:50:00 -070021from chromite.lib import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040022from chromite.lib import commandline
23from chromite.lib import cros_build_lib
Ralph Nathan5a582ff2015-03-20 18:18:30 -070024from chromite.lib import cros_logging as logging
Mike Frysingerd13faeb2013-09-05 16:00:46 -040025from chromite.lib import gs
26from chromite.lib import osutils
27from chromite.lib import signing
28
29
30# This will split a fully qualified ChromeOS version string up.
31# R34-5126.0.0 will break into "34" and "5126.0.0".
32VERSION_REGEX = r'^R([0-9]+)-([^-]+)'
33
Mike Frysingerdad40d62014-02-09 02:18:02 -050034# The test signers will scan this dir looking for test work.
35# Keep it in sync with the signer config files [gs_test_buckets].
36TEST_SIGN_BUCKET_BASE = 'gs://chromeos-throw-away-bucket/signer-tests'
37
David Rileyf8205122015-09-04 13:46:36 -070038# Keysets that are only valid in the above test bucket.
39TEST_KEYSET_PREFIX = 'test-keys'
40TEST_KEYSETS = set((
41 'mp',
42 'premp',
43 'nvidia-premp',
44))
Mike Frysingerdad40d62014-02-09 02:18:02 -050045
Amey Deshpandea936c622015-08-12 17:27:54 -070046# Supported image types for signing.
47_SUPPORTED_IMAGE_TYPES = (
48 constants.IMAGE_TYPE_RECOVERY,
49 constants.IMAGE_TYPE_FACTORY,
50 constants.IMAGE_TYPE_FIRMWARE,
David Rileya04d19d2015-09-04 16:11:50 -070051 constants.IMAGE_TYPE_NV_LP0_FIRMWARE,
Vincent Palatind599c662015-10-26 09:51:41 -070052 constants.IMAGE_TYPE_ACCESSORY_USBPD,
53 constants.IMAGE_TYPE_ACCESSORY_RWSIG,
Amey Deshpandea936c622015-08-12 17:27:54 -070054 constants.IMAGE_TYPE_BASE,
55)
56
Mike Frysingerd13faeb2013-09-05 16:00:46 -040057
Mike Frysinger4495b032014-03-05 17:24:03 -050058class PushError(Exception):
59 """When an (unknown) error happened while trying to push artifacts."""
60
61
Mike Frysingerd13faeb2013-09-05 16:00:46 -040062class MissingBoardInstructions(Exception):
63 """Raised when a board lacks any signer instructions."""
64
Mike Frysingerd84d91e2015-11-05 18:02:24 -050065 def __init__(self, board, image_type, input_insns):
66 Exception.__init__(self, 'Board %s lacks insns for %s image: %s not found' %
67 (board, image_type, input_insns))
68
Mike Frysingerd13faeb2013-09-05 16:00:46 -040069
70class InputInsns(object):
71 """Object to hold settings for a signable board.
72
73 Note: The format of the instruction file pushimage outputs (and the signer
74 reads) is not exactly the same as the instruction file pushimage reads.
75 """
76
Mike Frysingerd84d91e2015-11-05 18:02:24 -050077 def __init__(self, board, image_type=None):
78 """Initialization.
79
80 Args:
81 board: The board to look up details.
82 image_type: The type of image we will be signing (see --sign-types).
83 """
Mike Frysingerd13faeb2013-09-05 16:00:46 -040084 self.board = board
85
86 config = ConfigParser.ConfigParser()
87 config.readfp(open(self.GetInsnFile('DEFAULT')))
Mike Frysingerd84d91e2015-11-05 18:02:24 -050088
Amey Deshpandea936c622015-08-12 17:27:54 -070089 # What pushimage internally refers to as 'recovery', are the basic signing
90 # instructions in practice, and other types are stacked on top.
Mike Frysingerd84d91e2015-11-05 18:02:24 -050091 if image_type is None:
92 image_type = constants.IMAGE_TYPE_RECOVERY
93 self.image_type = image_type
Amey Deshpandea936c622015-08-12 17:27:54 -070094 input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY)
95 if not os.path.exists(input_insns):
96 # This board doesn't have any signing instructions.
Mike Frysingerd84d91e2015-11-05 18:02:24 -050097 raise MissingBoardInstructions(self.board, image_type, input_insns)
Amey Deshpandea936c622015-08-12 17:27:54 -070098 config.readfp(open(input_insns))
Mike Frysingerd84d91e2015-11-05 18:02:24 -050099
100 if image_type is not None:
101 input_insns = self.GetInsnFile(image_type)
102 if not os.path.exists(input_insns):
103 # This type doesn't have any signing instructions.
104 raise MissingBoardInstructions(self.board, image_type, input_insns)
105
106 self.image_type = image_type
107 config.readfp(open(input_insns))
108
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400109 self.cfg = config
110
111 def GetInsnFile(self, image_type):
112 """Find the signer instruction files for this board/image type.
113
114 Args:
115 image_type: The type of instructions to load. It can be a common file
116 (like "DEFAULT"), or one of the --sign-types.
117
118 Returns:
119 Full path to the instruction file using |image_type| and |self.board|.
120 """
121 if image_type == image_type.upper():
122 name = image_type
Amey Deshpandea936c622015-08-12 17:27:54 -0700123 elif image_type in (constants.IMAGE_TYPE_RECOVERY,
124 constants.IMAGE_TYPE_BASE):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400125 name = self.board
126 else:
127 name = '%s.%s' % (self.board, image_type)
128
129 return os.path.join(signing.INPUT_INSN_DIR, '%s.instructions' % name)
130
131 @staticmethod
132 def SplitCfgField(val):
133 """Split a string into multiple elements.
134
135 This centralizes our convention for multiple elements in the input files
136 being delimited by either a space or comma.
137
138 Args:
139 val: The string to split.
140
141 Returns:
142 The list of elements from having done split the string.
143 """
144 return val.replace(',', ' ').split()
145
146 def GetChannels(self):
147 """Return the list of channels to sign for this board.
148
149 If the board-specific config doesn't specify a preference, we'll use the
150 common settings.
151 """
152 return self.SplitCfgField(self.cfg.get('insns', 'channel'))
153
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500154 def GetKeysets(self, insns_merge=None):
155 """Return the list of keysets to sign for this board.
156
157 Args:
158 insns_merge: The additional section to look at over [insns].
159 """
160 # First load the default value from [insns.keyset] if available.
161 sections = ['insns']
162 # Then overlay the [insns.xxx.keyset] if requested.
163 if insns_merge is not None:
164 sections += [insns_merge]
165
166 keyset = ''
167 for section in sections:
168 try:
169 keyset = self.cfg.get(section, 'keyset')
170 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
171 pass
172
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500173 # We do not perturb the order (e.g. using sorted() or making a set())
174 # because we want the behavior stable, and we want the input insns to
175 # explicitly control the order (since it has an impact on naming).
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500176 return self.SplitCfgField(keyset)
177
178 def GetAltInsnSets(self):
179 """Return the list of alternative insn sections."""
180 # We do not perturb the order (e.g. using sorted() or making a set())
181 # because we want the behavior stable, and we want the input insns to
182 # explicitly control the order (since it has an impact on naming).
183 ret = [x for x in self.cfg.sections() if x.startswith('insns.')]
184 return ret if ret else [None]
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400185
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500186 @staticmethod
187 def CopyConfigParser(config):
188 """Return a copy of a ConfigParser object.
189
190 The python guys broke the ability to use something like deepcopy:
191 https://bugs.python.org/issue16058
192 """
193 # Write the current config to a string io object.
194 data = cStringIO.StringIO()
195 config.write(data)
196 data.seek(0)
197
198 # Create a new ConfigParser from the serialized data.
199 ret = ConfigParser.ConfigParser()
200 ret.readfp(data)
201
202 return ret
203
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500204 def OutputInsns(self, output_file, sect_insns, sect_general,
205 insns_merge=None):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400206 """Generate the output instruction file for sending to the signer.
207
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500208 The override order is (later has precedence):
209 [insns]
210 [insns_merge] (should be named "insns.xxx")
211 sect_insns
212
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400213 Note: The format of the instruction file pushimage outputs (and the signer
214 reads) is not exactly the same as the instruction file pushimage reads.
215
216 Args:
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400217 output_file: The file to write the new instruction file to.
218 sect_insns: Items to set/override in the [insns] section.
219 sect_general: Items to set/override in the [general] section.
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500220 insns_merge: The alternative insns.xxx section to merge.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400221 """
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500222 # Create a copy so we can clobber certain fields.
223 config = self.CopyConfigParser(self.cfg)
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500224 sect_insns = sect_insns.copy()
225
226 # Merge in the alternative insns section if need be.
227 if insns_merge is not None:
228 for k, v in config.items(insns_merge):
229 sect_insns.setdefault(k, v)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400230
231 # Clear channel entry in instructions file, ensuring we only get
232 # one channel for the signer to look at. Then provide all the
233 # other details for this signing request to avoid any ambiguity
234 # and to avoid relying on encoding data into filenames.
235 for sect, fields in zip(('insns', 'general'), (sect_insns, sect_general)):
236 if not config.has_section(sect):
237 config.add_section(sect)
238 for k, v in fields.iteritems():
239 config.set(sect, k, v)
240
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500241 # Now prune the alternative sections.
242 for alt in self.GetAltInsnSets():
243 config.remove_section(alt)
244
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400245 output = cStringIO.StringIO()
246 config.write(output)
247 data = output.getvalue()
248 osutils.WriteFile(output_file, data)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500249 logging.debug('generated insns file for %s:\n%s', self.image_type, data)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400250
251
252def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
253 """Mark an instructions file for signing.
254
255 This will upload a file to the GS bucket flagging an image for signing by
256 the signers.
257
258 Args:
259 ctx: A viable gs.GSContext.
260 tbs_base: The full path to where the tobesigned directory lives.
261 insns_path: The path (relative to |tbs_base|) of the file to sign.
262 priority: Set the signing priority (lower == higher prio).
263
264 Returns:
265 The full path to the remote tobesigned file.
266 """
267 if priority < 0 or priority > 99:
268 raise ValueError('priority must be [0, 99] inclusive')
269
270 if insns_path.startswith(tbs_base):
271 insns_path = insns_path[len(tbs_base):].lstrip('/')
272
273 tbs_path = '%s/tobesigned/%02i,%s' % (tbs_base, priority,
274 insns_path.replace('/', ','))
275
Mike Frysinger6430d132014-10-27 23:43:30 -0400276 # The caller will catch gs.GSContextException for us.
277 ctx.Copy('-', tbs_path, input=cros_build_lib.MachineDetails())
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400278
279 return tbs_path
280
281
282def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
Mike Frysingerdad40d62014-02-09 02:18:02 -0500283 sign_types=None, dry_run=False, mock=False, force_keysets=()):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400284 """Push the image from the archive bucket to the release bucket.
285
286 Args:
287 src_path: Where to copy the files from; can be a local path or gs:// URL.
288 Should be a full path to the artifacts in either case.
289 board: The board we're uploading artifacts for (e.g. $BOARD).
290 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
291 profile: The board profile in use (e.g. "asan").
292 priority: Set the signing priority (lower == higher prio).
293 sign_types: If set, a set of types which we'll restrict ourselves to
294 signing. See the --sign-types option for more details.
295 dry_run: Show what would be done, but do not upload anything.
296 mock: Upload to a testing bucket rather than the real one.
Mike Frysingerdad40d62014-02-09 02:18:02 -0500297 force_keysets: Set of keysets to use rather than what the inputs say.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400298
299 Returns:
Don Garrett9459c2f2014-01-22 18:20:24 -0800300 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
301 'gs://signer_instruction_uri2',
302 ...]
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400303 """
Mike Frysinger4495b032014-03-05 17:24:03 -0500304 # Whether we hit an unknown error. If so, we'll throw an error, but only
305 # at the end (so that we still upload as many files as possible).
Amey Deshpandea936c622015-08-12 17:27:54 -0700306 # It's implemented using a list to deal with variable scopes in nested
307 # functions below.
308 unknown_error = [False]
Mike Frysinger4495b032014-03-05 17:24:03 -0500309
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400310 if versionrev is None:
311 # Extract milestone/version from the directory name.
312 versionrev = os.path.basename(src_path)
313
314 # We only support the latest format here. Older releases can use pushimage
315 # from the respective branch which deals with legacy cruft.
316 m = re.match(VERSION_REGEX, versionrev)
317 if not m:
318 raise ValueError('version %s does not match %s' %
319 (versionrev, VERSION_REGEX))
320 milestone = m.group(1)
321 version = m.group(2)
322
323 # Normalize board to always use dashes not underscores. This is mostly a
324 # historical artifact at this point, but we can't really break it since the
325 # value is used in URLs.
326 boardpath = board.replace('_', '-')
327 if profile is not None:
328 boardpath += '-%s' % profile.replace('_', '-')
329
330 ctx = gs.GSContext(dry_run=dry_run)
331
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400332 try:
333 input_insns = InputInsns(board)
334 except MissingBoardInstructions as e:
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500335 logging.warning('Missing base instruction file: %s', e)
Ralph Nathan446aee92015-03-23 14:44:56 -0700336 logging.warning('not uploading anything for signing')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400337 return
338 channels = input_insns.GetChannels()
Mike Frysingerdad40d62014-02-09 02:18:02 -0500339
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500340 # We want force_keysets as a set.
Mike Frysingerdad40d62014-02-09 02:18:02 -0500341 force_keysets = set(force_keysets)
Mike Frysingerdad40d62014-02-09 02:18:02 -0500342
343 if mock:
Ralph Nathan03047282015-03-23 11:09:32 -0700344 logging.info('Upload mode: mock; signers will not process anything')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500345 tbs_base = gs_base = os.path.join(constants.TRASH_BUCKET, 'pushimage-tests',
346 getpass.getuser())
David Rileyf8205122015-09-04 13:46:36 -0700347 elif set(['%s-%s' % (TEST_KEYSET_PREFIX, x)
348 for x in TEST_KEYSETS]) & force_keysets:
Ralph Nathan03047282015-03-23 11:09:32 -0700349 logging.info('Upload mode: test; signers will process test keys')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500350 # We need the tbs_base to be in the place the signer will actually scan.
351 tbs_base = TEST_SIGN_BUCKET_BASE
352 gs_base = os.path.join(tbs_base, getpass.getuser())
353 else:
Ralph Nathan03047282015-03-23 11:09:32 -0700354 logging.info('Upload mode: normal; signers will process the images')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500355 tbs_base = gs_base = constants.RELEASE_BUCKET
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400356
357 sect_general = {
358 'config_board': board,
359 'board': boardpath,
360 'version': version,
361 'versionrev': versionrev,
362 'milestone': milestone,
363 }
364 sect_insns = {}
365
366 if dry_run:
Ralph Nathan03047282015-03-23 11:09:32 -0700367 logging.info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED')
368 logging.info('Signing for channels: %s', ' '.join(channels))
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400369
Don Garrett9459c2f2014-01-22 18:20:24 -0800370 instruction_urls = {}
371
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400372 def _ImageNameBase(image_type=None):
373 lmid = ('%s-' % image_type) if image_type else ''
374 return 'ChromeOS-%s%s-%s' % (lmid, versionrev, boardpath)
375
Amey Deshpandea936c622015-08-12 17:27:54 -0700376 # These variables are defined outside the loop so that the nested functions
377 # below can access them without 'cell-var-from-loop' linter warning.
378 dst_path = ""
379 files_to_sign = []
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400380 for channel in channels:
Ralph Nathan5a582ff2015-03-20 18:18:30 -0700381 logging.debug('\n\n#### CHANNEL: %s ####\n', channel)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400382 sect_insns['channel'] = channel
383 sub_path = '%s-channel/%s/%s' % (channel, boardpath, version)
384 dst_path = '%s/%s' % (gs_base, sub_path)
Ralph Nathan03047282015-03-23 11:09:32 -0700385 logging.info('Copying images to %s', dst_path)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400386
Amey Deshpandea936c622015-08-12 17:27:54 -0700387 recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY)
388 factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY)
389 firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE)
David Rileya04d19d2015-09-04 16:11:50 -0700390 nv_lp0_firmware_basename = _ImageNameBase(
391 constants.IMAGE_TYPE_NV_LP0_FIRMWARE)
Vincent Palatind599c662015-10-26 09:51:41 -0700392 acc_usbpd_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_USBPD)
393 acc_rwsig_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_RWSIG)
Amey Deshpandea936c622015-08-12 17:27:54 -0700394 test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST)
395 base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400396 hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev)
397
Amey Deshpandea936c622015-08-12 17:27:54 -0700398 # The following build artifacts, if present, are always copied regardless of
399 # requested signing types.
400 files_to_copy_only = (
401 # (<src>, <dst>, <suffix>),
402 ('image.zip', _ImageNameBase(), 'zip'),
403 (constants.TEST_IMAGE_TAR, test_basename, 'tar.xz'),
404 ('debug.tgz', 'debug-%s' % boardpath, 'tgz'),
405 (hwqual_tarball, '', ''),
406 ('au-generator.zip', '', ''),
407 ('stateful.tgz', '', ''),
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400408 )
Amey Deshpandea936c622015-08-12 17:27:54 -0700409
410 # The following build artifacts, if present, are always copied.
411 # If |sign_types| is None, all of them are marked for signing, otherwise
412 # only the image types specified in |sign_types| are marked for signing.
413 files_to_copy_and_maybe_sign = (
414 # (<src>, <dst>, <suffix>, <signing type>),
415 (constants.RECOVERY_IMAGE_TAR, recovery_basename, 'tar.xz',
416 constants.IMAGE_TYPE_RECOVERY),
417
418 ('factory_image.zip', factory_basename, 'zip',
419 constants.IMAGE_TYPE_FACTORY),
420
421 ('firmware_from_source.tar.bz2', firmware_basename, 'tar.bz2',
422 constants.IMAGE_TYPE_FIRMWARE),
David Rileya04d19d2015-09-04 16:11:50 -0700423
424 ('firmware_from_source.tar.bz2', nv_lp0_firmware_basename, 'tar.bz2',
425 constants.IMAGE_TYPE_NV_LP0_FIRMWARE),
Vincent Palatind599c662015-10-26 09:51:41 -0700426
427 ('firmware_from_source.tar.bz2', acc_usbpd_basename, 'tar.bz2',
428 constants.IMAGE_TYPE_ACCESSORY_USBPD),
429
430 ('firmware_from_source.tar.bz2', acc_rwsig_basename, 'tar.bz2',
431 constants.IMAGE_TYPE_ACCESSORY_RWSIG),
Amey Deshpandea936c622015-08-12 17:27:54 -0700432 )
433
434 # The following build artifacts are copied and marked for signing, if
435 # they are present *and* if the image type is specified via |sign_types|.
436 files_to_maybe_copy_and_sign = (
437 # (<src>, <dst>, <suffix>, <signing type>),
438 (constants.BASE_IMAGE_TAR, base_basename, 'tar.xz',
439 constants.IMAGE_TYPE_BASE),
440 )
441
442 def _CopyFileToGS(src, dst, suffix):
443 """Returns |dst| file name if the copying was successful."""
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400444 if not dst:
445 dst = src
Amey Deshpandea936c622015-08-12 17:27:54 -0700446 elif suffix:
447 dst = '%s.%s' % (dst, suffix)
448 success = False
Mike Frysingere51a2652014-01-18 02:36:16 -0500449 try:
450 ctx.Copy(os.path.join(src_path, src), os.path.join(dst_path, dst))
Amey Deshpandea936c622015-08-12 17:27:54 -0700451 success = True
Mike Frysingere51a2652014-01-18 02:36:16 -0500452 except gs.GSNoSuchKey:
Ralph Nathan446aee92015-03-23 14:44:56 -0700453 logging.warning('Skipping %s as it does not exist', src)
Mike Frysinger4495b032014-03-05 17:24:03 -0500454 except gs.GSContextException:
Amey Deshpandea936c622015-08-12 17:27:54 -0700455 unknown_error[0] = True
Ralph Nathan59900422015-03-24 10:41:17 -0700456 logging.error('Skipping %s due to unknown GS error', src, exc_info=True)
Amey Deshpandea936c622015-08-12 17:27:54 -0700457 return dst if success else None
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400458
Amey Deshpandea936c622015-08-12 17:27:54 -0700459 for src, dst, suffix in files_to_copy_only:
460 _CopyFileToGS(src, dst, suffix)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400461
Amey Deshpandea936c622015-08-12 17:27:54 -0700462 # Clear the list of files to sign before adding new artifacts.
463 files_to_sign = []
464
465 def _AddToFilesToSign(image_type, dst, suffix):
466 assert dst.endswith('.' + suffix), (
467 'dst: %s, suffix: %s' % (dst, suffix))
468 dst_base = dst[:-(len(suffix) + 1)]
469 files_to_sign.append([image_type, dst_base, suffix])
470
471 for src, dst, suffix, image_type in files_to_copy_and_maybe_sign:
472 dst = _CopyFileToGS(src, dst, suffix)
473 if dst and (not sign_types or image_type in sign_types):
474 _AddToFilesToSign(image_type, dst, suffix)
475
476 for src, dst, suffix, image_type in files_to_maybe_copy_and_sign:
477 if sign_types and image_type in sign_types:
478 dst = _CopyFileToGS(src, dst, suffix)
479 if dst:
480 _AddToFilesToSign(image_type, dst, suffix)
481
482 logging.debug('Files to sign: %s', files_to_sign)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400483 # Now go through the subset for signing.
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500484 for image_type, dst_name, suffix in files_to_sign:
485 try:
486 input_insns = InputInsns(board, image_type=image_type)
487 except MissingBoardInstructions as e:
488 logging.info('Nothing to sign: %s', e)
489 continue
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400490
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500491 dst_archive = '%s.%s' % (dst_name, suffix)
492 sect_general['archive'] = dst_archive
493 sect_general['type'] = image_type
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400494
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500495 # In the default/automatic mode, only flag files for signing if the
496 # archives were actually uploaded in a previous stage. This additional
497 # check can be removed in future once |sign_types| becomes a required
498 # argument.
499 # TODO: Make |sign_types| a required argument.
500 gs_artifact_path = os.path.join(dst_path, dst_archive)
501 exists = False
502 try:
503 exists = ctx.Exists(gs_artifact_path)
504 except gs.GSContextException:
505 unknown_error[0] = True
506 logging.error('Unknown error while checking %s', gs_artifact_path,
507 exc_info=True)
508 if not exists:
509 logging.info('%s does not exist. Nothing to sign.',
510 gs_artifact_path)
511 continue
512
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500513 first_image = True
514 for alt_insn_set in input_insns.GetAltInsnSets():
515 # Figure out which keysets have been requested for this type.
516 # We sort the forced set so tests/runtime behavior is stable.
517 keysets = sorted(force_keysets)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500518 if not keysets:
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500519 keysets = input_insns.GetKeysets(insns_merge=alt_insn_set)
520 if not keysets:
521 logging.warning('Skipping %s image signing due to no keysets',
522 image_type)
Mike Frysingerd84d91e2015-11-05 18:02:24 -0500523
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500524 for keyset in keysets:
525 sect_insns['keyset'] = keyset
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400526
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500527 # Generate the insn file for this artifact that the signer will use,
528 # and flag it for signing.
529 with tempfile.NamedTemporaryFile(
530 bufsize=0, prefix='pushimage.insns.') as insns_path:
531 input_insns.OutputInsns(insns_path.name, sect_insns, sect_general,
532 insns_merge=alt_insn_set)
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400533
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500534 gs_insns_path = '%s/%s' % (dst_path, dst_name)
535 if not first_image:
536 gs_insns_path += '-%s' % keyset
537 first_image = False
538 gs_insns_path += '.instructions'
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400539
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500540 try:
541 ctx.Copy(insns_path.name, gs_insns_path)
542 except gs.GSContextException:
543 unknown_error[0] = True
544 logging.error('Unknown error while uploading insns %s',
545 gs_insns_path, exc_info=True)
546 continue
Mike Frysinger4495b032014-03-05 17:24:03 -0500547
Mike Frysinger37ccc2b2015-11-11 17:16:51 -0500548 try:
549 MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
550 except gs.GSContextException:
551 unknown_error[0] = True
552 logging.error('Unknown error while marking for signing %s',
553 gs_insns_path, exc_info=True)
554 continue
555 logging.info('Signing %s image with keyset %s at %s', image_type,
556 keyset, gs_insns_path)
557 instruction_urls.setdefault(channel, []).append(gs_insns_path)
Don Garrett9459c2f2014-01-22 18:20:24 -0800558
Amey Deshpandea936c622015-08-12 17:27:54 -0700559 if unknown_error[0]:
Mike Frysinger4495b032014-03-05 17:24:03 -0500560 raise PushError('hit some unknown error(s)', instruction_urls)
561
Don Garrett9459c2f2014-01-22 18:20:24 -0800562 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400563
564
565def main(argv):
566 parser = commandline.ArgumentParser(description=__doc__)
567
568 # The type of image_dir will strip off trailing slashes (makes later
569 # processing simpler and the display prettier).
570 parser.add_argument('image_dir', default=None, type='local_or_gs_path',
571 help='full path of source artifacts to upload')
572 parser.add_argument('--board', default=None, required=True,
573 help='board to generate symbols for')
574 parser.add_argument('--profile', default=None,
575 help='board profile in use (e.g. "asan")')
576 parser.add_argument('--version', default=None,
577 help='version info (normally extracted from image_dir)')
578 parser.add_argument('-n', '--dry-run', default=False, action='store_true',
579 help='show what would be done, but do not upload')
580 parser.add_argument('-M', '--mock', default=False, action='store_true',
581 help='upload things to a testing bucket (dev testing)')
David Rileyf8205122015-09-04 13:46:36 -0700582 parser.add_argument('--test-sign', default=[], action='append',
583 choices=TEST_KEYSETS,
584 help='mung signing behavior to sign w/ test keys')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400585 parser.add_argument('--priority', type=int, default=50,
586 help='set signing priority (lower == higher prio)')
587 parser.add_argument('--sign-types', default=None, nargs='+',
Amey Deshpandea936c622015-08-12 17:27:54 -0700588 choices=_SUPPORTED_IMAGE_TYPES,
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400589 help='only sign specified image types')
Mike Frysinger09fe0122014-02-09 02:44:05 -0500590 parser.add_argument('--yes', action='store_true', default=False,
591 help='answer yes to all prompts')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400592
593 opts = parser.parse_args(argv)
594 opts.Freeze()
595
David Rileyf8205122015-09-04 13:46:36 -0700596 force_keysets = set(['%s-%s' % (TEST_KEYSET_PREFIX, x)
597 for x in opts.test_sign])
Mike Frysingerdad40d62014-02-09 02:18:02 -0500598
Mike Frysinger09fe0122014-02-09 02:44:05 -0500599 # If we aren't using mock or test or dry run mode, then let's prompt the user
600 # to make sure they actually want to do this. It's rare that people want to
601 # run this directly and hit the release bucket.
602 if not (opts.mock or force_keysets or opts.dry_run) and not opts.yes:
603 prolog = '\n'.join(textwrap.wrap(textwrap.dedent(
604 'Uploading images for signing to the *release* bucket is not something '
605 'you generally should be doing yourself.'), 80)).strip()
606 if not cros_build_lib.BooleanPrompt(
607 prompt='Are you sure you want to sign these images',
608 default=False, prolog=prolog):
609 cros_build_lib.Die('better safe than sorry')
610
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400611 PushImage(opts.image_dir, opts.board, versionrev=opts.version,
612 profile=opts.profile, priority=opts.priority,
Mike Frysingerdad40d62014-02-09 02:18:02 -0500613 sign_types=opts.sign_types, dry_run=opts.dry_run, mock=opts.mock,
614 force_keysets=force_keysets)