blob: f904f4d80db23121dbbddce385a909a765589928 [file] [log] [blame]
Mike Frysingerd13faeb2013-09-05 16:00:46 -04001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""ChromeOS image pusher (from cbuildbot to signer).
6
7This pushes files from the archive bucket to the signer bucket and marks
8artifacts for signing (which a signing process will look for).
9"""
10
Mike Frysinger1d4752b2014-11-08 04:00:18 -050011# pylint: disable=bad-whitespace
12
Mike Frysingerd13faeb2013-09-05 16:00:46 -040013from __future__ import print_function
14
15import ConfigParser
16import cStringIO
17import errno
18import getpass
19import os
20import re
21import tempfile
Mike Frysinger09fe0122014-02-09 02:44:05 -050022import textwrap
Mike Frysingerd13faeb2013-09-05 16:00:46 -040023
Don Garrett88b8d782014-05-13 17:30:55 -070024from chromite.cbuildbot import constants
Mike Frysingerd13faeb2013-09-05 16:00:46 -040025from chromite.lib import commandline
26from chromite.lib import cros_build_lib
Mike Frysingerd13faeb2013-09-05 16:00:46 -040027from chromite.lib import gs
28from chromite.lib import osutils
29from chromite.lib import signing
30
31
32# This will split a fully qualified ChromeOS version string up.
33# R34-5126.0.0 will break into "34" and "5126.0.0".
34VERSION_REGEX = r'^R([0-9]+)-([^-]+)'
35
Mike Frysingerdad40d62014-02-09 02:18:02 -050036# The test signers will scan this dir looking for test work.
37# Keep it in sync with the signer config files [gs_test_buckets].
38TEST_SIGN_BUCKET_BASE = 'gs://chromeos-throw-away-bucket/signer-tests'
39
40# Ketsets that are only valid in the above test bucket.
41TEST_KEYSETS = set(('test-keys-mp', 'test-keys-premp'))
42
Mike Frysingerd13faeb2013-09-05 16:00:46 -040043
Mike Frysinger4495b032014-03-05 17:24:03 -050044class PushError(Exception):
45 """When an (unknown) error happened while trying to push artifacts."""
46
47
Mike Frysingerd13faeb2013-09-05 16:00:46 -040048class MissingBoardInstructions(Exception):
49 """Raised when a board lacks any signer instructions."""
50
51
52class InputInsns(object):
53 """Object to hold settings for a signable board.
54
55 Note: The format of the instruction file pushimage outputs (and the signer
56 reads) is not exactly the same as the instruction file pushimage reads.
57 """
58
59 def __init__(self, board):
60 self.board = board
61
62 config = ConfigParser.ConfigParser()
63 config.readfp(open(self.GetInsnFile('DEFAULT')))
64 try:
65 input_insn = self.GetInsnFile('recovery')
66 config.readfp(open(input_insn))
67 except IOError as e:
68 if e.errno == errno.ENOENT:
69 # This board doesn't have any signing instructions.
70 # This is normal for new or experimental boards.
71 raise MissingBoardInstructions(input_insn)
72 raise
73 self.cfg = config
74
75 def GetInsnFile(self, image_type):
76 """Find the signer instruction files for this board/image type.
77
78 Args:
79 image_type: The type of instructions to load. It can be a common file
80 (like "DEFAULT"), or one of the --sign-types.
81
82 Returns:
83 Full path to the instruction file using |image_type| and |self.board|.
84 """
85 if image_type == image_type.upper():
86 name = image_type
87 elif image_type == 'recovery':
88 name = self.board
89 else:
90 name = '%s.%s' % (self.board, image_type)
91
92 return os.path.join(signing.INPUT_INSN_DIR, '%s.instructions' % name)
93
94 @staticmethod
95 def SplitCfgField(val):
96 """Split a string into multiple elements.
97
98 This centralizes our convention for multiple elements in the input files
99 being delimited by either a space or comma.
100
101 Args:
102 val: The string to split.
103
104 Returns:
105 The list of elements from having done split the string.
106 """
107 return val.replace(',', ' ').split()
108
109 def GetChannels(self):
110 """Return the list of channels to sign for this board.
111
112 If the board-specific config doesn't specify a preference, we'll use the
113 common settings.
114 """
115 return self.SplitCfgField(self.cfg.get('insns', 'channel'))
116
117 def GetKeysets(self):
118 """Return the list of keysets to sign for this board."""
119 return self.SplitCfgField(self.cfg.get('insns', 'keyset'))
120
121 def OutputInsns(self, image_type, output_file, sect_insns, sect_general):
122 """Generate the output instruction file for sending to the signer.
123
124 Note: The format of the instruction file pushimage outputs (and the signer
125 reads) is not exactly the same as the instruction file pushimage reads.
126
127 Args:
128 image_type: The type of image we will be signing (see --sign-types).
129 output_file: The file to write the new instruction file to.
130 sect_insns: Items to set/override in the [insns] section.
131 sect_general: Items to set/override in the [general] section.
132 """
133 config = ConfigParser.ConfigParser()
134 config.readfp(open(self.GetInsnFile(image_type)))
135
136 # Clear channel entry in instructions file, ensuring we only get
137 # one channel for the signer to look at. Then provide all the
138 # other details for this signing request to avoid any ambiguity
139 # and to avoid relying on encoding data into filenames.
140 for sect, fields in zip(('insns', 'general'), (sect_insns, sect_general)):
141 if not config.has_section(sect):
142 config.add_section(sect)
143 for k, v in fields.iteritems():
144 config.set(sect, k, v)
145
146 output = cStringIO.StringIO()
147 config.write(output)
148 data = output.getvalue()
149 osutils.WriteFile(output_file, data)
150 cros_build_lib.Debug('generated insns file for %s:\n%s', image_type, data)
151
152
153def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority):
154 """Mark an instructions file for signing.
155
156 This will upload a file to the GS bucket flagging an image for signing by
157 the signers.
158
159 Args:
160 ctx: A viable gs.GSContext.
161 tbs_base: The full path to where the tobesigned directory lives.
162 insns_path: The path (relative to |tbs_base|) of the file to sign.
163 priority: Set the signing priority (lower == higher prio).
164
165 Returns:
166 The full path to the remote tobesigned file.
167 """
168 if priority < 0 or priority > 99:
169 raise ValueError('priority must be [0, 99] inclusive')
170
171 if insns_path.startswith(tbs_base):
172 insns_path = insns_path[len(tbs_base):].lstrip('/')
173
174 tbs_path = '%s/tobesigned/%02i,%s' % (tbs_base, priority,
175 insns_path.replace('/', ','))
176
Mike Frysinger6430d132014-10-27 23:43:30 -0400177 # The caller will catch gs.GSContextException for us.
178 ctx.Copy('-', tbs_path, input=cros_build_lib.MachineDetails())
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400179
180 return tbs_path
181
182
183def PushImage(src_path, board, versionrev=None, profile=None, priority=50,
Mike Frysingerdad40d62014-02-09 02:18:02 -0500184 sign_types=None, dry_run=False, mock=False, force_keysets=()):
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400185 """Push the image from the archive bucket to the release bucket.
186
187 Args:
188 src_path: Where to copy the files from; can be a local path or gs:// URL.
189 Should be a full path to the artifacts in either case.
190 board: The board we're uploading artifacts for (e.g. $BOARD).
191 versionrev: The full Chromium OS version string (e.g. R34-5126.0.0).
192 profile: The board profile in use (e.g. "asan").
193 priority: Set the signing priority (lower == higher prio).
194 sign_types: If set, a set of types which we'll restrict ourselves to
195 signing. See the --sign-types option for more details.
196 dry_run: Show what would be done, but do not upload anything.
197 mock: Upload to a testing bucket rather than the real one.
Mike Frysingerdad40d62014-02-09 02:18:02 -0500198 force_keysets: Set of keysets to use rather than what the inputs say.
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400199
200 Returns:
Don Garrett9459c2f2014-01-22 18:20:24 -0800201 A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1',
202 'gs://signer_instruction_uri2',
203 ...]
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400204 """
Mike Frysinger4495b032014-03-05 17:24:03 -0500205 # Whether we hit an unknown error. If so, we'll throw an error, but only
206 # at the end (so that we still upload as many files as possible).
207 unknown_error = False
208
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400209 if versionrev is None:
210 # Extract milestone/version from the directory name.
211 versionrev = os.path.basename(src_path)
212
213 # We only support the latest format here. Older releases can use pushimage
214 # from the respective branch which deals with legacy cruft.
215 m = re.match(VERSION_REGEX, versionrev)
216 if not m:
217 raise ValueError('version %s does not match %s' %
218 (versionrev, VERSION_REGEX))
219 milestone = m.group(1)
220 version = m.group(2)
221
222 # Normalize board to always use dashes not underscores. This is mostly a
223 # historical artifact at this point, but we can't really break it since the
224 # value is used in URLs.
225 boardpath = board.replace('_', '-')
226 if profile is not None:
227 boardpath += '-%s' % profile.replace('_', '-')
228
229 ctx = gs.GSContext(dry_run=dry_run)
230
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400231 try:
232 input_insns = InputInsns(board)
233 except MissingBoardInstructions as e:
234 cros_build_lib.Warning('board "%s" is missing base instruction file: %s',
235 board, e)
236 cros_build_lib.Warning('not uploading anything for signing')
237 return
238 channels = input_insns.GetChannels()
Mike Frysingerdad40d62014-02-09 02:18:02 -0500239
240 # We want force_keysets as a set, and keysets as a list.
241 force_keysets = set(force_keysets)
242 keysets = list(force_keysets) if force_keysets else input_insns.GetKeysets()
243
244 if mock:
245 cros_build_lib.Info('Upload mode: mock; signers will not process anything')
246 tbs_base = gs_base = os.path.join(constants.TRASH_BUCKET, 'pushimage-tests',
247 getpass.getuser())
248 elif TEST_KEYSETS & force_keysets:
249 cros_build_lib.Info('Upload mode: test; signers will process test keys')
250 # We need the tbs_base to be in the place the signer will actually scan.
251 tbs_base = TEST_SIGN_BUCKET_BASE
252 gs_base = os.path.join(tbs_base, getpass.getuser())
253 else:
254 cros_build_lib.Info('Upload mode: normal; signers will process the images')
255 tbs_base = gs_base = constants.RELEASE_BUCKET
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400256
257 sect_general = {
258 'config_board': board,
259 'board': boardpath,
260 'version': version,
261 'versionrev': versionrev,
262 'milestone': milestone,
263 }
264 sect_insns = {}
265
266 if dry_run:
267 cros_build_lib.Info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED')
268 cros_build_lib.Info('Signing for channels: %s', ' '.join(channels))
269 cros_build_lib.Info('Signing for keysets : %s', ' '.join(keysets))
270
Don Garrett9459c2f2014-01-22 18:20:24 -0800271 instruction_urls = {}
272
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400273 def _ImageNameBase(image_type=None):
274 lmid = ('%s-' % image_type) if image_type else ''
275 return 'ChromeOS-%s%s-%s' % (lmid, versionrev, boardpath)
276
277 for channel in channels:
278 cros_build_lib.Debug('\n\n#### CHANNEL: %s ####\n', channel)
279 sect_insns['channel'] = channel
280 sub_path = '%s-channel/%s/%s' % (channel, boardpath, version)
281 dst_path = '%s/%s' % (gs_base, sub_path)
282 cros_build_lib.Info('Copying images to %s', dst_path)
283
284 recovery_base = _ImageNameBase('recovery')
285 factory_base = _ImageNameBase('factory')
286 firmware_base = _ImageNameBase('firmware')
287 test_base = _ImageNameBase('test')
288 hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev)
289
290 # Upload all the files first before flagging them for signing.
291 files_to_copy = (
292 # <src> <dst>
293 # <signing type> <sfx>
294 ('recovery_image.tar.xz', recovery_base, 'tar.xz',
295 'recovery'),
296
297 ('factory_image.zip', factory_base, 'zip',
298 'factory'),
299
300 ('firmware_from_source.tar.bz2', firmware_base, 'tar.bz2',
301 'firmware'),
302
303 ('image.zip', _ImageNameBase(), 'zip', ''),
304 ('chromiumos_test_image.tar.xz', test_base, 'tar.xz', ''),
305 ('debug.tgz', 'debug-%s' % boardpath, 'tgz', ''),
306 (hwqual_tarball, '', '', ''),
307 ('au-generator.zip', '', '', ''),
308 )
309 files_to_sign = []
310 for src, dst, sfx, image_type in files_to_copy:
311 if not dst:
312 dst = src
313 elif sfx:
314 dst += '.%s' % sfx
Mike Frysingere51a2652014-01-18 02:36:16 -0500315 try:
316 ctx.Copy(os.path.join(src_path, src), os.path.join(dst_path, dst))
317 except gs.GSNoSuchKey:
318 cros_build_lib.Warning('Skipping %s as it does not exist', src)
319 continue
Mike Frysinger4495b032014-03-05 17:24:03 -0500320 except gs.GSContextException:
321 unknown_error = True
322 cros_build_lib.Error('Skipping %s due to unknown GS error', src,
323 exc_info=True)
324 continue
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400325
326 if image_type:
327 dst_base = dst[:-(len(sfx) + 1)]
328 assert dst == '%s.%s' % (dst_base, sfx)
329 files_to_sign += [[image_type, dst_base, '.%s' % sfx]]
330
331 # Now go through the subset for signing.
332 for keyset in keysets:
333 cros_build_lib.Debug('\n\n#### KEYSET: %s ####\n', keyset)
334 sect_insns['keyset'] = keyset
335 for image_type, dst_name, suffix in files_to_sign:
336 dst_archive = '%s%s' % (dst_name, suffix)
337 sect_general['archive'] = dst_archive
338 sect_general['type'] = image_type
339
340 # See if the caller has requested we only sign certain types.
341 if sign_types:
342 if not image_type in sign_types:
343 cros_build_lib.Info('Skipping %s signing as it was not requested',
344 image_type)
345 continue
346 else:
347 # In the default/automatic mode, only flag files for signing if the
348 # archives were actually uploaded in a previous stage.
349 gs_artifact_path = os.path.join(dst_path, dst_archive)
Mike Frysinger4495b032014-03-05 17:24:03 -0500350 try:
351 exists = ctx.Exists(gs_artifact_path)
352 except gs.GSContextException:
353 unknown_error = True
354 exists = False
355 cros_build_lib.Error('Unknown error while checking %s',
356 gs_artifact_path, exc_info=True)
357 if not exists:
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400358 cros_build_lib.Info('%s does not exist. Nothing to sign.',
359 gs_artifact_path)
360 continue
361
362 input_insn_path = input_insns.GetInsnFile(image_type)
363 if not os.path.exists(input_insn_path):
364 cros_build_lib.Info('%s does not exist. Nothing to sign.',
365 input_insn_path)
366 continue
367
368 # Generate the insn file for this artifact that the signer will use,
369 # and flag it for signing.
370 with tempfile.NamedTemporaryFile(
371 bufsize=0, prefix='pushimage.insns.') as insns_path:
372 input_insns.OutputInsns(image_type, insns_path.name, sect_insns,
373 sect_general)
374
375 gs_insns_path = '%s/%s' % (dst_path, dst_name)
376 if keyset != keysets[0]:
377 gs_insns_path += '-%s' % keyset
378 gs_insns_path += '.instructions'
379
Mike Frysinger4495b032014-03-05 17:24:03 -0500380 try:
381 ctx.Copy(insns_path.name, gs_insns_path)
382 except gs.GSContextException:
383 unknown_error = True
384 cros_build_lib.Error('Unknown error while uploading insns %s',
385 gs_insns_path, exc_info=True)
386 continue
387
388 try:
389 MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority)
390 except gs.GSContextException:
391 unknown_error = True
392 cros_build_lib.Error('Unknown error while marking for signing %s',
393 gs_insns_path, exc_info=True)
394 continue
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400395 cros_build_lib.Info('Signing %s image %s', image_type, gs_insns_path)
Don Garrett9459c2f2014-01-22 18:20:24 -0800396 instruction_urls.setdefault(channel, []).append(gs_insns_path)
397
Mike Frysinger4495b032014-03-05 17:24:03 -0500398 if unknown_error:
399 raise PushError('hit some unknown error(s)', instruction_urls)
400
Don Garrett9459c2f2014-01-22 18:20:24 -0800401 return instruction_urls
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400402
403
404def main(argv):
405 parser = commandline.ArgumentParser(description=__doc__)
406
407 # The type of image_dir will strip off trailing slashes (makes later
408 # processing simpler and the display prettier).
409 parser.add_argument('image_dir', default=None, type='local_or_gs_path',
410 help='full path of source artifacts to upload')
411 parser.add_argument('--board', default=None, required=True,
412 help='board to generate symbols for')
413 parser.add_argument('--profile', default=None,
414 help='board profile in use (e.g. "asan")')
415 parser.add_argument('--version', default=None,
416 help='version info (normally extracted from image_dir)')
417 parser.add_argument('-n', '--dry-run', default=False, action='store_true',
418 help='show what would be done, but do not upload')
419 parser.add_argument('-M', '--mock', default=False, action='store_true',
420 help='upload things to a testing bucket (dev testing)')
Mike Frysingerdad40d62014-02-09 02:18:02 -0500421 parser.add_argument('--test-sign-mp', default=False, action='store_true',
422 help='mung signing behavior to sign w/test mp keys')
423 parser.add_argument('--test-sign-premp', default=False, action='store_true',
424 help='mung signing behavior to sign w/test premp keys')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400425 parser.add_argument('--priority', type=int, default=50,
426 help='set signing priority (lower == higher prio)')
427 parser.add_argument('--sign-types', default=None, nargs='+',
428 choices=('recovery', 'factory', 'firmware'),
429 help='only sign specified image types')
Mike Frysinger09fe0122014-02-09 02:44:05 -0500430 parser.add_argument('--yes', action='store_true', default=False,
431 help='answer yes to all prompts')
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400432
433 opts = parser.parse_args(argv)
434 opts.Freeze()
435
Mike Frysingerdad40d62014-02-09 02:18:02 -0500436 force_keysets = set()
437 if opts.test_sign_mp:
438 force_keysets.add('test-keys-mp')
439 if opts.test_sign_premp:
440 force_keysets.add('test-keys-premp')
441
Mike Frysinger09fe0122014-02-09 02:44:05 -0500442 # If we aren't using mock or test or dry run mode, then let's prompt the user
443 # to make sure they actually want to do this. It's rare that people want to
444 # run this directly and hit the release bucket.
445 if not (opts.mock or force_keysets or opts.dry_run) and not opts.yes:
446 prolog = '\n'.join(textwrap.wrap(textwrap.dedent(
447 'Uploading images for signing to the *release* bucket is not something '
448 'you generally should be doing yourself.'), 80)).strip()
449 if not cros_build_lib.BooleanPrompt(
450 prompt='Are you sure you want to sign these images',
451 default=False, prolog=prolog):
452 cros_build_lib.Die('better safe than sorry')
453
Mike Frysingerd13faeb2013-09-05 16:00:46 -0400454 PushImage(opts.image_dir, opts.board, versionrev=opts.version,
455 profile=opts.profile, priority=opts.priority,
Mike Frysingerdad40d62014-02-09 02:18:02 -0500456 sign_types=opts.sign_types, dry_run=opts.dry_run, mock=opts.mock,
457 force_keysets=force_keysets)