Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """ChromeOS image pusher (from cbuildbot to signer). |
| 6 | |
| 7 | This pushes files from the archive bucket to the signer bucket and marks |
| 8 | artifacts for signing (which a signing process will look for). |
| 9 | """ |
| 10 | |
| 11 | from __future__ import print_function |
| 12 | |
| 13 | import ConfigParser |
| 14 | import cStringIO |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 15 | import getpass |
| 16 | import os |
| 17 | import re |
| 18 | import tempfile |
Mike Frysinger | 09fe012 | 2014-02-09 02:44:05 -0500 | [diff] [blame] | 19 | import textwrap |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 20 | |
Aviv Keshet | b7519e1 | 2016-10-04 00:50:00 -0700 | [diff] [blame] | 21 | from chromite.lib import constants |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 22 | from chromite.lib import commandline |
| 23 | from chromite.lib import cros_build_lib |
Ralph Nathan | 5a582ff | 2015-03-20 18:18:30 -0700 | [diff] [blame] | 24 | from chromite.lib import cros_logging as logging |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 25 | from chromite.lib import gs |
| 26 | from chromite.lib import osutils |
| 27 | from chromite.lib import signing |
| 28 | |
| 29 | |
| 30 | # This will split a fully qualified ChromeOS version string up. |
| 31 | # R34-5126.0.0 will break into "34" and "5126.0.0". |
| 32 | VERSION_REGEX = r'^R([0-9]+)-([^-]+)' |
| 33 | |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 34 | # The test signers will scan this dir looking for test work. |
| 35 | # Keep it in sync with the signer config files [gs_test_buckets]. |
| 36 | TEST_SIGN_BUCKET_BASE = 'gs://chromeos-throw-away-bucket/signer-tests' |
| 37 | |
David Riley | f820512 | 2015-09-04 13:46:36 -0700 | [diff] [blame] | 38 | # Keysets that are only valid in the above test bucket. |
| 39 | TEST_KEYSET_PREFIX = 'test-keys' |
| 40 | TEST_KEYSETS = set(( |
| 41 | 'mp', |
| 42 | 'premp', |
| 43 | 'nvidia-premp', |
| 44 | )) |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 45 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 46 | # Supported image types for signing. |
| 47 | _SUPPORTED_IMAGE_TYPES = ( |
| 48 | constants.IMAGE_TYPE_RECOVERY, |
| 49 | constants.IMAGE_TYPE_FACTORY, |
| 50 | constants.IMAGE_TYPE_FIRMWARE, |
David Riley | a04d19d | 2015-09-04 16:11:50 -0700 | [diff] [blame] | 51 | constants.IMAGE_TYPE_NV_LP0_FIRMWARE, |
Vincent Palatin | d599c66 | 2015-10-26 09:51:41 -0700 | [diff] [blame] | 52 | constants.IMAGE_TYPE_ACCESSORY_USBPD, |
| 53 | constants.IMAGE_TYPE_ACCESSORY_RWSIG, |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 54 | constants.IMAGE_TYPE_BASE, |
| 55 | ) |
| 56 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 57 | |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 58 | class PushError(Exception): |
| 59 | """When an (unknown) error happened while trying to push artifacts.""" |
| 60 | |
| 61 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 62 | class MissingBoardInstructions(Exception): |
| 63 | """Raised when a board lacks any signer instructions.""" |
| 64 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 65 | def __init__(self, board, image_type, input_insns): |
| 66 | Exception.__init__(self, 'Board %s lacks insns for %s image: %s not found' % |
| 67 | (board, image_type, input_insns)) |
| 68 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 69 | |
| 70 | class InputInsns(object): |
| 71 | """Object to hold settings for a signable board. |
| 72 | |
| 73 | Note: The format of the instruction file pushimage outputs (and the signer |
| 74 | reads) is not exactly the same as the instruction file pushimage reads. |
| 75 | """ |
| 76 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 77 | def __init__(self, board, image_type=None): |
| 78 | """Initialization. |
| 79 | |
| 80 | Args: |
| 81 | board: The board to look up details. |
| 82 | image_type: The type of image we will be signing (see --sign-types). |
| 83 | """ |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 84 | self.board = board |
| 85 | |
| 86 | config = ConfigParser.ConfigParser() |
| 87 | config.readfp(open(self.GetInsnFile('DEFAULT'))) |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 88 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 89 | # What pushimage internally refers to as 'recovery', are the basic signing |
| 90 | # instructions in practice, and other types are stacked on top. |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 91 | if image_type is None: |
| 92 | image_type = constants.IMAGE_TYPE_RECOVERY |
| 93 | self.image_type = image_type |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 94 | input_insns = self.GetInsnFile(constants.IMAGE_TYPE_RECOVERY) |
| 95 | if not os.path.exists(input_insns): |
| 96 | # This board doesn't have any signing instructions. |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 97 | raise MissingBoardInstructions(self.board, image_type, input_insns) |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 98 | config.readfp(open(input_insns)) |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 99 | |
| 100 | if image_type is not None: |
| 101 | input_insns = self.GetInsnFile(image_type) |
| 102 | if not os.path.exists(input_insns): |
| 103 | # This type doesn't have any signing instructions. |
| 104 | raise MissingBoardInstructions(self.board, image_type, input_insns) |
| 105 | |
| 106 | self.image_type = image_type |
| 107 | config.readfp(open(input_insns)) |
| 108 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 109 | self.cfg = config |
| 110 | |
| 111 | def GetInsnFile(self, image_type): |
| 112 | """Find the signer instruction files for this board/image type. |
| 113 | |
| 114 | Args: |
| 115 | image_type: The type of instructions to load. It can be a common file |
| 116 | (like "DEFAULT"), or one of the --sign-types. |
| 117 | |
| 118 | Returns: |
| 119 | Full path to the instruction file using |image_type| and |self.board|. |
| 120 | """ |
| 121 | if image_type == image_type.upper(): |
| 122 | name = image_type |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 123 | elif image_type in (constants.IMAGE_TYPE_RECOVERY, |
| 124 | constants.IMAGE_TYPE_BASE): |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 125 | name = self.board |
| 126 | else: |
| 127 | name = '%s.%s' % (self.board, image_type) |
| 128 | |
| 129 | return os.path.join(signing.INPUT_INSN_DIR, '%s.instructions' % name) |
| 130 | |
| 131 | @staticmethod |
| 132 | def SplitCfgField(val): |
| 133 | """Split a string into multiple elements. |
| 134 | |
| 135 | This centralizes our convention for multiple elements in the input files |
| 136 | being delimited by either a space or comma. |
| 137 | |
| 138 | Args: |
| 139 | val: The string to split. |
| 140 | |
| 141 | Returns: |
| 142 | The list of elements from having done split the string. |
| 143 | """ |
| 144 | return val.replace(',', ' ').split() |
| 145 | |
| 146 | def GetChannels(self): |
| 147 | """Return the list of channels to sign for this board. |
| 148 | |
| 149 | If the board-specific config doesn't specify a preference, we'll use the |
| 150 | common settings. |
| 151 | """ |
| 152 | return self.SplitCfgField(self.cfg.get('insns', 'channel')) |
| 153 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 154 | def GetKeysets(self, insns_merge=None): |
| 155 | """Return the list of keysets to sign for this board. |
| 156 | |
| 157 | Args: |
| 158 | insns_merge: The additional section to look at over [insns]. |
| 159 | """ |
| 160 | # First load the default value from [insns.keyset] if available. |
| 161 | sections = ['insns'] |
| 162 | # Then overlay the [insns.xxx.keyset] if requested. |
| 163 | if insns_merge is not None: |
| 164 | sections += [insns_merge] |
| 165 | |
| 166 | keyset = '' |
| 167 | for section in sections: |
| 168 | try: |
| 169 | keyset = self.cfg.get(section, 'keyset') |
| 170 | except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): |
| 171 | pass |
| 172 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 173 | # We do not perturb the order (e.g. using sorted() or making a set()) |
| 174 | # because we want the behavior stable, and we want the input insns to |
| 175 | # explicitly control the order (since it has an impact on naming). |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 176 | return self.SplitCfgField(keyset) |
| 177 | |
| 178 | def GetAltInsnSets(self): |
| 179 | """Return the list of alternative insn sections.""" |
| 180 | # We do not perturb the order (e.g. using sorted() or making a set()) |
| 181 | # because we want the behavior stable, and we want the input insns to |
| 182 | # explicitly control the order (since it has an impact on naming). |
| 183 | ret = [x for x in self.cfg.sections() if x.startswith('insns.')] |
| 184 | return ret if ret else [None] |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 185 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 186 | @staticmethod |
| 187 | def CopyConfigParser(config): |
| 188 | """Return a copy of a ConfigParser object. |
| 189 | |
Thiemo Nagel | 9fb9972 | 2017-05-26 16:26:40 +0200 | [diff] [blame] | 190 | The python folks broke the ability to use something like deepcopy: |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 191 | https://bugs.python.org/issue16058 |
| 192 | """ |
| 193 | # Write the current config to a string io object. |
| 194 | data = cStringIO.StringIO() |
| 195 | config.write(data) |
| 196 | data.seek(0) |
| 197 | |
| 198 | # Create a new ConfigParser from the serialized data. |
| 199 | ret = ConfigParser.ConfigParser() |
| 200 | ret.readfp(data) |
| 201 | |
| 202 | return ret |
| 203 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 204 | def OutputInsns(self, output_file, sect_insns, sect_general, |
| 205 | insns_merge=None): |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 206 | """Generate the output instruction file for sending to the signer. |
| 207 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 208 | The override order is (later has precedence): |
| 209 | [insns] |
| 210 | [insns_merge] (should be named "insns.xxx") |
| 211 | sect_insns |
| 212 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 213 | Note: The format of the instruction file pushimage outputs (and the signer |
| 214 | reads) is not exactly the same as the instruction file pushimage reads. |
| 215 | |
| 216 | Args: |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 217 | output_file: The file to write the new instruction file to. |
| 218 | sect_insns: Items to set/override in the [insns] section. |
| 219 | sect_general: Items to set/override in the [general] section. |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 220 | insns_merge: The alternative insns.xxx section to merge. |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 221 | """ |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 222 | # Create a copy so we can clobber certain fields. |
| 223 | config = self.CopyConfigParser(self.cfg) |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 224 | sect_insns = sect_insns.copy() |
| 225 | |
| 226 | # Merge in the alternative insns section if need be. |
| 227 | if insns_merge is not None: |
| 228 | for k, v in config.items(insns_merge): |
| 229 | sect_insns.setdefault(k, v) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 230 | |
| 231 | # Clear channel entry in instructions file, ensuring we only get |
| 232 | # one channel for the signer to look at. Then provide all the |
| 233 | # other details for this signing request to avoid any ambiguity |
| 234 | # and to avoid relying on encoding data into filenames. |
| 235 | for sect, fields in zip(('insns', 'general'), (sect_insns, sect_general)): |
| 236 | if not config.has_section(sect): |
| 237 | config.add_section(sect) |
| 238 | for k, v in fields.iteritems(): |
| 239 | config.set(sect, k, v) |
| 240 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 241 | # Now prune the alternative sections. |
| 242 | for alt in self.GetAltInsnSets(): |
| 243 | config.remove_section(alt) |
| 244 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 245 | output = cStringIO.StringIO() |
| 246 | config.write(output) |
| 247 | data = output.getvalue() |
| 248 | osutils.WriteFile(output_file, data) |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 249 | logging.debug('generated insns file for %s:\n%s', self.image_type, data) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 250 | |
| 251 | |
| 252 | def MarkImageToBeSigned(ctx, tbs_base, insns_path, priority): |
| 253 | """Mark an instructions file for signing. |
| 254 | |
| 255 | This will upload a file to the GS bucket flagging an image for signing by |
| 256 | the signers. |
| 257 | |
| 258 | Args: |
| 259 | ctx: A viable gs.GSContext. |
| 260 | tbs_base: The full path to where the tobesigned directory lives. |
| 261 | insns_path: The path (relative to |tbs_base|) of the file to sign. |
| 262 | priority: Set the signing priority (lower == higher prio). |
| 263 | |
| 264 | Returns: |
| 265 | The full path to the remote tobesigned file. |
| 266 | """ |
| 267 | if priority < 0 or priority > 99: |
| 268 | raise ValueError('priority must be [0, 99] inclusive') |
| 269 | |
| 270 | if insns_path.startswith(tbs_base): |
| 271 | insns_path = insns_path[len(tbs_base):].lstrip('/') |
| 272 | |
| 273 | tbs_path = '%s/tobesigned/%02i,%s' % (tbs_base, priority, |
| 274 | insns_path.replace('/', ',')) |
| 275 | |
Mike Frysinger | 6430d13 | 2014-10-27 23:43:30 -0400 | [diff] [blame] | 276 | # The caller will catch gs.GSContextException for us. |
| 277 | ctx.Copy('-', tbs_path, input=cros_build_lib.MachineDetails()) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 278 | |
| 279 | return tbs_path |
| 280 | |
| 281 | |
| 282 | def PushImage(src_path, board, versionrev=None, profile=None, priority=50, |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 283 | sign_types=None, dry_run=False, mock=False, force_keysets=()): |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 284 | """Push the image from the archive bucket to the release bucket. |
| 285 | |
| 286 | Args: |
| 287 | src_path: Where to copy the files from; can be a local path or gs:// URL. |
| 288 | Should be a full path to the artifacts in either case. |
| 289 | board: The board we're uploading artifacts for (e.g. $BOARD). |
| 290 | versionrev: The full Chromium OS version string (e.g. R34-5126.0.0). |
| 291 | profile: The board profile in use (e.g. "asan"). |
| 292 | priority: Set the signing priority (lower == higher prio). |
| 293 | sign_types: If set, a set of types which we'll restrict ourselves to |
| 294 | signing. See the --sign-types option for more details. |
| 295 | dry_run: Show what would be done, but do not upload anything. |
| 296 | mock: Upload to a testing bucket rather than the real one. |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 297 | force_keysets: Set of keysets to use rather than what the inputs say. |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 298 | |
| 299 | Returns: |
Don Garrett | 9459c2f | 2014-01-22 18:20:24 -0800 | [diff] [blame] | 300 | A dictionary that maps 'channel' -> ['gs://signer_instruction_uri1', |
| 301 | 'gs://signer_instruction_uri2', |
| 302 | ...] |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 303 | """ |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 304 | # Whether we hit an unknown error. If so, we'll throw an error, but only |
| 305 | # at the end (so that we still upload as many files as possible). |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 306 | # It's implemented using a list to deal with variable scopes in nested |
| 307 | # functions below. |
| 308 | unknown_error = [False] |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 309 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 310 | if versionrev is None: |
| 311 | # Extract milestone/version from the directory name. |
| 312 | versionrev = os.path.basename(src_path) |
| 313 | |
| 314 | # We only support the latest format here. Older releases can use pushimage |
| 315 | # from the respective branch which deals with legacy cruft. |
| 316 | m = re.match(VERSION_REGEX, versionrev) |
| 317 | if not m: |
| 318 | raise ValueError('version %s does not match %s' % |
| 319 | (versionrev, VERSION_REGEX)) |
| 320 | milestone = m.group(1) |
| 321 | version = m.group(2) |
| 322 | |
| 323 | # Normalize board to always use dashes not underscores. This is mostly a |
| 324 | # historical artifact at this point, but we can't really break it since the |
| 325 | # value is used in URLs. |
| 326 | boardpath = board.replace('_', '-') |
| 327 | if profile is not None: |
| 328 | boardpath += '-%s' % profile.replace('_', '-') |
| 329 | |
| 330 | ctx = gs.GSContext(dry_run=dry_run) |
| 331 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 332 | try: |
| 333 | input_insns = InputInsns(board) |
| 334 | except MissingBoardInstructions as e: |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 335 | logging.warning('Missing base instruction file: %s', e) |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 336 | logging.warning('not uploading anything for signing') |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 337 | return |
| 338 | channels = input_insns.GetChannels() |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 339 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 340 | # We want force_keysets as a set. |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 341 | force_keysets = set(force_keysets) |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 342 | |
| 343 | if mock: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 344 | logging.info('Upload mode: mock; signers will not process anything') |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 345 | tbs_base = gs_base = os.path.join(constants.TRASH_BUCKET, 'pushimage-tests', |
| 346 | getpass.getuser()) |
David Riley | f820512 | 2015-09-04 13:46:36 -0700 | [diff] [blame] | 347 | elif set(['%s-%s' % (TEST_KEYSET_PREFIX, x) |
| 348 | for x in TEST_KEYSETS]) & force_keysets: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 349 | logging.info('Upload mode: test; signers will process test keys') |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 350 | # We need the tbs_base to be in the place the signer will actually scan. |
| 351 | tbs_base = TEST_SIGN_BUCKET_BASE |
| 352 | gs_base = os.path.join(tbs_base, getpass.getuser()) |
| 353 | else: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 354 | logging.info('Upload mode: normal; signers will process the images') |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 355 | tbs_base = gs_base = constants.RELEASE_BUCKET |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 356 | |
| 357 | sect_general = { |
| 358 | 'config_board': board, |
| 359 | 'board': boardpath, |
| 360 | 'version': version, |
| 361 | 'versionrev': versionrev, |
| 362 | 'milestone': milestone, |
| 363 | } |
| 364 | sect_insns = {} |
| 365 | |
| 366 | if dry_run: |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 367 | logging.info('DRY RUN MODE ACTIVE: NOTHING WILL BE UPLOADED') |
| 368 | logging.info('Signing for channels: %s', ' '.join(channels)) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 369 | |
Don Garrett | 9459c2f | 2014-01-22 18:20:24 -0800 | [diff] [blame] | 370 | instruction_urls = {} |
| 371 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 372 | def _ImageNameBase(image_type=None): |
| 373 | lmid = ('%s-' % image_type) if image_type else '' |
| 374 | return 'ChromeOS-%s%s-%s' % (lmid, versionrev, boardpath) |
| 375 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 376 | # These variables are defined outside the loop so that the nested functions |
| 377 | # below can access them without 'cell-var-from-loop' linter warning. |
| 378 | dst_path = "" |
| 379 | files_to_sign = [] |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 380 | for channel in channels: |
Ralph Nathan | 5a582ff | 2015-03-20 18:18:30 -0700 | [diff] [blame] | 381 | logging.debug('\n\n#### CHANNEL: %s ####\n', channel) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 382 | sect_insns['channel'] = channel |
| 383 | sub_path = '%s-channel/%s/%s' % (channel, boardpath, version) |
| 384 | dst_path = '%s/%s' % (gs_base, sub_path) |
Ralph Nathan | 0304728 | 2015-03-23 11:09:32 -0700 | [diff] [blame] | 385 | logging.info('Copying images to %s', dst_path) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 386 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 387 | recovery_basename = _ImageNameBase(constants.IMAGE_TYPE_RECOVERY) |
| 388 | factory_basename = _ImageNameBase(constants.IMAGE_TYPE_FACTORY) |
| 389 | firmware_basename = _ImageNameBase(constants.IMAGE_TYPE_FIRMWARE) |
David Riley | a04d19d | 2015-09-04 16:11:50 -0700 | [diff] [blame] | 390 | nv_lp0_firmware_basename = _ImageNameBase( |
| 391 | constants.IMAGE_TYPE_NV_LP0_FIRMWARE) |
Vincent Palatin | d599c66 | 2015-10-26 09:51:41 -0700 | [diff] [blame] | 392 | acc_usbpd_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_USBPD) |
| 393 | acc_rwsig_basename = _ImageNameBase(constants.IMAGE_TYPE_ACCESSORY_RWSIG) |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 394 | test_basename = _ImageNameBase(constants.IMAGE_TYPE_TEST) |
| 395 | base_basename = _ImageNameBase(constants.IMAGE_TYPE_BASE) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 396 | hwqual_tarball = 'chromeos-hwqual-%s-%s.tar.bz2' % (board, versionrev) |
| 397 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 398 | # The following build artifacts, if present, are always copied regardless of |
| 399 | # requested signing types. |
| 400 | files_to_copy_only = ( |
| 401 | # (<src>, <dst>, <suffix>), |
| 402 | ('image.zip', _ImageNameBase(), 'zip'), |
| 403 | (constants.TEST_IMAGE_TAR, test_basename, 'tar.xz'), |
| 404 | ('debug.tgz', 'debug-%s' % boardpath, 'tgz'), |
| 405 | (hwqual_tarball, '', ''), |
| 406 | ('au-generator.zip', '', ''), |
| 407 | ('stateful.tgz', '', ''), |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 408 | ) |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 409 | |
| 410 | # The following build artifacts, if present, are always copied. |
| 411 | # If |sign_types| is None, all of them are marked for signing, otherwise |
| 412 | # only the image types specified in |sign_types| are marked for signing. |
| 413 | files_to_copy_and_maybe_sign = ( |
| 414 | # (<src>, <dst>, <suffix>, <signing type>), |
| 415 | (constants.RECOVERY_IMAGE_TAR, recovery_basename, 'tar.xz', |
| 416 | constants.IMAGE_TYPE_RECOVERY), |
| 417 | |
| 418 | ('factory_image.zip', factory_basename, 'zip', |
| 419 | constants.IMAGE_TYPE_FACTORY), |
| 420 | |
| 421 | ('firmware_from_source.tar.bz2', firmware_basename, 'tar.bz2', |
| 422 | constants.IMAGE_TYPE_FIRMWARE), |
David Riley | a04d19d | 2015-09-04 16:11:50 -0700 | [diff] [blame] | 423 | |
| 424 | ('firmware_from_source.tar.bz2', nv_lp0_firmware_basename, 'tar.bz2', |
| 425 | constants.IMAGE_TYPE_NV_LP0_FIRMWARE), |
Vincent Palatin | d599c66 | 2015-10-26 09:51:41 -0700 | [diff] [blame] | 426 | |
| 427 | ('firmware_from_source.tar.bz2', acc_usbpd_basename, 'tar.bz2', |
| 428 | constants.IMAGE_TYPE_ACCESSORY_USBPD), |
| 429 | |
| 430 | ('firmware_from_source.tar.bz2', acc_rwsig_basename, 'tar.bz2', |
| 431 | constants.IMAGE_TYPE_ACCESSORY_RWSIG), |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 432 | ) |
| 433 | |
| 434 | # The following build artifacts are copied and marked for signing, if |
| 435 | # they are present *and* if the image type is specified via |sign_types|. |
| 436 | files_to_maybe_copy_and_sign = ( |
| 437 | # (<src>, <dst>, <suffix>, <signing type>), |
| 438 | (constants.BASE_IMAGE_TAR, base_basename, 'tar.xz', |
| 439 | constants.IMAGE_TYPE_BASE), |
| 440 | ) |
| 441 | |
| 442 | def _CopyFileToGS(src, dst, suffix): |
| 443 | """Returns |dst| file name if the copying was successful.""" |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 444 | if not dst: |
| 445 | dst = src |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 446 | elif suffix: |
| 447 | dst = '%s.%s' % (dst, suffix) |
| 448 | success = False |
Mike Frysinger | e51a265 | 2014-01-18 02:36:16 -0500 | [diff] [blame] | 449 | try: |
| 450 | ctx.Copy(os.path.join(src_path, src), os.path.join(dst_path, dst)) |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 451 | success = True |
Mike Frysinger | e51a265 | 2014-01-18 02:36:16 -0500 | [diff] [blame] | 452 | except gs.GSNoSuchKey: |
Ralph Nathan | 446aee9 | 2015-03-23 14:44:56 -0700 | [diff] [blame] | 453 | logging.warning('Skipping %s as it does not exist', src) |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 454 | except gs.GSContextException: |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 455 | unknown_error[0] = True |
Ralph Nathan | 5990042 | 2015-03-24 10:41:17 -0700 | [diff] [blame] | 456 | logging.error('Skipping %s due to unknown GS error', src, exc_info=True) |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 457 | return dst if success else None |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 458 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 459 | for src, dst, suffix in files_to_copy_only: |
| 460 | _CopyFileToGS(src, dst, suffix) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 461 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 462 | # Clear the list of files to sign before adding new artifacts. |
| 463 | files_to_sign = [] |
| 464 | |
| 465 | def _AddToFilesToSign(image_type, dst, suffix): |
| 466 | assert dst.endswith('.' + suffix), ( |
| 467 | 'dst: %s, suffix: %s' % (dst, suffix)) |
| 468 | dst_base = dst[:-(len(suffix) + 1)] |
| 469 | files_to_sign.append([image_type, dst_base, suffix]) |
| 470 | |
| 471 | for src, dst, suffix, image_type in files_to_copy_and_maybe_sign: |
| 472 | dst = _CopyFileToGS(src, dst, suffix) |
| 473 | if dst and (not sign_types or image_type in sign_types): |
| 474 | _AddToFilesToSign(image_type, dst, suffix) |
| 475 | |
| 476 | for src, dst, suffix, image_type in files_to_maybe_copy_and_sign: |
| 477 | if sign_types and image_type in sign_types: |
| 478 | dst = _CopyFileToGS(src, dst, suffix) |
| 479 | if dst: |
| 480 | _AddToFilesToSign(image_type, dst, suffix) |
| 481 | |
| 482 | logging.debug('Files to sign: %s', files_to_sign) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 483 | # Now go through the subset for signing. |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 484 | for image_type, dst_name, suffix in files_to_sign: |
| 485 | try: |
| 486 | input_insns = InputInsns(board, image_type=image_type) |
| 487 | except MissingBoardInstructions as e: |
| 488 | logging.info('Nothing to sign: %s', e) |
| 489 | continue |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 490 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 491 | dst_archive = '%s.%s' % (dst_name, suffix) |
| 492 | sect_general['archive'] = dst_archive |
| 493 | sect_general['type'] = image_type |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 494 | |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 495 | # In the default/automatic mode, only flag files for signing if the |
| 496 | # archives were actually uploaded in a previous stage. This additional |
| 497 | # check can be removed in future once |sign_types| becomes a required |
| 498 | # argument. |
| 499 | # TODO: Make |sign_types| a required argument. |
| 500 | gs_artifact_path = os.path.join(dst_path, dst_archive) |
| 501 | exists = False |
| 502 | try: |
| 503 | exists = ctx.Exists(gs_artifact_path) |
| 504 | except gs.GSContextException: |
| 505 | unknown_error[0] = True |
| 506 | logging.error('Unknown error while checking %s', gs_artifact_path, |
| 507 | exc_info=True) |
| 508 | if not exists: |
| 509 | logging.info('%s does not exist. Nothing to sign.', |
| 510 | gs_artifact_path) |
| 511 | continue |
| 512 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 513 | first_image = True |
| 514 | for alt_insn_set in input_insns.GetAltInsnSets(): |
| 515 | # Figure out which keysets have been requested for this type. |
| 516 | # We sort the forced set so tests/runtime behavior is stable. |
| 517 | keysets = sorted(force_keysets) |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 518 | if not keysets: |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 519 | keysets = input_insns.GetKeysets(insns_merge=alt_insn_set) |
| 520 | if not keysets: |
| 521 | logging.warning('Skipping %s image signing due to no keysets', |
| 522 | image_type) |
Mike Frysinger | d84d91e | 2015-11-05 18:02:24 -0500 | [diff] [blame] | 523 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 524 | for keyset in keysets: |
| 525 | sect_insns['keyset'] = keyset |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 526 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 527 | # Generate the insn file for this artifact that the signer will use, |
| 528 | # and flag it for signing. |
| 529 | with tempfile.NamedTemporaryFile( |
| 530 | bufsize=0, prefix='pushimage.insns.') as insns_path: |
| 531 | input_insns.OutputInsns(insns_path.name, sect_insns, sect_general, |
| 532 | insns_merge=alt_insn_set) |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 533 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 534 | gs_insns_path = '%s/%s' % (dst_path, dst_name) |
| 535 | if not first_image: |
| 536 | gs_insns_path += '-%s' % keyset |
| 537 | first_image = False |
| 538 | gs_insns_path += '.instructions' |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 539 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 540 | try: |
| 541 | ctx.Copy(insns_path.name, gs_insns_path) |
| 542 | except gs.GSContextException: |
| 543 | unknown_error[0] = True |
| 544 | logging.error('Unknown error while uploading insns %s', |
| 545 | gs_insns_path, exc_info=True) |
| 546 | continue |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 547 | |
Mike Frysinger | 37ccc2b | 2015-11-11 17:16:51 -0500 | [diff] [blame] | 548 | try: |
| 549 | MarkImageToBeSigned(ctx, tbs_base, gs_insns_path, priority) |
| 550 | except gs.GSContextException: |
| 551 | unknown_error[0] = True |
| 552 | logging.error('Unknown error while marking for signing %s', |
| 553 | gs_insns_path, exc_info=True) |
| 554 | continue |
| 555 | logging.info('Signing %s image with keyset %s at %s', image_type, |
| 556 | keyset, gs_insns_path) |
| 557 | instruction_urls.setdefault(channel, []).append(gs_insns_path) |
Don Garrett | 9459c2f | 2014-01-22 18:20:24 -0800 | [diff] [blame] | 558 | |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 559 | if unknown_error[0]: |
Mike Frysinger | 4495b03 | 2014-03-05 17:24:03 -0500 | [diff] [blame] | 560 | raise PushError('hit some unknown error(s)', instruction_urls) |
| 561 | |
Don Garrett | 9459c2f | 2014-01-22 18:20:24 -0800 | [diff] [blame] | 562 | return instruction_urls |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 563 | |
| 564 | |
| 565 | def main(argv): |
| 566 | parser = commandline.ArgumentParser(description=__doc__) |
| 567 | |
| 568 | # The type of image_dir will strip off trailing slashes (makes later |
| 569 | # processing simpler and the display prettier). |
| 570 | parser.add_argument('image_dir', default=None, type='local_or_gs_path', |
| 571 | help='full path of source artifacts to upload') |
| 572 | parser.add_argument('--board', default=None, required=True, |
| 573 | help='board to generate symbols for') |
| 574 | parser.add_argument('--profile', default=None, |
| 575 | help='board profile in use (e.g. "asan")') |
| 576 | parser.add_argument('--version', default=None, |
| 577 | help='version info (normally extracted from image_dir)') |
| 578 | parser.add_argument('-n', '--dry-run', default=False, action='store_true', |
| 579 | help='show what would be done, but do not upload') |
| 580 | parser.add_argument('-M', '--mock', default=False, action='store_true', |
| 581 | help='upload things to a testing bucket (dev testing)') |
David Riley | f820512 | 2015-09-04 13:46:36 -0700 | [diff] [blame] | 582 | parser.add_argument('--test-sign', default=[], action='append', |
| 583 | choices=TEST_KEYSETS, |
| 584 | help='mung signing behavior to sign w/ test keys') |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 585 | parser.add_argument('--priority', type=int, default=50, |
| 586 | help='set signing priority (lower == higher prio)') |
| 587 | parser.add_argument('--sign-types', default=None, nargs='+', |
Amey Deshpande | a936c62 | 2015-08-12 17:27:54 -0700 | [diff] [blame] | 588 | choices=_SUPPORTED_IMAGE_TYPES, |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 589 | help='only sign specified image types') |
Mike Frysinger | 09fe012 | 2014-02-09 02:44:05 -0500 | [diff] [blame] | 590 | parser.add_argument('--yes', action='store_true', default=False, |
| 591 | help='answer yes to all prompts') |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 592 | |
| 593 | opts = parser.parse_args(argv) |
| 594 | opts.Freeze() |
| 595 | |
David Riley | f820512 | 2015-09-04 13:46:36 -0700 | [diff] [blame] | 596 | force_keysets = set(['%s-%s' % (TEST_KEYSET_PREFIX, x) |
| 597 | for x in opts.test_sign]) |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 598 | |
Mike Frysinger | 09fe012 | 2014-02-09 02:44:05 -0500 | [diff] [blame] | 599 | # If we aren't using mock or test or dry run mode, then let's prompt the user |
| 600 | # to make sure they actually want to do this. It's rare that people want to |
| 601 | # run this directly and hit the release bucket. |
| 602 | if not (opts.mock or force_keysets or opts.dry_run) and not opts.yes: |
| 603 | prolog = '\n'.join(textwrap.wrap(textwrap.dedent( |
| 604 | 'Uploading images for signing to the *release* bucket is not something ' |
| 605 | 'you generally should be doing yourself.'), 80)).strip() |
| 606 | if not cros_build_lib.BooleanPrompt( |
| 607 | prompt='Are you sure you want to sign these images', |
| 608 | default=False, prolog=prolog): |
| 609 | cros_build_lib.Die('better safe than sorry') |
| 610 | |
Mike Frysinger | d13faeb | 2013-09-05 16:00:46 -0400 | [diff] [blame] | 611 | PushImage(opts.image_dir, opts.board, versionrev=opts.version, |
| 612 | profile=opts.profile, priority=opts.priority, |
Mike Frysinger | dad40d6 | 2014-02-09 02:18:02 -0500 | [diff] [blame] | 613 | sign_types=opts.sign_types, dry_run=opts.dry_run, mock=opts.mock, |
| 614 | force_keysets=force_keysets) |