blob: 26209d2708b3af173038374a06ea4f1479c4b04c [file] [log] [blame]
David James8c846492011-01-25 17:07:29 -08001#!/usr/bin/python
David Jamesb619a782012-07-25 19:37:57 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
David James8c846492011-01-25 17:07:29 -08003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Brian Harringaf019fb2012-05-10 15:06:13 -07006"""This script is used to upload host prebuilts as well as board BINHOSTS.
David James8c846492011-01-25 17:07:29 -08007
David James015af872012-06-19 15:24:36 -07008Prebuilts are uploaded using gsutil to Google Storage. After these prebuilts
9are successfully uploaded, a file is updated with the proper BINHOST version.
David James8c846492011-01-25 17:07:29 -080010
11To read more about prebuilts/binhost binary packages please refer to:
David James015af872012-06-19 15:24:36 -070012http://goto/chromeos-prebuilts
David James8c846492011-01-25 17:07:29 -080013
14Example of uploading prebuilt amd64 host files to Google Storage:
David Jamesc5cbd472012-06-19 16:25:45 -070015upload_prebuilts -p /b/cbuild/build -s -u gs://chromeos-prebuilt
David James8c846492011-01-25 17:07:29 -080016
17Example of uploading x86-dogfood binhosts to Google Storage:
David Jamesc5cbd472012-06-19 16:25:45 -070018upload_prebuilts -b x86-dogfood -p /b/cbuild/build/ -u gs://chromeos-prebuilt -g
David James8c846492011-01-25 17:07:29 -080019"""
20
Mike Frysingerff441bf2014-05-24 13:47:21 -040021from __future__ import print_function
22
Chris Sosa1dc96132012-05-11 15:40:50 -070023import datetime
Mike Frysinger540883b2014-05-24 13:46:16 -040024import functools
Chris Sosa1dc96132012-05-11 15:40:50 -070025import multiprocessing
Chris Sosa1dc96132012-05-11 15:40:50 -070026import os
27import sys
Mike Frysinger212e4292014-05-24 15:15:44 -040028import tempfile
Chris Sosa1dc96132012-05-11 15:40:50 -070029
Don Garrett88b8d782014-05-13 17:30:55 -070030from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070031from chromite.cbuildbot import commands
Don Garrett88b8d782014-05-13 17:30:55 -070032from chromite.cbuildbot import portage_utilities
David James6450a0a2012-12-04 07:59:53 -080033from chromite.lib import binpkg
Mike Frysinger86509232014-05-24 13:18:37 -040034from chromite.lib import commandline
Chris Sosa1dc96132012-05-11 15:40:50 -070035from chromite.lib import cros_build_lib
David James97d95872012-11-16 15:09:56 -080036from chromite.lib import git
Brian Harring7904b482012-08-08 02:54:12 -070037from chromite.lib import gs
Brian Harringaf019fb2012-05-10 15:06:13 -070038from chromite.lib import osutils
David James6450a0a2012-12-04 07:59:53 -080039from chromite.lib import parallel
Mike Frysinger8e727a32013-01-16 16:57:53 -050040from chromite.lib import toolchain
Chris Sosa1dc96132012-05-11 15:40:50 -070041
David James015af872012-06-19 15:24:36 -070042# How many times to retry uploads.
43_RETRIES = 10
44
45# Multiplier for how long to sleep (in seconds) between retries; will delay
46# (1*sleep) the first time, then (2*sleep), continuing via attempt * sleep.
47_SLEEP_TIME = 60
48
David James8c846492011-01-25 17:07:29 -080049_HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
David James05bcb2b2011-02-09 09:25:47 -080050_CATEGORIES_PATH = 'chroot/etc/portage/categories'
David James615e5b52011-06-03 11:10:15 -070051_PYM_PATH = 'chroot/usr/lib/portage/pym'
David James4058b0d2011-12-08 21:24:50 -080052_HOST_ARCH = 'amd64'
David James8c846492011-01-25 17:07:29 -080053_BOARD_PATH = 'chroot/build/%(board)s'
David James4058b0d2011-12-08 21:24:50 -080054_REL_BOARD_PATH = 'board/%(target)s/%(version)s'
55_REL_HOST_PATH = 'host/%(host_arch)s/%(target)s/%(version)s'
David James8c846492011-01-25 17:07:29 -080056# Private overlays to look at for builds to filter
57# relative to build path
58_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
Scott Zawalskiab1bed32011-03-16 15:24:24 -070059_GOOGLESTORAGE_ACL_FILE = 'googlestorage_acl.xml'
David James3753d942014-04-23 10:55:48 -070060_BINHOST_BASE_URL = 'gs://chromeos-prebuilt'
David James8c846492011-01-25 17:07:29 -080061_PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
62# Created in the event of new host targets becoming available
63_PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
64 'make.conf.amd64-host')}
David James8c846492011-01-25 17:07:29 -080065
66
David James4058b0d2011-12-08 21:24:50 -080067class BuildTarget(object):
68 """A board/variant/profile tuple."""
69
70 def __init__(self, board_variant, profile=None):
71 self.board_variant = board_variant
72 self.board, _, self.variant = board_variant.partition('_')
73 self.profile = profile
74
75 def __str__(self):
76 if self.profile:
77 return '%s_%s' % (self.board_variant, self.profile)
78 else:
79 return self.board_variant
80
81 def __eq__(self, other):
82 return str(other) == str(self)
83
84 def __hash__(self):
85 return hash(str(self))
86
87
David James8c846492011-01-25 17:07:29 -080088def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
89 """Update the key in file with the value passed.
Mike Frysinger1a736a82013-12-12 01:50:59 -050090
David James8c846492011-01-25 17:07:29 -080091 File format:
92 key="value"
93 Note quotes are added automatically
94
95 Args:
96 filename: Name of file to modify.
97 value: Value to write with the key.
98 key: The variable key to update. (Default: PORTAGE_BINHOST)
99 """
100 if os.path.exists(filename):
101 file_fh = open(filename)
102 else:
103 file_fh = open(filename, 'w+')
104 file_lines = []
105 found = False
106 keyval_str = '%(key)s=%(value)s'
107 for line in file_fh:
108 # Strip newlines from end of line. We already add newlines below.
109 line = line.rstrip("\n")
110
111 if len(line.split('=')) != 2:
112 # Skip any line that doesn't fit key=val.
113 file_lines.append(line)
114 continue
115
116 file_var, file_val = line.split('=')
117 if file_var == key:
118 found = True
Mike Frysingerff441bf2014-05-24 13:47:21 -0400119 print('Updating %s=%s to %s="%s"' % (file_var, file_val, key, value))
David James8c846492011-01-25 17:07:29 -0800120 value = '"%s"' % value
121 file_lines.append(keyval_str % {'key': key, 'value': value})
122 else:
123 file_lines.append(keyval_str % {'key': file_var, 'value': file_val})
124
125 if not found:
Brian Harring2a014302012-05-12 00:53:33 -0700126 value = '"%s"' % value
David James8c846492011-01-25 17:07:29 -0800127 file_lines.append(keyval_str % {'key': key, 'value': value})
128
129 file_fh.close()
130 # write out new file
Brian Harringaf019fb2012-05-10 15:06:13 -0700131 osutils.WriteFile(filename, '\n'.join(file_lines) + '\n')
David James8c846492011-01-25 17:07:29 -0800132
133
Mike Frysinger5b34d732013-01-17 15:11:58 -0500134def RevGitFile(filename, data, retries=5, dryrun=False):
David James8c846492011-01-25 17:07:29 -0800135 """Update and push the git file.
136
Mike Frysinger5b34d732013-01-17 15:11:58 -0500137 Args:
138 filename: file to modify that is in a git repo already
139 data: A dict of key/values to update in |filename|
140 retries: The number of times to retry before giving up, default: 5
Matt Tennante8179042013-10-01 15:47:32 -0700141 dryrun: If True, do not actually commit the change.
David James8c846492011-01-25 17:07:29 -0800142 """
143 prebuilt_branch = 'prebuilt_branch'
David James1b6e67a2011-05-19 21:32:38 -0700144 cwd = os.path.abspath(os.path.dirname(filename))
David James97d95872012-11-16 15:09:56 -0800145 commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
Mike Frysinger5b34d732013-01-17 15:11:58 -0500146 description = '%s: updating %s' % (os.path.basename(filename),
147 ', '.join(data.keys()))
148 # UpdateLocalFile will print out the keys/values for us.
Mike Frysingerff441bf2014-05-24 13:47:21 -0400149 print('Revving git file %s' % filename)
David James66009462012-03-25 10:08:38 -0700150
David James8c846492011-01-25 17:07:29 -0800151 try:
David James97d95872012-11-16 15:09:56 -0800152 git.CreatePushBranch(prebuilt_branch, cwd)
Mike Frysinger5b34d732013-01-17 15:11:58 -0500153 for key, value in data.iteritems():
154 UpdateLocalFile(filename, value, key)
David James97d95872012-11-16 15:09:56 -0800155 git.RunGit(cwd, ['add', filename])
156 git.RunGit(cwd, ['commit', '-m', description])
157 git.PushWithRetry(prebuilt_branch, cwd, dryrun=dryrun, retries=retries)
David James8c846492011-01-25 17:07:29 -0800158 finally:
David James67d73252013-09-19 17:33:12 -0700159 git.RunGit(cwd, ['checkout', commit])
David James8c846492011-01-25 17:07:29 -0800160
161
162def GetVersion():
163 """Get the version to put in LATEST and update the git version with."""
Mike Frysinger0b92cfe2012-12-13 02:13:45 -0500164 return datetime.datetime.now().strftime('%Y.%m.%d.%H%M%S')
David James8c846492011-01-25 17:07:29 -0800165
166
Mike Frysinger540883b2014-05-24 13:46:16 -0400167def _GsUpload(gs_context, acl, local_file, remote_file):
David James8c846492011-01-25 17:07:29 -0800168 """Upload to GS bucket.
169
170 Args:
Mike Frysinger540883b2014-05-24 13:46:16 -0400171 gs_context: A lib.gs.GSContext instance.
172 acl: The ACL to use for uploading the file.
Matt Tennante8179042013-10-01 15:47:32 -0700173 local_file: The local file to be uploaded.
174 remote_file: The remote location to upload to.
David James8c846492011-01-25 17:07:29 -0800175
176 Returns:
177 Return the arg tuple of two if the upload failed
178 """
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700179 CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
180 'authenticated-read', 'bucket-owner-full-control',
181 'public-read-write']
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700182 if acl in CANNED_ACLS:
David James9374aac2013-10-08 16:00:17 -0700183 gs_context.Copy(local_file, remote_file, acl=acl)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700184 else:
185 # For private uploads we assume that the overlay board is set up properly
David James015af872012-06-19 15:24:36 -0700186 # and a googlestore_acl.xml is present. Otherwise, this script errors.
David James9374aac2013-10-08 16:00:17 -0700187 gs_context.Copy(local_file, remote_file, acl='private')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700188 # Apply the passed in ACL xml file to the uploaded object.
David James9374aac2013-10-08 16:00:17 -0700189 gs_context.SetACL(remote_file, acl=acl)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700190
Mike Frysingercc838832014-05-24 13:10:30 -0400191
Mike Frysinger540883b2014-05-24 13:46:16 -0400192def RemoteUpload(gs_context, acl, files, pool=10):
David James8c846492011-01-25 17:07:29 -0800193 """Upload to google storage.
194
195 Create a pool of process and call _GsUpload with the proper arguments.
196
197 Args:
Mike Frysinger540883b2014-05-24 13:46:16 -0400198 gs_context: A lib.gs.GSContext instance.
David Jamesfd0b0852011-02-23 11:15:36 -0800199 acl: The canned acl used for uploading. acl can be one of: "public-read",
200 "public-read-write", "authenticated-read", "bucket-owner-read",
201 "bucket-owner-full-control", or "private".
David James8c846492011-01-25 17:07:29 -0800202 files: dictionary with keys to local files and values to remote path.
203 pool: integer of maximum proesses to have at the same time.
204
205 Returns:
206 Return a set of tuple arguments of the failed uploads
207 """
Mike Frysinger540883b2014-05-24 13:46:16 -0400208 upload = functools.partial(_GsUpload, gs_context, acl)
209 tasks = [[key, value] for key, value in files.iteritems()]
210 parallel.RunTasksInProcessPool(upload, tasks, pool)
David James8c846492011-01-25 17:07:29 -0800211
212
213def GenerateUploadDict(base_local_path, base_remote_path, pkgs):
214 """Build a dictionary of local remote file key pairs to upload.
215
216 Args:
217 base_local_path: The base path to the files on the local hard drive.
Matt Tennante8179042013-10-01 15:47:32 -0700218 base_remote_path: The base path to the remote paths.
David James8c846492011-01-25 17:07:29 -0800219 pkgs: The packages to upload.
220
221 Returns:
222 Returns a dictionary of local_path/remote_path pairs
223 """
224 upload_files = {}
225 for pkg in pkgs:
226 suffix = pkg['CPV'] + '.tbz2'
227 local_path = os.path.join(base_local_path, suffix)
228 assert os.path.exists(local_path)
229 remote_path = '%s/%s' % (base_remote_path.rstrip('/'), suffix)
230 upload_files[local_path] = remote_path
231
232 return upload_files
233
Mike Frysingercc838832014-05-24 13:10:30 -0400234
Peter Mayo950e41a2014-02-06 21:07:33 +0000235def GetBoardOverlay(build_path, target):
David Jamese5867812012-10-19 12:02:20 -0700236 """Get the path to the board variant.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500237
David James8c846492011-01-25 17:07:29 -0800238 Args:
239 build_path: The path to the root of the build directory
David Jamese5867812012-10-19 12:02:20 -0700240 target: The target board as a BuildTarget object.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500241
David James8c846492011-01-25 17:07:29 -0800242 Returns:
David Jamese5867812012-10-19 12:02:20 -0700243 The last overlay configured for the given board as a string.
David James8c846492011-01-25 17:07:29 -0800244 """
David Jamese5867812012-10-19 12:02:20 -0700245 board = target.board_variant
246 overlays = portage_utilities.FindOverlays(constants.BOTH_OVERLAYS, board,
247 buildroot=build_path)
248 # We only care about the last entry.
249 return overlays[-1]
David James8c846492011-01-25 17:07:29 -0800250
251
252def DeterminePrebuiltConfFile(build_path, target):
253 """Determine the prebuilt.conf file that needs to be updated for prebuilts.
254
Mike Frysinger1a736a82013-12-12 01:50:59 -0500255 Args:
256 build_path: The path to the root of the build directory
257 target: String representation of the board. This includes host and board
258 targets
David James8c846492011-01-25 17:07:29 -0800259
Mike Frysinger1a736a82013-12-12 01:50:59 -0500260 Returns:
261 A string path to a prebuilt.conf file to be updated.
David James8c846492011-01-25 17:07:29 -0800262 """
David James4058b0d2011-12-08 21:24:50 -0800263 if _HOST_ARCH == target:
David James8c846492011-01-25 17:07:29 -0800264 # We are host.
265 # Without more examples of hosts this is a kludge for now.
266 # TODO(Scottz): as new host targets come online expand this to
267 # work more like boards.
Chris Sosa471532a2011-02-01 15:10:06 -0800268 make_path = _PREBUILT_MAKE_CONF[target]
David James8c846492011-01-25 17:07:29 -0800269 else:
270 # We are a board
Peter Mayo950e41a2014-02-06 21:07:33 +0000271 board = GetBoardOverlay(build_path, target)
David James8c846492011-01-25 17:07:29 -0800272 make_path = os.path.join(board, 'prebuilt.conf')
273
274 return make_path
275
276
277def UpdateBinhostConfFile(path, key, value):
278 """Update binhost config file file with key=value.
279
280 Args:
281 path: Filename to update.
282 key: Key to update.
283 value: New value for key.
284 """
285 cwd = os.path.dirname(os.path.abspath(path))
286 filename = os.path.basename(path)
Brian Harringaf019fb2012-05-10 15:06:13 -0700287 osutils.SafeMakedirs(cwd)
David Jamesf6e8fb72013-05-10 08:58:43 -0700288 if not git.GetCurrentBranch(cwd):
289 git.CreatePushBranch(constants.STABLE_EBUILD_BRANCH, cwd, sync=False)
Brian Harring22edb442012-05-11 23:55:18 -0700290 osutils.WriteFile(path, '', mode='a')
David James8c846492011-01-25 17:07:29 -0800291 UpdateLocalFile(path, value, key)
David James67d73252013-09-19 17:33:12 -0700292 git.RunGit(cwd, ['add', filename])
Mike Frysinger109cada2013-01-17 15:50:44 -0500293 description = '%s: updating %s' % (os.path.basename(filename), key)
David James67d73252013-09-19 17:33:12 -0700294 git.RunGit(cwd, ['commit', '-m', description])
David James8c846492011-01-25 17:07:29 -0800295
296
Mike Frysinger212e4292014-05-24 15:15:44 -0400297def GenerateHtmlIndex(files, index, board, version):
298 """Given the list of |files|, generate an index.html at |index|.
299
300 Args:
301 files: The list of files to link to.
302 index: The path to the html index.
303 board: Name of the board this index is for.
304 version: Build version this index is for.
305 """
306 head = """<html>
307<head>
308 <title>Package Prebuilt Index: %(board)s / %(version)s</title>
309</head>
310<body>
311<h2>Package Prebuilt Index: %(board)s / %(version)s</h2>"""
312 head %= {
313 'board': board,
314 'version': version,
315 }
316
317 files = files + [
318 '.|Google Storage Index',
319 '..|',
320 ]
321 commands.GenerateHtmlIndex(index, files, head=head)
322
323
David Jamesce093af2011-02-23 15:21:58 -0800324def _GrabAllRemotePackageIndexes(binhost_urls):
David James05bcb2b2011-02-09 09:25:47 -0800325 """Grab all of the packages files associated with a list of binhost_urls.
326
David James05bcb2b2011-02-09 09:25:47 -0800327 Args:
328 binhost_urls: The URLs for the directories containing the Packages files we
329 want to grab.
David James05bcb2b2011-02-09 09:25:47 -0800330
331 Returns:
332 A list of PackageIndex objects.
333 """
334 pkg_indexes = []
335 for url in binhost_urls:
David James32faafe2012-06-08 14:25:03 -0700336 pkg_index = binpkg.GrabRemotePackageIndex(url)
David James05bcb2b2011-02-09 09:25:47 -0800337 if pkg_index:
338 pkg_indexes.append(pkg_index)
David James05bcb2b2011-02-09 09:25:47 -0800339 return pkg_indexes
340
341
David Jamesc0f158a2011-02-22 16:07:29 -0800342class PrebuiltUploader(object):
343 """Synchronize host and board prebuilts."""
David James8c846492011-01-25 17:07:29 -0800344
Mike Frysinger86509232014-05-24 13:18:37 -0400345 def __init__(self, upload_location, acl, binhost_base_url, pkg_indexes,
346 build_path, packages, skip_upload, binhost_conf_dir, dryrun,
Mike Frysinger8092a632014-05-24 13:25:46 -0400347 target, slave_targets, version):
David Jamesc0f158a2011-02-22 16:07:29 -0800348 """Constructor for prebuilt uploader object.
David James8c846492011-01-25 17:07:29 -0800349
David Jamesc0f158a2011-02-22 16:07:29 -0800350 This object can upload host or prebuilt files to Google Storage.
David James8c846492011-01-25 17:07:29 -0800351
David Jamesc0f158a2011-02-22 16:07:29 -0800352 Args:
353 upload_location: The upload location.
David Jamesfd0b0852011-02-23 11:15:36 -0800354 acl: The canned acl used for uploading to Google Storage. acl can be one
355 of: "public-read", "public-read-write", "authenticated-read",
Matt Tennante8179042013-10-01 15:47:32 -0700356 "bucket-owner-read", "bucket-owner-full-control", "project-private",
357 or "private" (see "gsutil help acls"). If we are not uploading to
358 Google Storage, this parameter is unused.
David Jamesfd0b0852011-02-23 11:15:36 -0800359 binhost_base_url: The URL used for downloading the prebuilts.
David Jamesc0f158a2011-02-22 16:07:29 -0800360 pkg_indexes: Old uploaded prebuilts to compare against. Instead of
361 uploading duplicate files, we just link to the old files.
David James615e5b52011-06-03 11:10:15 -0700362 build_path: The path to the directory containing the chroot.
363 packages: Packages to upload.
David James32b0b2f2011-07-13 20:56:50 -0700364 skip_upload: Don't actually upload the tarballs.
365 binhost_conf_dir: Directory where to store binhost.conf files.
Mike Frysinger86509232014-05-24 13:18:37 -0400366 dryrun: Don't push or upload prebuilts.
David James4058b0d2011-12-08 21:24:50 -0800367 target: BuildTarget managed by this builder.
368 slave_targets: List of BuildTargets managed by slave builders.
Mike Frysinger8092a632014-05-24 13:25:46 -0400369 version: A unique string, intended to be included in the upload path,
370 which identifies the version number of the uploaded prebuilts.
David Jamesc0f158a2011-02-22 16:07:29 -0800371 """
372 self._upload_location = upload_location
David Jamesfd0b0852011-02-23 11:15:36 -0800373 self._acl = acl
David Jamesc0f158a2011-02-22 16:07:29 -0800374 self._binhost_base_url = binhost_base_url
375 self._pkg_indexes = pkg_indexes
David James615e5b52011-06-03 11:10:15 -0700376 self._build_path = build_path
377 self._packages = set(packages)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500378 self._found_packages = set()
David James8ece7ee2011-06-29 16:02:30 -0700379 self._skip_upload = skip_upload
David James32b0b2f2011-07-13 20:56:50 -0700380 self._binhost_conf_dir = binhost_conf_dir
Mike Frysinger86509232014-05-24 13:18:37 -0400381 self._dryrun = dryrun
David James4058b0d2011-12-08 21:24:50 -0800382 self._target = target
383 self._slave_targets = slave_targets
Mike Frysinger8092a632014-05-24 13:25:46 -0400384 self._version = version
Mike Frysinger540883b2014-05-24 13:46:16 -0400385 self._gs_context = gs.GSContext(retries=_RETRIES, sleep=_SLEEP_TIME,
386 dry_run=self._dryrun)
387
388 def _Upload(self, local_file, remote_file):
389 """Wrapper around _GsUpload"""
390 _GsUpload(self._gs_context, self._acl, local_file, remote_file)
David James615e5b52011-06-03 11:10:15 -0700391
392 def _ShouldFilterPackage(self, pkg):
393 if not self._packages:
394 return False
395 pym_path = os.path.abspath(os.path.join(self._build_path, _PYM_PATH))
David James710b7dc2012-02-07 16:49:59 -0800396 sys.path.insert(0, pym_path)
Don Garrett25f309a2014-03-19 14:02:12 -0700397 # pylint: disable=F0401
David James615e5b52011-06-03 11:10:15 -0700398 import portage.versions
399 cat, pkgname = portage.versions.catpkgsplit(pkg['CPV'])[0:2]
400 cp = '%s/%s' % (cat, pkgname)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500401 self._found_packages.add(cp)
David James615e5b52011-06-03 11:10:15 -0700402 return pkgname not in self._packages and cp not in self._packages
David James8c846492011-01-25 17:07:29 -0800403
David Jamesc0f158a2011-02-22 16:07:29 -0800404 def _UploadPrebuilt(self, package_path, url_suffix):
405 """Upload host or board prebuilt files to Google Storage space.
David James8c846492011-01-25 17:07:29 -0800406
David Jamesc0f158a2011-02-22 16:07:29 -0800407 Args:
408 package_path: The path to the packages dir.
David Jamesce093af2011-02-23 15:21:58 -0800409 url_suffix: The remote subdirectory where we should upload the packages.
David Jamesc0f158a2011-02-22 16:07:29 -0800410 """
David Jamesc0f158a2011-02-22 16:07:29 -0800411 # Process Packages file, removing duplicates and filtered packages.
David James32faafe2012-06-08 14:25:03 -0700412 pkg_index = binpkg.GrabLocalPackageIndex(package_path)
David Jamesc0f158a2011-02-22 16:07:29 -0800413 pkg_index.SetUploadLocation(self._binhost_base_url, url_suffix)
David James615e5b52011-06-03 11:10:15 -0700414 pkg_index.RemoveFilteredPackages(self._ShouldFilterPackage)
David Jamesc0f158a2011-02-22 16:07:29 -0800415 uploads = pkg_index.ResolveDuplicateUploads(self._pkg_indexes)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500416 unmatched_pkgs = self._packages - self._found_packages
417 if unmatched_pkgs:
418 cros_build_lib.Warning('unable to match packages: %r' % unmatched_pkgs)
David James05bcb2b2011-02-09 09:25:47 -0800419
David Jamesc0f158a2011-02-22 16:07:29 -0800420 # Write Packages file.
421 tmp_packages_file = pkg_index.WriteToNamedTemporaryFile()
David James05bcb2b2011-02-09 09:25:47 -0800422
David Jamesc0f158a2011-02-22 16:07:29 -0800423 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
David James015af872012-06-19 15:24:36 -0700424 assert remote_location.startswith('gs://')
David James05bcb2b2011-02-09 09:25:47 -0800425
David James015af872012-06-19 15:24:36 -0700426 # Build list of files to upload.
427 upload_files = GenerateUploadDict(package_path, remote_location, uploads)
428 remote_file = '%s/Packages' % remote_location.rstrip('/')
429 upload_files[tmp_packages_file.name] = remote_file
430
Mike Frysinger540883b2014-05-24 13:46:16 -0400431 RemoteUpload(self._gs_context, self._acl, upload_files)
David James8c846492011-01-25 17:07:29 -0800432
Mike Frysinger212e4292014-05-24 15:15:44 -0400433 with tempfile.NamedTemporaryFile(
434 prefix='chromite.upload_prebuilts.index.') as index:
435 GenerateHtmlIndex(
436 [x[len(remote_location) + 1:] for x in upload_files.values()],
437 index.name, self._target, self._version)
438 self._Upload(index.name, '%s/index.html' % remote_location.rstrip('/'))
439
440 link_name = 'Prebuilts[%s]: %s' % (self._target, self._version)
441 url = '%s%s/index.html' % (gs.PUBLIC_BASE_HTTPS_URL,
442 remote_location[len(gs.BASE_GS_URL):])
443 cros_build_lib.PrintBuildbotLink(link_name, url)
444
Mike Frysinger8092a632014-05-24 13:25:46 -0400445 def _UploadSdkTarball(self, board_path, url_suffix, prepackaged,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500446 toolchain_tarballs, toolchain_upload_path):
447 """Upload a tarball of the sdk at the specified path to Google Storage.
David James8fa34ea2011-04-15 13:00:20 -0700448
449 Args:
450 board_path: The path to the board dir.
451 url_suffix: The remote subdirectory where we should upload the packages.
Zdenek Behan62a57792012-08-31 15:09:08 +0200452 prepackaged: If given, a tarball that has been packaged outside of this
453 script and should be used.
Mike Frysinger9e979b92012-11-29 02:55:09 -0500454 toolchain_tarballs: List of toolchain tarballs to upload.
455 toolchain_upload_path: Path under the bucket to place toolchain tarballs.
David James8fa34ea2011-04-15 13:00:20 -0700456 """
457 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
458 assert remote_location.startswith('gs://')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200459 boardname = os.path.basename(board_path.rstrip('/'))
460 # We do not upload non SDK board tarballs,
461 assert boardname == constants.CHROOT_BUILDER_BOARD
462 assert prepackaged is not None
Zdenek Behan62a57792012-08-31 15:09:08 +0200463
Mike Frysinger8092a632014-05-24 13:25:46 -0400464 version_str = self._version[len('chroot-'):]
Mike Frysinger8e727a32013-01-16 16:57:53 -0500465 remote_tarfile = toolchain.GetSdkURL(
466 for_gsutil=True, suburl='cros-sdk-%s.tar.xz' % (version_str,))
Zdenek Behan86c15e92012-10-13 00:55:47 +0200467 # For SDK, also upload the manifest which is guaranteed to exist
468 # by the builderstage.
Mike Frysinger540883b2014-05-24 13:46:16 -0400469 self._Upload(prepackaged + '.Manifest', remote_tarfile + '.Manifest')
470 self._Upload(prepackaged, remote_tarfile)
Zdenek Behan86c15e92012-10-13 00:55:47 +0200471
Mike Frysinger9e979b92012-11-29 02:55:09 -0500472 # Post the toolchain tarballs too.
473 for tarball in toolchain_tarballs:
474 target, local_path = tarball.split(':')
475 suburl = toolchain_upload_path % {'target': target}
476 remote_path = toolchain.GetSdkURL(for_gsutil=True, suburl=suburl)
Mike Frysinger540883b2014-05-24 13:46:16 -0400477 self._Upload(local_path, remote_path)
Mike Frysinger9e979b92012-11-29 02:55:09 -0500478
Zdenek Behan86c15e92012-10-13 00:55:47 +0200479 # Finally, also update the pointer to the latest SDK on which polling
480 # scripts rely.
David James4bc13702013-03-26 08:08:04 -0700481 with osutils.TempDir() as tmpdir:
Zdenek Behan86c15e92012-10-13 00:55:47 +0200482 pointerfile = os.path.join(tmpdir, 'cros-sdk-latest.conf')
Mike Frysinger8e727a32013-01-16 16:57:53 -0500483 remote_pointerfile = toolchain.GetSdkURL(for_gsutil=True,
484 suburl='cros-sdk-latest.conf')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200485 osutils.WriteFile(pointerfile, 'LATEST_SDK="%s"' % version_str)
Mike Frysinger540883b2014-05-24 13:46:16 -0400486 self._Upload(pointerfile, remote_pointerfile)
Zdenek Behan33a34112012-09-10 21:07:51 +0200487
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700488 def _GetTargets(self):
489 """Retuns the list of targets to use."""
490 targets = self._slave_targets[:]
491 if self._target:
492 targets.append(self._target)
493
494 return targets
495
Mike Frysinger8092a632014-05-24 13:25:46 -0400496 def SyncHostPrebuilts(self, key, git_sync, sync_binhost_conf):
David Jamesc0f158a2011-02-22 16:07:29 -0800497 """Synchronize host prebuilt files.
David James05bcb2b2011-02-09 09:25:47 -0800498
David Jamesc0f158a2011-02-22 16:07:29 -0800499 This function will sync both the standard host packages, plus the host
500 packages associated with all targets that have been "setup" with the
501 current host's chroot. For instance, if this host has been used to build
502 x86-generic, it will sync the host packages associated with
503 'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
504 it will also sync the host packages associated with
505 'armv7a-cros-linux-gnueabi'.
David James05bcb2b2011-02-09 09:25:47 -0800506
David Jamesc0f158a2011-02-22 16:07:29 -0800507 Args:
David Jamesc0f158a2011-02-22 16:07:29 -0800508 key: The variable key to update in the git file.
509 git_sync: If set, update make.conf of target to reference the latest
510 prebuilt packages generated here.
511 sync_binhost_conf: If set, update binhost config file in
512 chromiumos-overlay for the host.
513 """
David Jamese2488642011-11-14 16:15:20 -0800514 # Slave boards are listed before the master board so that the master board
515 # takes priority (i.e. x86-generic preflight host prebuilts takes priority
516 # over preflight host prebuilts from other builders.)
517 binhost_urls = []
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700518 for target in self._GetTargets():
Mike Frysinger8092a632014-05-24 13:25:46 -0400519 url_suffix = _REL_HOST_PATH % {'version': self._version,
David James4058b0d2011-12-08 21:24:50 -0800520 'host_arch': _HOST_ARCH,
521 'target': target}
David Jamese2488642011-11-14 16:15:20 -0800522 packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
David James05bcb2b2011-02-09 09:25:47 -0800523
Mike Frysinger540883b2014-05-24 13:46:16 -0400524 if self._target == target and not self._skip_upload:
David Jamese2488642011-11-14 16:15:20 -0800525 # Upload prebuilts.
526 package_path = os.path.join(self._build_path, _HOST_PACKAGES_PATH)
527 self._UploadPrebuilt(package_path, packages_url_suffix)
David James8ece7ee2011-06-29 16:02:30 -0700528
David Jamese2488642011-11-14 16:15:20 -0800529 # Record URL where prebuilts were uploaded.
530 binhost_urls.append('%s/%s/' % (self._binhost_base_url.rstrip('/'),
531 packages_url_suffix.rstrip('/')))
532
David James20b2b6f2011-11-18 15:11:58 -0800533 binhost = ' '.join(binhost_urls)
David James8ece7ee2011-06-29 16:02:30 -0700534 if git_sync:
535 git_file = os.path.join(self._build_path,
David James4058b0d2011-12-08 21:24:50 -0800536 _PREBUILT_MAKE_CONF[_HOST_ARCH])
Mike Frysinger86509232014-05-24 13:18:37 -0400537 RevGitFile(git_file, {key: binhost}, dryrun=self._dryrun)
David James8ece7ee2011-06-29 16:02:30 -0700538 if sync_binhost_conf:
David James32b0b2f2011-07-13 20:56:50 -0700539 binhost_conf = os.path.join(self._build_path, self._binhost_conf_dir,
David James4058b0d2011-12-08 21:24:50 -0800540 'host', '%s-%s.conf' % (_HOST_ARCH, key))
David Jamese2488642011-11-14 16:15:20 -0800541 UpdateBinhostConfFile(binhost_conf, key, binhost)
David Jamesc0f158a2011-02-22 16:07:29 -0800542
Mike Frysinger8092a632014-05-24 13:25:46 -0400543 def SyncBoardPrebuilts(self, key, git_sync, sync_binhost_conf,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500544 upload_board_tarball, prepackaged_board,
545 toolchain_tarballs, toolchain_upload_path):
David Jamesc0f158a2011-02-22 16:07:29 -0800546 """Synchronize board prebuilt files.
547
548 Args:
David Jamesc0f158a2011-02-22 16:07:29 -0800549 key: The variable key to update in the git file.
550 git_sync: If set, update make.conf of target to reference the latest
551 prebuilt packages generated here.
552 sync_binhost_conf: If set, update binhost config file in
553 chromiumos-overlay for the current board.
David James8fa34ea2011-04-15 13:00:20 -0700554 upload_board_tarball: Include a tarball of the board in our upload.
Zdenek Behan62a57792012-08-31 15:09:08 +0200555 prepackaged_board: A tarball of the board built outside of this script.
Mike Frysinger9e979b92012-11-29 02:55:09 -0500556 toolchain_tarballs: A list of toolchain tarballs to upload.
557 toolchain_upload_path: Path under the bucket to place toolchain tarballs.
David Jamesc0f158a2011-02-22 16:07:29 -0800558 """
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700559 for target in self._GetTargets():
David Jamese2488642011-11-14 16:15:20 -0800560 board_path = os.path.join(self._build_path,
David James4058b0d2011-12-08 21:24:50 -0800561 _BOARD_PATH % {'board': target.board_variant})
David Jamese2488642011-11-14 16:15:20 -0800562 package_path = os.path.join(board_path, 'packages')
Mike Frysinger8092a632014-05-24 13:25:46 -0400563 url_suffix = _REL_BOARD_PATH % {'target': target,
564 'version': self._version}
David Jamese2488642011-11-14 16:15:20 -0800565 packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
David James8fa34ea2011-04-15 13:00:20 -0700566
Matt Tennante8179042013-10-01 15:47:32 -0700567 # Process the target board differently if it is the main --board.
Mike Frysinger540883b2014-05-24 13:46:16 -0400568 if self._target == target and not self._skip_upload:
Matt Tennante8179042013-10-01 15:47:32 -0700569 # This strips "chroot" prefix because that is sometimes added as the
570 # --prepend-version argument (e.g. by chromiumos-sdk bot).
571 # TODO(build): Clean it up to be less hard-coded.
Mike Frysinger8092a632014-05-24 13:25:46 -0400572 version_str = self._version[len('chroot-'):]
Mike Frysinger9e979b92012-11-29 02:55:09 -0500573
David Jamese2488642011-11-14 16:15:20 -0800574 # Upload board tarballs in the background.
575 if upload_board_tarball:
Mike Frysinger9e979b92012-11-29 02:55:09 -0500576 if toolchain_upload_path:
577 toolchain_upload_path %= {'version': version_str}
578 tar_process = multiprocessing.Process(
579 target=self._UploadSdkTarball,
Mike Frysinger8092a632014-05-24 13:25:46 -0400580 args=(board_path, url_suffix, prepackaged_board,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500581 toolchain_tarballs, toolchain_upload_path))
David Jamese2488642011-11-14 16:15:20 -0800582 tar_process.start()
David James8fa34ea2011-04-15 13:00:20 -0700583
David Jamese2488642011-11-14 16:15:20 -0800584 # Upload prebuilts.
585 self._UploadPrebuilt(package_path, packages_url_suffix)
David James8fa34ea2011-04-15 13:00:20 -0700586
David Jamese2488642011-11-14 16:15:20 -0800587 # Make sure we finished uploading the board tarballs.
588 if upload_board_tarball:
589 tar_process.join()
590 assert tar_process.exitcode == 0
591 # TODO(zbehan): This should be done cleaner.
Zdenek Behan33a34112012-09-10 21:07:51 +0200592 if target.board == constants.CHROOT_BUILDER_BOARD:
David Jamese2488642011-11-14 16:15:20 -0800593 sdk_conf = os.path.join(self._build_path, self._binhost_conf_dir,
Mike Frysinger5b34d732013-01-17 15:11:58 -0500594 'host/sdk_version.conf')
595 sdk_settings = {
Mike Frysinger9e979b92012-11-29 02:55:09 -0500596 'SDK_LATEST_VERSION': version_str,
597 'TC_PATH': toolchain_upload_path,
Mike Frysinger5b34d732013-01-17 15:11:58 -0500598 }
Mike Frysinger86509232014-05-24 13:18:37 -0400599 RevGitFile(sdk_conf, sdk_settings, dryrun=self._dryrun)
David Jamesc0f158a2011-02-22 16:07:29 -0800600
David Jamese2488642011-11-14 16:15:20 -0800601 # Record URL where prebuilts were uploaded.
602 url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
603 packages_url_suffix.rstrip('/'))
604
605 if git_sync:
David James4058b0d2011-12-08 21:24:50 -0800606 git_file = DeterminePrebuiltConfFile(self._build_path, target)
Mike Frysinger86509232014-05-24 13:18:37 -0400607 RevGitFile(git_file, {key: url_value}, dryrun=self._dryrun)
Matt Tennante8179042013-10-01 15:47:32 -0700608
David Jamese2488642011-11-14 16:15:20 -0800609 if sync_binhost_conf:
Matt Tennante8179042013-10-01 15:47:32 -0700610 # Update the binhost configuration file in git.
David Jamese2488642011-11-14 16:15:20 -0800611 binhost_conf = os.path.join(self._build_path, self._binhost_conf_dir,
David James4058b0d2011-12-08 21:24:50 -0800612 'target', '%s-%s.conf' % (target, key))
David Jamese2488642011-11-14 16:15:20 -0800613 UpdateBinhostConfFile(binhost_conf, key, url_value)
David James05bcb2b2011-02-09 09:25:47 -0800614
615
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700616def _AddSlaveBoard(_option, _opt_str, value, parser):
617 """Callback that adds a slave board to the list of slave targets."""
David James4058b0d2011-12-08 21:24:50 -0800618 parser.values.slave_targets.append(BuildTarget(value))
619
620
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700621def _AddSlaveProfile(_option, _opt_str, value, parser):
622 """Callback that adds a slave profile to the list of slave targets."""
David James4058b0d2011-12-08 21:24:50 -0800623 if not parser.values.slave_targets:
Mike Frysinger86509232014-05-24 13:18:37 -0400624 parser.error('Must specify --slave-board before --slave-profile')
David James4058b0d2011-12-08 21:24:50 -0800625 if parser.values.slave_targets[-1].profile is not None:
Mike Frysinger86509232014-05-24 13:18:37 -0400626 parser.error('Cannot specify --slave-profile twice for same board')
David James4058b0d2011-12-08 21:24:50 -0800627 parser.values.slave_targets[-1].profile = value
628
629
Mike Frysinger86509232014-05-24 13:18:37 -0400630def ParseOptions(argv):
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700631 """Returns options given by the user and the target specified.
632
Mike Frysinger86509232014-05-24 13:18:37 -0400633 Args:
634 argv: The args to parse.
635
Mike Frysinger1a736a82013-12-12 01:50:59 -0500636 Returns:
637 A tuple containing a parsed options object and BuildTarget.
638 The target instance is None if no board is specified.
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700639 """
Mike Frysinger86509232014-05-24 13:18:37 -0400640 parser = commandline.OptionParser()
David James8c846492011-01-25 17:07:29 -0800641 parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url',
642 default=_BINHOST_BASE_URL,
643 help='Base URL to use for binhost in make.conf updates')
644 parser.add_option('', '--previous-binhost-url', action='append',
645 default=[], dest='previous_binhost_url',
646 help='Previous binhost URL')
647 parser.add_option('-b', '--board', dest='board', default=None,
648 help='Board type that was built on this machine')
Zdenek Behan62a57792012-08-31 15:09:08 +0200649 parser.add_option('-B', '--prepackaged-tarball', dest='prepackaged_tarball',
650 default=None,
651 help='Board tarball prebuilt outside of this script.')
Mike Frysinger9e979b92012-11-29 02:55:09 -0500652 parser.add_option('--toolchain-tarball', dest='toolchain_tarballs',
653 action='append', default=[],
654 help='Redistributable toolchain tarball.')
655 parser.add_option('--toolchain-upload-path', default='',
656 help='Path to place toolchain tarballs in the sdk tree.')
David James4058b0d2011-12-08 21:24:50 -0800657 parser.add_option('', '--profile', dest='profile', default=None,
658 help='Profile that was built on this machine')
659 parser.add_option('', '--slave-board', default=[], action='callback',
660 dest='slave_targets', type='string',
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700661 callback=_AddSlaveBoard,
David James4058b0d2011-12-08 21:24:50 -0800662 help='Board type that was built on a slave machine. To '
663 'add a profile to this board, use --slave-profile.')
664 parser.add_option('', '--slave-profile', action='callback', type='string',
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700665 callback=_AddSlaveProfile,
David James4058b0d2011-12-08 21:24:50 -0800666 help='Board profile that was built on a slave machine. '
667 'Applies to previous slave board.')
David James8c846492011-01-25 17:07:29 -0800668 parser.add_option('-p', '--build-path', dest='build_path',
David James05bcb2b2011-02-09 09:25:47 -0800669 help='Path to the directory containing the chroot')
David James615e5b52011-06-03 11:10:15 -0700670 parser.add_option('', '--packages', action='append',
671 default=[], dest='packages',
672 help='Only include the specified packages. '
673 '(Default is to include all packages.)')
David James8c846492011-01-25 17:07:29 -0800674 parser.add_option('-s', '--sync-host', dest='sync_host',
675 default=False, action='store_true',
676 help='Sync host prebuilts')
677 parser.add_option('-g', '--git-sync', dest='git_sync',
678 default=False, action='store_true',
David Jamese2488642011-11-14 16:15:20 -0800679 help='Enable git version sync (This commits to a repo.) '
680 'This is used by full builders to commit directly '
681 'to board overlays.')
David James8c846492011-01-25 17:07:29 -0800682 parser.add_option('-u', '--upload', dest='upload',
683 default=None,
684 help='Upload location')
685 parser.add_option('-V', '--prepend-version', dest='prepend_version',
686 default=None,
687 help='Add an identifier to the front of the version')
688 parser.add_option('-f', '--filters', dest='filters', action='store_true',
689 default=False,
690 help='Turn on filtering of private ebuild packages')
691 parser.add_option('-k', '--key', dest='key',
692 default='PORTAGE_BINHOST',
693 help='Key to update in make.conf / binhost.conf')
David James8ece7ee2011-06-29 16:02:30 -0700694 parser.add_option('', '--set-version', dest='set_version',
695 default=None,
696 help='Specify the version string')
David James8c846492011-01-25 17:07:29 -0800697 parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
698 default=False, action='store_true',
David Jamese2488642011-11-14 16:15:20 -0800699 help='Update binhost.conf in chromiumos-overlay or '
700 'chromeos-overlay. Commit the changes, but don\'t '
701 'push them. This is used for preflight binhosts.')
David James32b0b2f2011-07-13 20:56:50 -0700702 parser.add_option('', '--binhost-conf-dir', dest='binhost_conf_dir',
David James32b0b2f2011-07-13 20:56:50 -0700703 help='Directory to commit binhost config with '
704 '--sync-binhost-conf.')
David Jamesfd0b0852011-02-23 11:15:36 -0800705 parser.add_option('-P', '--private', dest='private', action='store_true',
706 default=False, help='Mark gs:// uploads as private.')
David James8ece7ee2011-06-29 16:02:30 -0700707 parser.add_option('', '--skip-upload', dest='skip_upload',
708 action='store_true', default=False,
709 help='Skip upload step.')
David James8fa34ea2011-04-15 13:00:20 -0700710 parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
711 action='store_true', default=False,
712 help='Upload board tarball to Google Storage.')
Mike Frysinger86509232014-05-24 13:18:37 -0400713 parser.add_option('-n', '--dry-run', dest='dryrun',
David James27fa7d12011-06-29 17:24:14 -0700714 action='store_true', default=False,
715 help='Don\'t push or upload prebuilts.')
David James8c846492011-01-25 17:07:29 -0800716
Mike Frysinger86509232014-05-24 13:18:37 -0400717 options, args = parser.parse_args(argv)
David James8c846492011-01-25 17:07:29 -0800718 if not options.build_path:
Mike Frysinger86509232014-05-24 13:18:37 -0400719 parser.error('you need provide a chroot path')
David James8ece7ee2011-06-29 16:02:30 -0700720 if not options.upload and not options.skip_upload:
Mike Frysinger86509232014-05-24 13:18:37 -0400721 parser.error('you need to provide an upload location using -u')
David James8ece7ee2011-06-29 16:02:30 -0700722 if not options.set_version and options.skip_upload:
Mike Frysinger86509232014-05-24 13:18:37 -0400723 parser.error('If you are using --skip-upload, you must specify a '
724 'version number using --set-version.')
David James9417f272011-05-26 13:24:47 -0700725 if args:
Mike Frysinger86509232014-05-24 13:18:37 -0400726 parser.error('invalid arguments passed to upload_prebuilts: %r' % args)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700727
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700728 target = None
729 if options.board:
730 target = BuildTarget(options.board, options.profile)
731
732 if target in options.slave_targets:
Mike Frysinger86509232014-05-24 13:18:37 -0400733 parser.error('--board/--profile must not also be a slave target.')
David Jamese2488642011-11-14 16:15:20 -0800734
David James4058b0d2011-12-08 21:24:50 -0800735 if len(set(options.slave_targets)) != len(options.slave_targets):
Mike Frysinger86509232014-05-24 13:18:37 -0400736 parser.error('--slave-boards must not have duplicates.')
David Jamese2488642011-11-14 16:15:20 -0800737
David James4058b0d2011-12-08 21:24:50 -0800738 if options.slave_targets and options.git_sync:
Mike Frysinger86509232014-05-24 13:18:37 -0400739 parser.error('--slave-boards is not compatible with --git-sync')
David Jamese2488642011-11-14 16:15:20 -0800740
David James8ece7ee2011-06-29 16:02:30 -0700741 if (options.upload_board_tarball and options.skip_upload and
742 options.board == 'amd64-host'):
Mike Frysinger86509232014-05-24 13:18:37 -0400743 parser.error('--skip-upload is not compatible with '
744 '--upload-board-tarball and --board=amd64-host')
David James8fa34ea2011-04-15 13:00:20 -0700745
David James8ece7ee2011-06-29 16:02:30 -0700746 if (options.upload_board_tarball and not options.skip_upload and
747 not options.upload.startswith('gs://')):
Mike Frysinger86509232014-05-24 13:18:37 -0400748 parser.error('--upload-board-tarball only works with gs:// URLs.\n'
749 '--upload must be a gs:// URL.')
David James8fa34ea2011-04-15 13:00:20 -0700750
Zdenek Behan86c15e92012-10-13 00:55:47 +0200751 if options.upload_board_tarball and options.prepackaged_tarball is None:
Mike Frysinger86509232014-05-24 13:18:37 -0400752 parser.error('--upload-board-tarball requires --prepackaged-tarball')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200753
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700754 if options.private:
755 if options.sync_host:
Mike Frysinger86509232014-05-24 13:18:37 -0400756 parser.error('--private and --sync-host/-s cannot be specified '
757 'together; we do not support private host prebuilts')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700758
David James8ece7ee2011-06-29 16:02:30 -0700759 if not options.upload or not options.upload.startswith('gs://'):
Mike Frysinger86509232014-05-24 13:18:37 -0400760 parser.error('--private is only valid for gs:// URLs; '
761 '--upload must be a gs:// URL.')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700762
763 if options.binhost_base_url != _BINHOST_BASE_URL:
Mike Frysinger86509232014-05-24 13:18:37 -0400764 parser.error('when using --private the --binhost-base-url '
765 'is automatically derived.')
David James27fa7d12011-06-29 17:24:14 -0700766
David Jamesc31168e2014-06-05 14:40:05 -0700767 if options.sync_binhost_conf and not options.binhost_conf_dir:
768 parser.error('--sync-binhost-conf requires --binhost-conf-dir')
769
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700770 return options, target
David Jamesc0f158a2011-02-22 16:07:29 -0800771
Mike Frysingercc838832014-05-24 13:10:30 -0400772
Mike Frysinger86509232014-05-24 13:18:37 -0400773def main(argv):
David Jamesdb401072011-06-10 12:17:16 -0700774 # Set umask to a sane value so that files created as root are readable.
Mike Frysinger60ec1012013-10-21 00:11:10 -0400775 os.umask(0o22)
David Jamesdb401072011-06-10 12:17:16 -0700776
Mike Frysinger86509232014-05-24 13:18:37 -0400777 options, target = ParseOptions(argv)
David Jamesc0f158a2011-02-22 16:07:29 -0800778
David James05bcb2b2011-02-09 09:25:47 -0800779 # Calculate a list of Packages index files to compare against. Whenever we
780 # upload a package, we check to make sure it's not already stored in one of
781 # the packages files we uploaded. This list of packages files might contain
782 # both board and host packages.
David Jamesce093af2011-02-23 15:21:58 -0800783 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url)
David James8c846492011-01-25 17:07:29 -0800784
David James8ece7ee2011-06-29 16:02:30 -0700785 if options.set_version:
786 version = options.set_version
787 else:
788 version = GetVersion()
Matt Tennante8179042013-10-01 15:47:32 -0700789
David Jamesc0f158a2011-02-22 16:07:29 -0800790 if options.prepend_version:
791 version = '%s-%s' % (options.prepend_version, version)
792
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700793 acl = 'public-read'
794 binhost_base_url = options.binhost_base_url
795
David Jamesadd21432013-05-21 10:04:07 -0700796 if options.private:
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700797 binhost_base_url = options.upload
David Jamesadd21432013-05-21 10:04:07 -0700798 if target:
Peter Mayo950e41a2014-02-06 21:07:33 +0000799 board_path = GetBoardOverlay(options.build_path, target)
David Jamesadd21432013-05-21 10:04:07 -0700800 acl = os.path.join(board_path, _GOOGLESTORAGE_ACL_FILE)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700801
802 uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
David James615e5b52011-06-03 11:10:15 -0700803 pkg_indexes, options.build_path,
David James27fa7d12011-06-29 17:24:14 -0700804 options.packages, options.skip_upload,
Mike Frysinger86509232014-05-24 13:18:37 -0400805 options.binhost_conf_dir, options.dryrun,
Mike Frysinger8092a632014-05-24 13:25:46 -0400806 target, options.slave_targets, version)
David Jamesc0f158a2011-02-22 16:07:29 -0800807
David James8c846492011-01-25 17:07:29 -0800808 if options.sync_host:
Mike Frysinger8092a632014-05-24 13:25:46 -0400809 uploader.SyncHostPrebuilts(options.key, options.git_sync,
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700810 options.sync_binhost_conf)
David James8c846492011-01-25 17:07:29 -0800811
Chris Sosa62c8ff52012-06-04 15:03:12 -0700812 if options.board or options.slave_targets:
Mike Frysinger8092a632014-05-24 13:25:46 -0400813 uploader.SyncBoardPrebuilts(options.key, options.git_sync,
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700814 options.sync_binhost_conf,
Zdenek Behan62a57792012-08-31 15:09:08 +0200815 options.upload_board_tarball,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500816 options.prepackaged_tarball,
817 options.toolchain_tarballs,
818 options.toolchain_upload_path)