blob: 58d78b8e6fc20d5422d77d148b92b3fed9c903c5 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Don Garrett13cbbff2016-08-09 14:18:38 -07002 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
David James8c846492011-01-25 17:07:29 -08003# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Brian Harringaf019fb2012-05-10 15:06:13 -07006"""This script is used to upload host prebuilts as well as board BINHOSTS.
David James8c846492011-01-25 17:07:29 -08007
David James015af872012-06-19 15:24:36 -07008Prebuilts are uploaded using gsutil to Google Storage. After these prebuilts
9are successfully uploaded, a file is updated with the proper BINHOST version.
David James8c846492011-01-25 17:07:29 -080010
11To read more about prebuilts/binhost binary packages please refer to:
David James015af872012-06-19 15:24:36 -070012http://goto/chromeos-prebuilts
David James8c846492011-01-25 17:07:29 -080013
14Example of uploading prebuilt amd64 host files to Google Storage:
David Jamesc5cbd472012-06-19 16:25:45 -070015upload_prebuilts -p /b/cbuild/build -s -u gs://chromeos-prebuilt
David James8c846492011-01-25 17:07:29 -080016
17Example of uploading x86-dogfood binhosts to Google Storage:
David Jamesc5cbd472012-06-19 16:25:45 -070018upload_prebuilts -b x86-dogfood -p /b/cbuild/build/ -u gs://chromeos-prebuilt -g
David James8c846492011-01-25 17:07:29 -080019"""
20
Mike Frysingerff441bf2014-05-24 13:47:21 -040021from __future__ import print_function
22
Mike Frysingerb87bb782015-06-04 02:46:50 -040023import argparse
Chris Sosa1dc96132012-05-11 15:40:50 -070024import datetime
Mike Frysinger540883b2014-05-24 13:46:16 -040025import functools
David Jamesb26b9312014-12-15 11:26:46 -080026import glob
Chris Sosa1dc96132012-05-11 15:40:50 -070027import multiprocessing
Chris Sosa1dc96132012-05-11 15:40:50 -070028import os
29import sys
Mike Frysinger212e4292014-05-24 15:15:44 -040030import tempfile
Chris Sosa1dc96132012-05-11 15:40:50 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
David James14e97772014-06-04 18:44:49 -070033from chromite.cbuildbot import commands
David James6450a0a2012-12-04 07:59:53 -080034from chromite.lib import binpkg
Mike Frysinger86509232014-05-24 13:18:37 -040035from chromite.lib import commandline
Chris Sosa1dc96132012-05-11 15:40:50 -070036from chromite.lib import cros_build_lib
Ralph Nathan446aee92015-03-23 14:44:56 -070037from chromite.lib import cros_logging as logging
David James97d95872012-11-16 15:09:56 -080038from chromite.lib import git
Brian Harring7904b482012-08-08 02:54:12 -070039from chromite.lib import gs
Brian Harringaf019fb2012-05-10 15:06:13 -070040from chromite.lib import osutils
David James6450a0a2012-12-04 07:59:53 -080041from chromite.lib import parallel
Alex Deymo075c2292014-09-04 18:31:50 -070042from chromite.lib import portage_util
Mike Frysingercdd98762017-09-12 00:05:37 -040043from chromite.lib import retry_util
Mike Frysinger8e727a32013-01-16 16:57:53 -050044from chromite.lib import toolchain
Chris Sosa1dc96132012-05-11 15:40:50 -070045
David James015af872012-06-19 15:24:36 -070046# How many times to retry uploads.
47_RETRIES = 10
48
49# Multiplier for how long to sleep (in seconds) between retries; will delay
50# (1*sleep) the first time, then (2*sleep), continuing via attempt * sleep.
51_SLEEP_TIME = 60
52
David James5ab67e32014-10-24 08:19:59 -070053# The length of time (in seconds) that Portage should wait before refetching
54# binpkgs from the same binhost. We don't ever modify binhosts, so this should
55# be something big.
56_BINPKG_TTL = 60 * 60 * 24 * 365
57
David James8c846492011-01-25 17:07:29 -080058_HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
David James05bcb2b2011-02-09 09:25:47 -080059_CATEGORIES_PATH = 'chroot/etc/portage/categories'
David James615e5b52011-06-03 11:10:15 -070060_PYM_PATH = 'chroot/usr/lib/portage/pym'
David James4058b0d2011-12-08 21:24:50 -080061_HOST_ARCH = 'amd64'
David James8c846492011-01-25 17:07:29 -080062_BOARD_PATH = 'chroot/build/%(board)s'
David James4058b0d2011-12-08 21:24:50 -080063_REL_BOARD_PATH = 'board/%(target)s/%(version)s'
64_REL_HOST_PATH = 'host/%(host_arch)s/%(target)s/%(version)s'
David James8c846492011-01-25 17:07:29 -080065# Private overlays to look at for builds to filter
66# relative to build path
67_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
Gabe Black40169e62014-06-17 15:23:47 -070068_GOOGLESTORAGE_GSUTIL_FILE = 'googlestorage_acl.txt'
David James3753d942014-04-23 10:55:48 -070069_BINHOST_BASE_URL = 'gs://chromeos-prebuilt'
David James8c846492011-01-25 17:07:29 -080070_PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
71# Created in the event of new host targets becoming available
72_PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
73 'make.conf.amd64-host')}
David James8c846492011-01-25 17:07:29 -080074
75
David James4058b0d2011-12-08 21:24:50 -080076class BuildTarget(object):
77 """A board/variant/profile tuple."""
78
79 def __init__(self, board_variant, profile=None):
80 self.board_variant = board_variant
81 self.board, _, self.variant = board_variant.partition('_')
82 self.profile = profile
83
84 def __str__(self):
85 if self.profile:
86 return '%s_%s' % (self.board_variant, self.profile)
87 else:
88 return self.board_variant
89
90 def __eq__(self, other):
91 return str(other) == str(self)
92
93 def __hash__(self):
94 return hash(str(self))
95
96
David James8c846492011-01-25 17:07:29 -080097def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
98 """Update the key in file with the value passed.
Mike Frysinger1a736a82013-12-12 01:50:59 -050099
David James8c846492011-01-25 17:07:29 -0800100 File format:
101 key="value"
102 Note quotes are added automatically
103
104 Args:
105 filename: Name of file to modify.
106 value: Value to write with the key.
107 key: The variable key to update. (Default: PORTAGE_BINHOST)
David Jamesb26b9312014-12-15 11:26:46 -0800108
109 Returns:
110 True if changes were made to the file.
David James8c846492011-01-25 17:07:29 -0800111 """
112 if os.path.exists(filename):
113 file_fh = open(filename)
114 else:
115 file_fh = open(filename, 'w+')
116 file_lines = []
117 found = False
David Jamesb26b9312014-12-15 11:26:46 -0800118 made_changes = False
David James8c846492011-01-25 17:07:29 -0800119 keyval_str = '%(key)s=%(value)s'
120 for line in file_fh:
121 # Strip newlines from end of line. We already add newlines below.
122 line = line.rstrip("\n")
123
124 if len(line.split('=')) != 2:
125 # Skip any line that doesn't fit key=val.
126 file_lines.append(line)
127 continue
128
129 file_var, file_val = line.split('=')
130 if file_var == key:
131 found = True
Mike Frysingerff441bf2014-05-24 13:47:21 -0400132 print('Updating %s=%s to %s="%s"' % (file_var, file_val, key, value))
David James8c846492011-01-25 17:07:29 -0800133 value = '"%s"' % value
David Jamesb26b9312014-12-15 11:26:46 -0800134 made_changes |= (file_val != value)
David James8c846492011-01-25 17:07:29 -0800135 file_lines.append(keyval_str % {'key': key, 'value': value})
136 else:
137 file_lines.append(keyval_str % {'key': file_var, 'value': file_val})
138
139 if not found:
Brian Harring2a014302012-05-12 00:53:33 -0700140 value = '"%s"' % value
David Jamesb26b9312014-12-15 11:26:46 -0800141 made_changes = True
David James8c846492011-01-25 17:07:29 -0800142 file_lines.append(keyval_str % {'key': key, 'value': value})
143
144 file_fh.close()
145 # write out new file
Brian Harringaf019fb2012-05-10 15:06:13 -0700146 osutils.WriteFile(filename, '\n'.join(file_lines) + '\n')
David Jamesb26b9312014-12-15 11:26:46 -0800147 return made_changes
David James8c846492011-01-25 17:07:29 -0800148
149
Mike Frysinger5b34d732013-01-17 15:11:58 -0500150def RevGitFile(filename, data, retries=5, dryrun=False):
David James8c846492011-01-25 17:07:29 -0800151 """Update and push the git file.
152
Mike Frysinger5b34d732013-01-17 15:11:58 -0500153 Args:
154 filename: file to modify that is in a git repo already
155 data: A dict of key/values to update in |filename|
156 retries: The number of times to retry before giving up, default: 5
Matt Tennante8179042013-10-01 15:47:32 -0700157 dryrun: If True, do not actually commit the change.
David James8c846492011-01-25 17:07:29 -0800158 """
159 prebuilt_branch = 'prebuilt_branch'
David James1b6e67a2011-05-19 21:32:38 -0700160 cwd = os.path.abspath(os.path.dirname(filename))
David James97d95872012-11-16 15:09:56 -0800161 commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
Mike Frysinger5b34d732013-01-17 15:11:58 -0500162 description = '%s: updating %s' % (os.path.basename(filename),
163 ', '.join(data.keys()))
164 # UpdateLocalFile will print out the keys/values for us.
Mike Frysingerff441bf2014-05-24 13:47:21 -0400165 print('Revving git file %s' % filename)
David James66009462012-03-25 10:08:38 -0700166
David James8c846492011-01-25 17:07:29 -0800167 try:
David James97d95872012-11-16 15:09:56 -0800168 git.CreatePushBranch(prebuilt_branch, cwd)
Mike Frysinger5b34d732013-01-17 15:11:58 -0500169 for key, value in data.iteritems():
170 UpdateLocalFile(filename, value, key)
David James97d95872012-11-16 15:09:56 -0800171 git.RunGit(cwd, ['add', filename])
172 git.RunGit(cwd, ['commit', '-m', description])
173 git.PushWithRetry(prebuilt_branch, cwd, dryrun=dryrun, retries=retries)
David James8c846492011-01-25 17:07:29 -0800174 finally:
Hidehiko Abecb26e282017-08-22 16:28:16 +0900175 # We reset the index and the working tree state in case there are any
176 # uncommitted or pending changes, but we don't change any existing commits.
177 git.RunGit(cwd, ['reset', '--hard'])
178
179 # Check out the last good commit as a sanity fallback.
David James67d73252013-09-19 17:33:12 -0700180 git.RunGit(cwd, ['checkout', commit])
David James8c846492011-01-25 17:07:29 -0800181
182
183def GetVersion():
184 """Get the version to put in LATEST and update the git version with."""
Mike Frysinger0b92cfe2012-12-13 02:13:45 -0500185 return datetime.datetime.now().strftime('%Y.%m.%d.%H%M%S')
David James8c846492011-01-25 17:07:29 -0800186
187
Mike Frysinger540883b2014-05-24 13:46:16 -0400188def _GsUpload(gs_context, acl, local_file, remote_file):
David James8c846492011-01-25 17:07:29 -0800189 """Upload to GS bucket.
190
191 Args:
Mike Frysinger540883b2014-05-24 13:46:16 -0400192 gs_context: A lib.gs.GSContext instance.
193 acl: The ACL to use for uploading the file.
Matt Tennante8179042013-10-01 15:47:32 -0700194 local_file: The local file to be uploaded.
195 remote_file: The remote location to upload to.
David James8c846492011-01-25 17:07:29 -0800196 """
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700197 CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
198 'authenticated-read', 'bucket-owner-full-control',
199 'public-read-write']
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700200 if acl in CANNED_ACLS:
David James9374aac2013-10-08 16:00:17 -0700201 gs_context.Copy(local_file, remote_file, acl=acl)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700202 else:
203 # For private uploads we assume that the overlay board is set up properly
David James015af872012-06-19 15:24:36 -0700204 # and a googlestore_acl.xml is present. Otherwise, this script errors.
David James7b7d7c32015-04-02 13:48:17 -0700205 # We set version=0 here to ensure that the ACL is set only once (see
206 # http://b/15883752#comment54).
207 try:
208 gs_context.Copy(local_file, remote_file, version=0)
209 except gs.GSContextPreconditionFailed as ex:
210 # If we received a GSContextPreconditionFailed error, we know that the
211 # file exists now, but we don't know whether our specific update
212 # succeeded. See http://b/15883752#comment62
213 logging.warning(
214 'Assuming upload succeeded despite PreconditionFailed errors: %s', ex)
215
Gabe Black40169e62014-06-17 15:23:47 -0700216 if acl.endswith('.xml'):
217 # Apply the passed in ACL xml file to the uploaded object.
218 gs_context.SetACL(remote_file, acl=acl)
219 else:
Mike Frysingercdd98762017-09-12 00:05:37 -0400220 # Some versions of gsutil bubble up precondition failures even when we
221 # didn't request it due to how ACL changes happen internally to gsutil.
222 # https://crbug.com/763450
223 retry_util.RetryException(
224 gs.GSContextPreconditionFailed, 3, gs_context.ChangeACL,
225 remote_file, acl_args_file=acl)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700226
Mike Frysingercc838832014-05-24 13:10:30 -0400227
Mike Frysinger540883b2014-05-24 13:46:16 -0400228def RemoteUpload(gs_context, acl, files, pool=10):
David James8c846492011-01-25 17:07:29 -0800229 """Upload to google storage.
230
231 Create a pool of process and call _GsUpload with the proper arguments.
232
233 Args:
Mike Frysinger540883b2014-05-24 13:46:16 -0400234 gs_context: A lib.gs.GSContext instance.
David Jamesfd0b0852011-02-23 11:15:36 -0800235 acl: The canned acl used for uploading. acl can be one of: "public-read",
236 "public-read-write", "authenticated-read", "bucket-owner-read",
237 "bucket-owner-full-control", or "private".
David James8c846492011-01-25 17:07:29 -0800238 files: dictionary with keys to local files and values to remote path.
239 pool: integer of maximum proesses to have at the same time.
240
241 Returns:
242 Return a set of tuple arguments of the failed uploads
243 """
Mike Frysinger540883b2014-05-24 13:46:16 -0400244 upload = functools.partial(_GsUpload, gs_context, acl)
245 tasks = [[key, value] for key, value in files.iteritems()]
246 parallel.RunTasksInProcessPool(upload, tasks, pool)
David James8c846492011-01-25 17:07:29 -0800247
248
249def GenerateUploadDict(base_local_path, base_remote_path, pkgs):
250 """Build a dictionary of local remote file key pairs to upload.
251
252 Args:
253 base_local_path: The base path to the files on the local hard drive.
Matt Tennante8179042013-10-01 15:47:32 -0700254 base_remote_path: The base path to the remote paths.
David James8c846492011-01-25 17:07:29 -0800255 pkgs: The packages to upload.
256
257 Returns:
258 Returns a dictionary of local_path/remote_path pairs
259 """
260 upload_files = {}
261 for pkg in pkgs:
262 suffix = pkg['CPV'] + '.tbz2'
263 local_path = os.path.join(base_local_path, suffix)
David James7b7d7c32015-04-02 13:48:17 -0700264 assert os.path.exists(local_path), '%s does not exist' % local_path
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800265 upload_files[local_path] = os.path.join(base_remote_path, suffix)
David James8c846492011-01-25 17:07:29 -0800266
Bertrand SIMONNET811bcde2014-11-20 15:21:25 -0800267 if pkg.get('DEBUG_SYMBOLS') == 'yes':
268 debugsuffix = pkg['CPV'] + '.debug.tbz2'
269 local_path = os.path.join(base_local_path, debugsuffix)
270 assert os.path.exists(local_path)
271 upload_files[local_path] = os.path.join(base_remote_path, debugsuffix)
Bertrand SIMONNET22e828b2014-11-11 16:27:06 -0800272
David James8c846492011-01-25 17:07:29 -0800273 return upload_files
274
Mike Frysingercc838832014-05-24 13:10:30 -0400275
Peter Mayo950e41a2014-02-06 21:07:33 +0000276def GetBoardOverlay(build_path, target):
David Jamese5867812012-10-19 12:02:20 -0700277 """Get the path to the board variant.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500278
Mike Frysinger6f3c48e2015-05-06 02:38:51 -0400279 Args:
280 build_path: The path to the root of the build directory
281 target: The target board as a BuildTarget object.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500282
Mike Frysinger6f3c48e2015-05-06 02:38:51 -0400283 Returns:
284 The last overlay configured for the given board as a string.
David James8c846492011-01-25 17:07:29 -0800285 """
David Jamese5867812012-10-19 12:02:20 -0700286 board = target.board_variant
Alex Deymo075c2292014-09-04 18:31:50 -0700287 overlays = portage_util.FindOverlays(constants.BOTH_OVERLAYS, board,
288 buildroot=build_path)
David Jamese5867812012-10-19 12:02:20 -0700289 # We only care about the last entry.
290 return overlays[-1]
David James8c846492011-01-25 17:07:29 -0800291
292
293def DeterminePrebuiltConfFile(build_path, target):
294 """Determine the prebuilt.conf file that needs to be updated for prebuilts.
295
Mike Frysinger1a736a82013-12-12 01:50:59 -0500296 Args:
297 build_path: The path to the root of the build directory
298 target: String representation of the board. This includes host and board
299 targets
David James8c846492011-01-25 17:07:29 -0800300
Mike Frysinger1a736a82013-12-12 01:50:59 -0500301 Returns:
302 A string path to a prebuilt.conf file to be updated.
David James8c846492011-01-25 17:07:29 -0800303 """
David James4058b0d2011-12-08 21:24:50 -0800304 if _HOST_ARCH == target:
David James8c846492011-01-25 17:07:29 -0800305 # We are host.
306 # Without more examples of hosts this is a kludge for now.
307 # TODO(Scottz): as new host targets come online expand this to
308 # work more like boards.
Chris Sosa471532a2011-02-01 15:10:06 -0800309 make_path = _PREBUILT_MAKE_CONF[target]
David James8c846492011-01-25 17:07:29 -0800310 else:
311 # We are a board
Peter Mayo950e41a2014-02-06 21:07:33 +0000312 board = GetBoardOverlay(build_path, target)
David James8c846492011-01-25 17:07:29 -0800313 make_path = os.path.join(board, 'prebuilt.conf')
314
315 return make_path
316
317
318def UpdateBinhostConfFile(path, key, value):
319 """Update binhost config file file with key=value.
320
321 Args:
322 path: Filename to update.
323 key: Key to update.
324 value: New value for key.
325 """
David Jamesb26b9312014-12-15 11:26:46 -0800326 cwd, filename = os.path.split(os.path.abspath(path))
Brian Harringaf019fb2012-05-10 15:06:13 -0700327 osutils.SafeMakedirs(cwd)
David Jamesf6e8fb72013-05-10 08:58:43 -0700328 if not git.GetCurrentBranch(cwd):
329 git.CreatePushBranch(constants.STABLE_EBUILD_BRANCH, cwd, sync=False)
Brian Harring22edb442012-05-11 23:55:18 -0700330 osutils.WriteFile(path, '', mode='a')
David Jamesb26b9312014-12-15 11:26:46 -0800331 if UpdateLocalFile(path, value, key):
332 desc = '%s: %s %s' % (filename, 'updating' if value else 'clearing', key)
333 git.AddPath(path)
334 git.Commit(cwd, desc)
David James8c846492011-01-25 17:07:29 -0800335
Achuith Bhandarkar03d924a2018-01-02 10:50:44 -0800336def GenerateHtmlIndex(files, index, board, version, remote_location):
Mike Frysinger212e4292014-05-24 15:15:44 -0400337 """Given the list of |files|, generate an index.html at |index|.
338
339 Args:
340 files: The list of files to link to.
341 index: The path to the html index.
342 board: Name of the board this index is for.
343 version: Build version this index is for.
Achuith Bhandarkar03d924a2018-01-02 10:50:44 -0800344 remote_location: Remote gs location prebuilts are uploaded to.
Mike Frysinger212e4292014-05-24 15:15:44 -0400345 """
Don Garrett13cbbff2016-08-09 14:18:38 -0700346 title = 'Package Prebuilt Index: %s / %s' % (board, version)
Mike Frysinger212e4292014-05-24 15:15:44 -0400347
348 files = files + [
349 '.|Google Storage Index',
350 '..|',
351 ]
Achuith Bhandarkar03d924a2018-01-02 10:50:44 -0800352 commands.GenerateHtmlIndex(index, files, title=title,
353 url_base=gs.GsUrlToHttp(remote_location))
Mike Frysinger212e4292014-05-24 15:15:44 -0400354
355
David Jamesce093af2011-02-23 15:21:58 -0800356def _GrabAllRemotePackageIndexes(binhost_urls):
David James05bcb2b2011-02-09 09:25:47 -0800357 """Grab all of the packages files associated with a list of binhost_urls.
358
David James05bcb2b2011-02-09 09:25:47 -0800359 Args:
360 binhost_urls: The URLs for the directories containing the Packages files we
361 want to grab.
David James05bcb2b2011-02-09 09:25:47 -0800362
363 Returns:
364 A list of PackageIndex objects.
365 """
366 pkg_indexes = []
367 for url in binhost_urls:
David James32faafe2012-06-08 14:25:03 -0700368 pkg_index = binpkg.GrabRemotePackageIndex(url)
David James05bcb2b2011-02-09 09:25:47 -0800369 if pkg_index:
370 pkg_indexes.append(pkg_index)
David James05bcb2b2011-02-09 09:25:47 -0800371 return pkg_indexes
372
373
David Jamesc0f158a2011-02-22 16:07:29 -0800374class PrebuiltUploader(object):
375 """Synchronize host and board prebuilts."""
David James8c846492011-01-25 17:07:29 -0800376
Mike Frysinger86509232014-05-24 13:18:37 -0400377 def __init__(self, upload_location, acl, binhost_base_url, pkg_indexes,
378 build_path, packages, skip_upload, binhost_conf_dir, dryrun,
Mike Frysinger8092a632014-05-24 13:25:46 -0400379 target, slave_targets, version):
David Jamesc0f158a2011-02-22 16:07:29 -0800380 """Constructor for prebuilt uploader object.
David James8c846492011-01-25 17:07:29 -0800381
David Jamesc0f158a2011-02-22 16:07:29 -0800382 This object can upload host or prebuilt files to Google Storage.
David James8c846492011-01-25 17:07:29 -0800383
David Jamesc0f158a2011-02-22 16:07:29 -0800384 Args:
385 upload_location: The upload location.
David Jamesfd0b0852011-02-23 11:15:36 -0800386 acl: The canned acl used for uploading to Google Storage. acl can be one
387 of: "public-read", "public-read-write", "authenticated-read",
Matt Tennante8179042013-10-01 15:47:32 -0700388 "bucket-owner-read", "bucket-owner-full-control", "project-private",
389 or "private" (see "gsutil help acls"). If we are not uploading to
390 Google Storage, this parameter is unused.
David Jamesfd0b0852011-02-23 11:15:36 -0800391 binhost_base_url: The URL used for downloading the prebuilts.
David Jamesc0f158a2011-02-22 16:07:29 -0800392 pkg_indexes: Old uploaded prebuilts to compare against. Instead of
393 uploading duplicate files, we just link to the old files.
David James615e5b52011-06-03 11:10:15 -0700394 build_path: The path to the directory containing the chroot.
395 packages: Packages to upload.
David James32b0b2f2011-07-13 20:56:50 -0700396 skip_upload: Don't actually upload the tarballs.
397 binhost_conf_dir: Directory where to store binhost.conf files.
Mike Frysinger86509232014-05-24 13:18:37 -0400398 dryrun: Don't push or upload prebuilts.
David James4058b0d2011-12-08 21:24:50 -0800399 target: BuildTarget managed by this builder.
400 slave_targets: List of BuildTargets managed by slave builders.
Mike Frysinger8092a632014-05-24 13:25:46 -0400401 version: A unique string, intended to be included in the upload path,
402 which identifies the version number of the uploaded prebuilts.
David Jamesc0f158a2011-02-22 16:07:29 -0800403 """
404 self._upload_location = upload_location
David Jamesfd0b0852011-02-23 11:15:36 -0800405 self._acl = acl
David Jamesc0f158a2011-02-22 16:07:29 -0800406 self._binhost_base_url = binhost_base_url
407 self._pkg_indexes = pkg_indexes
David James615e5b52011-06-03 11:10:15 -0700408 self._build_path = build_path
409 self._packages = set(packages)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500410 self._found_packages = set()
David James8ece7ee2011-06-29 16:02:30 -0700411 self._skip_upload = skip_upload
David James32b0b2f2011-07-13 20:56:50 -0700412 self._binhost_conf_dir = binhost_conf_dir
Mike Frysinger86509232014-05-24 13:18:37 -0400413 self._dryrun = dryrun
David James4058b0d2011-12-08 21:24:50 -0800414 self._target = target
415 self._slave_targets = slave_targets
Mike Frysinger8092a632014-05-24 13:25:46 -0400416 self._version = version
Mike Frysinger540883b2014-05-24 13:46:16 -0400417 self._gs_context = gs.GSContext(retries=_RETRIES, sleep=_SLEEP_TIME,
418 dry_run=self._dryrun)
419
420 def _Upload(self, local_file, remote_file):
421 """Wrapper around _GsUpload"""
422 _GsUpload(self._gs_context, self._acl, local_file, remote_file)
David James615e5b52011-06-03 11:10:15 -0700423
424 def _ShouldFilterPackage(self, pkg):
425 if not self._packages:
426 return False
427 pym_path = os.path.abspath(os.path.join(self._build_path, _PYM_PATH))
David James710b7dc2012-02-07 16:49:59 -0800428 sys.path.insert(0, pym_path)
Don Garrett25f309a2014-03-19 14:02:12 -0700429 # pylint: disable=F0401
David James615e5b52011-06-03 11:10:15 -0700430 import portage.versions
431 cat, pkgname = portage.versions.catpkgsplit(pkg['CPV'])[0:2]
432 cp = '%s/%s' % (cat, pkgname)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500433 self._found_packages.add(cp)
David James615e5b52011-06-03 11:10:15 -0700434 return pkgname not in self._packages and cp not in self._packages
David James8c846492011-01-25 17:07:29 -0800435
David Jamesc0f158a2011-02-22 16:07:29 -0800436 def _UploadPrebuilt(self, package_path, url_suffix):
437 """Upload host or board prebuilt files to Google Storage space.
David James8c846492011-01-25 17:07:29 -0800438
David Jamesc0f158a2011-02-22 16:07:29 -0800439 Args:
440 package_path: The path to the packages dir.
David Jamesce093af2011-02-23 15:21:58 -0800441 url_suffix: The remote subdirectory where we should upload the packages.
David Jamesc0f158a2011-02-22 16:07:29 -0800442 """
David Jamesc0f158a2011-02-22 16:07:29 -0800443 # Process Packages file, removing duplicates and filtered packages.
David James32faafe2012-06-08 14:25:03 -0700444 pkg_index = binpkg.GrabLocalPackageIndex(package_path)
David Jamesc0f158a2011-02-22 16:07:29 -0800445 pkg_index.SetUploadLocation(self._binhost_base_url, url_suffix)
David James615e5b52011-06-03 11:10:15 -0700446 pkg_index.RemoveFilteredPackages(self._ShouldFilterPackage)
David Jamesc0f158a2011-02-22 16:07:29 -0800447 uploads = pkg_index.ResolveDuplicateUploads(self._pkg_indexes)
Mike Frysingerf5f809a2013-02-12 13:47:37 -0500448 unmatched_pkgs = self._packages - self._found_packages
449 if unmatched_pkgs:
Ralph Nathan446aee92015-03-23 14:44:56 -0700450 logging.warning('unable to match packages: %r' % unmatched_pkgs)
David James05bcb2b2011-02-09 09:25:47 -0800451
David Jamesc0f158a2011-02-22 16:07:29 -0800452 # Write Packages file.
David James5ab67e32014-10-24 08:19:59 -0700453 pkg_index.header['TTL'] = _BINPKG_TTL
David Jamesc0f158a2011-02-22 16:07:29 -0800454 tmp_packages_file = pkg_index.WriteToNamedTemporaryFile()
David James05bcb2b2011-02-09 09:25:47 -0800455
David Jamesc0f158a2011-02-22 16:07:29 -0800456 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
David James015af872012-06-19 15:24:36 -0700457 assert remote_location.startswith('gs://')
David James05bcb2b2011-02-09 09:25:47 -0800458
Alex Deymo541ea6f2014-12-23 01:18:14 -0800459 # Build list of files to upload. Manually include the dev-only files but
460 # skip them if not present.
461 # TODO(deymo): Upload dev-only-extras.tbz2 as dev-only-extras.tar.bz2
462 # outside packages/ directory. See crbug.com/448178 for details.
463 if os.path.exists(os.path.join(package_path, 'dev-only-extras.tbz2')):
464 uploads.append({'CPV': 'dev-only-extras'})
David James015af872012-06-19 15:24:36 -0700465 upload_files = GenerateUploadDict(package_path, remote_location, uploads)
466 remote_file = '%s/Packages' % remote_location.rstrip('/')
467 upload_files[tmp_packages_file.name] = remote_file
468
Mike Frysinger540883b2014-05-24 13:46:16 -0400469 RemoteUpload(self._gs_context, self._acl, upload_files)
David James8c846492011-01-25 17:07:29 -0800470
Mike Frysinger212e4292014-05-24 15:15:44 -0400471 with tempfile.NamedTemporaryFile(
472 prefix='chromite.upload_prebuilts.index.') as index:
473 GenerateHtmlIndex(
474 [x[len(remote_location) + 1:] for x in upload_files.values()],
Achuith Bhandarkar03d924a2018-01-02 10:50:44 -0800475 index.name, self._target, self._version, remote_location)
Mike Frysinger212e4292014-05-24 15:15:44 -0400476 self._Upload(index.name, '%s/index.html' % remote_location.rstrip('/'))
477
478 link_name = 'Prebuilts[%s]: %s' % (self._target, self._version)
479 url = '%s%s/index.html' % (gs.PUBLIC_BASE_HTTPS_URL,
480 remote_location[len(gs.BASE_GS_URL):])
Prathmesh Prabhu17f07422015-07-17 11:40:40 -0700481 logging.PrintBuildbotLink(link_name, url)
Mike Frysinger212e4292014-05-24 15:15:44 -0400482
Mike Frysinger8092a632014-05-24 13:25:46 -0400483 def _UploadSdkTarball(self, board_path, url_suffix, prepackaged,
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700484 toolchains_overlay_tarballs,
485 toolchains_overlay_upload_path,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500486 toolchain_tarballs, toolchain_upload_path):
487 """Upload a tarball of the sdk at the specified path to Google Storage.
David James8fa34ea2011-04-15 13:00:20 -0700488
489 Args:
490 board_path: The path to the board dir.
491 url_suffix: The remote subdirectory where we should upload the packages.
Zdenek Behan62a57792012-08-31 15:09:08 +0200492 prepackaged: If given, a tarball that has been packaged outside of this
493 script and should be used.
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700494 toolchains_overlay_tarballs: List of toolchains overlay tarball
495 specifications to upload. Items take the form
496 "toolchains_spec:/path/to/tarball".
497 toolchains_overlay_upload_path: Path template under the bucket to place
498 toolchains overlay tarballs.
Mike Frysinger9e979b92012-11-29 02:55:09 -0500499 toolchain_tarballs: List of toolchain tarballs to upload.
500 toolchain_upload_path: Path under the bucket to place toolchain tarballs.
David James8fa34ea2011-04-15 13:00:20 -0700501 """
502 remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
503 assert remote_location.startswith('gs://')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200504 boardname = os.path.basename(board_path.rstrip('/'))
505 # We do not upload non SDK board tarballs,
506 assert boardname == constants.CHROOT_BUILDER_BOARD
507 assert prepackaged is not None
Zdenek Behan62a57792012-08-31 15:09:08 +0200508
Mike Frysinger8092a632014-05-24 13:25:46 -0400509 version_str = self._version[len('chroot-'):]
Mike Frysinger8e727a32013-01-16 16:57:53 -0500510 remote_tarfile = toolchain.GetSdkURL(
511 for_gsutil=True, suburl='cros-sdk-%s.tar.xz' % (version_str,))
Zdenek Behan86c15e92012-10-13 00:55:47 +0200512 # For SDK, also upload the manifest which is guaranteed to exist
513 # by the builderstage.
Mike Frysinger540883b2014-05-24 13:46:16 -0400514 self._Upload(prepackaged + '.Manifest', remote_tarfile + '.Manifest')
515 self._Upload(prepackaged, remote_tarfile)
Zdenek Behan86c15e92012-10-13 00:55:47 +0200516
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700517 # Upload SDK toolchains overlays and toolchain tarballs, if given.
Gilad Arnoldad333182015-05-27 15:50:41 -0700518 for tarball_list, upload_path, qualifier_name in (
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700519 (toolchains_overlay_tarballs, toolchains_overlay_upload_path,
520 'toolchains'),
Gilad Arnoldad333182015-05-27 15:50:41 -0700521 (toolchain_tarballs, toolchain_upload_path, 'target')):
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700522 for tarball_spec in tarball_list:
523 qualifier_val, local_path = tarball_spec.split(':')
Gilad Arnoldad333182015-05-27 15:50:41 -0700524 suburl = upload_path % {qualifier_name: qualifier_val}
525 remote_path = toolchain.GetSdkURL(for_gsutil=True, suburl=suburl)
526 self._Upload(local_path, remote_path)
Mike Frysinger9e979b92012-11-29 02:55:09 -0500527
Zdenek Behan86c15e92012-10-13 00:55:47 +0200528 # Finally, also update the pointer to the latest SDK on which polling
529 # scripts rely.
David James4bc13702013-03-26 08:08:04 -0700530 with osutils.TempDir() as tmpdir:
Zdenek Behan86c15e92012-10-13 00:55:47 +0200531 pointerfile = os.path.join(tmpdir, 'cros-sdk-latest.conf')
Mike Frysinger8e727a32013-01-16 16:57:53 -0500532 remote_pointerfile = toolchain.GetSdkURL(for_gsutil=True,
533 suburl='cros-sdk-latest.conf')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200534 osutils.WriteFile(pointerfile, 'LATEST_SDK="%s"' % version_str)
Mike Frysinger540883b2014-05-24 13:46:16 -0400535 self._Upload(pointerfile, remote_pointerfile)
Zdenek Behan33a34112012-09-10 21:07:51 +0200536
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700537 def _GetTargets(self):
538 """Retuns the list of targets to use."""
539 targets = self._slave_targets[:]
540 if self._target:
541 targets.append(self._target)
542
543 return targets
544
Mike Frysinger8092a632014-05-24 13:25:46 -0400545 def SyncHostPrebuilts(self, key, git_sync, sync_binhost_conf):
David Jamesc0f158a2011-02-22 16:07:29 -0800546 """Synchronize host prebuilt files.
David James05bcb2b2011-02-09 09:25:47 -0800547
David Jamesc0f158a2011-02-22 16:07:29 -0800548 This function will sync both the standard host packages, plus the host
549 packages associated with all targets that have been "setup" with the
550 current host's chroot. For instance, if this host has been used to build
551 x86-generic, it will sync the host packages associated with
552 'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
553 it will also sync the host packages associated with
554 'armv7a-cros-linux-gnueabi'.
David James05bcb2b2011-02-09 09:25:47 -0800555
David Jamesc0f158a2011-02-22 16:07:29 -0800556 Args:
David Jamesc0f158a2011-02-22 16:07:29 -0800557 key: The variable key to update in the git file.
558 git_sync: If set, update make.conf of target to reference the latest
559 prebuilt packages generated here.
560 sync_binhost_conf: If set, update binhost config file in
561 chromiumos-overlay for the host.
562 """
David Jamese2488642011-11-14 16:15:20 -0800563 # Slave boards are listed before the master board so that the master board
564 # takes priority (i.e. x86-generic preflight host prebuilts takes priority
565 # over preflight host prebuilts from other builders.)
566 binhost_urls = []
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700567 for target in self._GetTargets():
Mike Frysinger8092a632014-05-24 13:25:46 -0400568 url_suffix = _REL_HOST_PATH % {'version': self._version,
David James4058b0d2011-12-08 21:24:50 -0800569 'host_arch': _HOST_ARCH,
570 'target': target}
David Jamese2488642011-11-14 16:15:20 -0800571 packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
David James05bcb2b2011-02-09 09:25:47 -0800572
Mike Frysinger540883b2014-05-24 13:46:16 -0400573 if self._target == target and not self._skip_upload:
David Jamese2488642011-11-14 16:15:20 -0800574 # Upload prebuilts.
575 package_path = os.path.join(self._build_path, _HOST_PACKAGES_PATH)
576 self._UploadPrebuilt(package_path, packages_url_suffix)
David James8ece7ee2011-06-29 16:02:30 -0700577
David Jamese2488642011-11-14 16:15:20 -0800578 # Record URL where prebuilts were uploaded.
579 binhost_urls.append('%s/%s/' % (self._binhost_base_url.rstrip('/'),
580 packages_url_suffix.rstrip('/')))
581
David James20b2b6f2011-11-18 15:11:58 -0800582 binhost = ' '.join(binhost_urls)
David James8ece7ee2011-06-29 16:02:30 -0700583 if git_sync:
David Jamesb26b9312014-12-15 11:26:46 -0800584 git_file = os.path.join(self._build_path, _PREBUILT_MAKE_CONF[_HOST_ARCH])
Mike Frysinger86509232014-05-24 13:18:37 -0400585 RevGitFile(git_file, {key: binhost}, dryrun=self._dryrun)
David James8ece7ee2011-06-29 16:02:30 -0700586 if sync_binhost_conf:
David Jamesb26b9312014-12-15 11:26:46 -0800587 binhost_conf = os.path.join(
588 self._binhost_conf_dir, 'host', '%s-%s.conf' % (_HOST_ARCH, key))
David Jamese2488642011-11-14 16:15:20 -0800589 UpdateBinhostConfFile(binhost_conf, key, binhost)
David Jamesc0f158a2011-02-22 16:07:29 -0800590
Mike Frysinger8092a632014-05-24 13:25:46 -0400591 def SyncBoardPrebuilts(self, key, git_sync, sync_binhost_conf,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500592 upload_board_tarball, prepackaged_board,
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700593 toolchains_overlay_tarballs,
594 toolchains_overlay_upload_path,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500595 toolchain_tarballs, toolchain_upload_path):
David Jamesc0f158a2011-02-22 16:07:29 -0800596 """Synchronize board prebuilt files.
597
598 Args:
David Jamesc0f158a2011-02-22 16:07:29 -0800599 key: The variable key to update in the git file.
600 git_sync: If set, update make.conf of target to reference the latest
601 prebuilt packages generated here.
602 sync_binhost_conf: If set, update binhost config file in
603 chromiumos-overlay for the current board.
David James8fa34ea2011-04-15 13:00:20 -0700604 upload_board_tarball: Include a tarball of the board in our upload.
Zdenek Behan62a57792012-08-31 15:09:08 +0200605 prepackaged_board: A tarball of the board built outside of this script.
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700606 toolchains_overlay_tarballs: List of toolchains overlay tarball
607 specifications to upload. Items take the form
608 "toolchains_spec:/path/to/tarball".
609 toolchains_overlay_upload_path: Path template under the bucket to place
610 toolchains overlay tarballs.
Mike Frysinger9e979b92012-11-29 02:55:09 -0500611 toolchain_tarballs: A list of toolchain tarballs to upload.
612 toolchain_upload_path: Path under the bucket to place toolchain tarballs.
David Jamesc0f158a2011-02-22 16:07:29 -0800613 """
David Jamesb26b9312014-12-15 11:26:46 -0800614 updated_binhosts = set()
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700615 for target in self._GetTargets():
David Jamese2488642011-11-14 16:15:20 -0800616 board_path = os.path.join(self._build_path,
David James4058b0d2011-12-08 21:24:50 -0800617 _BOARD_PATH % {'board': target.board_variant})
David Jamese2488642011-11-14 16:15:20 -0800618 package_path = os.path.join(board_path, 'packages')
Mike Frysinger8092a632014-05-24 13:25:46 -0400619 url_suffix = _REL_BOARD_PATH % {'target': target,
620 'version': self._version}
David Jamese2488642011-11-14 16:15:20 -0800621 packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
David James8fa34ea2011-04-15 13:00:20 -0700622
Matt Tennante8179042013-10-01 15:47:32 -0700623 # Process the target board differently if it is the main --board.
Mike Frysinger540883b2014-05-24 13:46:16 -0400624 if self._target == target and not self._skip_upload:
Matt Tennante8179042013-10-01 15:47:32 -0700625 # This strips "chroot" prefix because that is sometimes added as the
626 # --prepend-version argument (e.g. by chromiumos-sdk bot).
627 # TODO(build): Clean it up to be less hard-coded.
Mike Frysinger8092a632014-05-24 13:25:46 -0400628 version_str = self._version[len('chroot-'):]
Mike Frysinger9e979b92012-11-29 02:55:09 -0500629
David Jamese2488642011-11-14 16:15:20 -0800630 # Upload board tarballs in the background.
631 if upload_board_tarball:
Mike Frysinger9e979b92012-11-29 02:55:09 -0500632 if toolchain_upload_path:
633 toolchain_upload_path %= {'version': version_str}
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700634 if toolchains_overlay_upload_path:
635 toolchains_overlay_upload_path %= {'version': version_str}
Mike Frysinger9e979b92012-11-29 02:55:09 -0500636 tar_process = multiprocessing.Process(
637 target=self._UploadSdkTarball,
Mike Frysinger8092a632014-05-24 13:25:46 -0400638 args=(board_path, url_suffix, prepackaged_board,
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700639 toolchains_overlay_tarballs,
640 toolchains_overlay_upload_path, toolchain_tarballs,
641 toolchain_upload_path))
David Jamese2488642011-11-14 16:15:20 -0800642 tar_process.start()
David James8fa34ea2011-04-15 13:00:20 -0700643
David Jamese2488642011-11-14 16:15:20 -0800644 # Upload prebuilts.
645 self._UploadPrebuilt(package_path, packages_url_suffix)
David James8fa34ea2011-04-15 13:00:20 -0700646
David Jamese2488642011-11-14 16:15:20 -0800647 # Make sure we finished uploading the board tarballs.
648 if upload_board_tarball:
649 tar_process.join()
650 assert tar_process.exitcode == 0
David Jamesc0f158a2011-02-22 16:07:29 -0800651
David Jamese2488642011-11-14 16:15:20 -0800652 # Record URL where prebuilts were uploaded.
653 url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
654 packages_url_suffix.rstrip('/'))
655
656 if git_sync:
David James4058b0d2011-12-08 21:24:50 -0800657 git_file = DeterminePrebuiltConfFile(self._build_path, target)
Mike Frysinger86509232014-05-24 13:18:37 -0400658 RevGitFile(git_file, {key: url_value}, dryrun=self._dryrun)
Matt Tennante8179042013-10-01 15:47:32 -0700659
David Jamese2488642011-11-14 16:15:20 -0800660 if sync_binhost_conf:
Matt Tennante8179042013-10-01 15:47:32 -0700661 # Update the binhost configuration file in git.
David Jamesb26b9312014-12-15 11:26:46 -0800662 binhost_conf = os.path.join(
663 self._binhost_conf_dir, 'target', '%s-%s.conf' % (target, key))
664 updated_binhosts.add(binhost_conf)
David Jamese2488642011-11-14 16:15:20 -0800665 UpdateBinhostConfFile(binhost_conf, key, url_value)
David James05bcb2b2011-02-09 09:25:47 -0800666
David Jamesb26b9312014-12-15 11:26:46 -0800667 if sync_binhost_conf:
668 # Clear all old binhosts. The files must be left empty in case anybody
669 # is referring to them.
670 all_binhosts = set(glob.glob(os.path.join(
671 self._binhost_conf_dir, 'target', '*-%s.conf' % key)))
672 for binhost_conf in all_binhosts - updated_binhosts:
673 UpdateBinhostConfFile(binhost_conf, key, '')
674
David James05bcb2b2011-02-09 09:25:47 -0800675
Mike Frysingerb87bb782015-06-04 02:46:50 -0400676class _AddSlaveBoardAction(argparse.Action):
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700677 """Callback that adds a slave board to the list of slave targets."""
Mike Frysingerb87bb782015-06-04 02:46:50 -0400678 def __call__(self, parser, namespace, values, option_string=None):
679 getattr(namespace, self.dest).append(BuildTarget(values))
David James4058b0d2011-12-08 21:24:50 -0800680
681
Mike Frysingerb87bb782015-06-04 02:46:50 -0400682class _AddSlaveProfileAction(argparse.Action):
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700683 """Callback that adds a slave profile to the list of slave targets."""
Mike Frysingerb87bb782015-06-04 02:46:50 -0400684 def __call__(self, parser, namespace, values, option_string=None):
685 if not namespace.slave_targets:
686 parser.error('Must specify --slave-board before --slave-profile')
687 if namespace.slave_targets[-1].profile is not None:
688 parser.error('Cannot specify --slave-profile twice for same board')
689 namespace.slave_targets[-1].profile = values
David James4058b0d2011-12-08 21:24:50 -0800690
691
Mike Frysinger86509232014-05-24 13:18:37 -0400692def ParseOptions(argv):
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700693 """Returns options given by the user and the target specified.
694
Mike Frysinger86509232014-05-24 13:18:37 -0400695 Args:
696 argv: The args to parse.
697
Mike Frysinger1a736a82013-12-12 01:50:59 -0500698 Returns:
699 A tuple containing a parsed options object and BuildTarget.
700 The target instance is None if no board is specified.
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700701 """
Mike Frysingerb87bb782015-06-04 02:46:50 -0400702 parser = commandline.ArgumentParser()
703 parser.add_argument('-H', '--binhost-base-url', default=_BINHOST_BASE_URL,
704 help='Base URL to use for binhost in make.conf updates')
705 parser.add_argument('--previous-binhost-url', action='append', default=[],
706 help='Previous binhost URL')
707 parser.add_argument('-b', '--board',
708 help='Board type that was built on this machine')
709 parser.add_argument('-B', '--prepackaged-tarball', type='path',
710 help='Board tarball prebuilt outside of this script.')
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700711 parser.add_argument('--toolchains-overlay-tarball',
712 dest='toolchains_overlay_tarballs',
Mike Frysingerb87bb782015-06-04 02:46:50 -0400713 action='append', default=[],
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700714 help='Toolchains overlay tarball specification to '
715 'upload. Takes the form '
716 '"toolchains_spec:/path/to/tarball".')
717 parser.add_argument('--toolchains-overlay-upload-path', default='',
718 help='Path template for uploading toolchains overlays.')
Mike Frysingerb87bb782015-06-04 02:46:50 -0400719 parser.add_argument('--toolchain-tarball', dest='toolchain_tarballs',
720 action='append', default=[],
721 help='Redistributable toolchain tarball.')
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700722 parser.add_argument('--toolchain-upload-path', default='',
Mike Frysingerb87bb782015-06-04 02:46:50 -0400723 help='Path to place toolchain tarballs in the sdk tree.')
724 parser.add_argument('--profile',
725 help='Profile that was built on this machine')
726 parser.add_argument('--slave-board', default=[], action=_AddSlaveBoardAction,
727 dest='slave_targets',
728 help='Board type that was built on a slave machine. To '
729 'add a profile to this board, use --slave-profile.')
730 parser.add_argument('--slave-profile', action=_AddSlaveProfileAction,
731 help='Board profile that was built on a slave machine. '
732 'Applies to previous slave board.')
733 parser.add_argument('-p', '--build-path', required=True,
734 help='Path to the directory containing the chroot')
735 parser.add_argument('--packages', action='append', default=[],
736 help='Only include the specified packages. '
737 '(Default is to include all packages.)')
738 parser.add_argument('-s', '--sync-host', default=False, action='store_true',
739 help='Sync host prebuilts')
740 parser.add_argument('-g', '--git-sync', default=False, action='store_true',
741 help='Enable git version sync (This commits to a repo.) '
742 'This is used by full builders to commit directly '
743 'to board overlays.')
744 parser.add_argument('-u', '--upload',
745 help='Upload location')
746 parser.add_argument('-V', '--prepend-version',
747 help='Add an identifier to the front of the version')
748 parser.add_argument('-f', '--filters', action='store_true', default=False,
749 help='Turn on filtering of private ebuild packages')
750 parser.add_argument('-k', '--key', default='PORTAGE_BINHOST',
751 help='Key to update in make.conf / binhost.conf')
752 parser.add_argument('--set-version',
753 help='Specify the version string')
754 parser.add_argument('--sync-binhost-conf', default=False, action='store_true',
755 help='Update binhost.conf in chromiumos-overlay or '
756 'chromeos-overlay. Commit the changes, but don\'t '
757 'push them. This is used for preflight binhosts.')
758 parser.add_argument('--binhost-conf-dir',
759 help='Directory to commit binhost config with '
760 '--sync-binhost-conf.')
761 parser.add_argument('-P', '--private', action='store_true', default=False,
762 help='Mark gs:// uploads as private.')
763 parser.add_argument('--skip-upload', action='store_true', default=False,
764 help='Skip upload step.')
765 parser.add_argument('--upload-board-tarball', action='store_true',
766 default=False,
767 help='Upload board tarball to Google Storage.')
768 parser.add_argument('-n', '--dry-run', dest='dryrun',
769 action='store_true', default=False,
770 help='Don\'t push or upload prebuilts.')
David James8c846492011-01-25 17:07:29 -0800771
Mike Frysingerb87bb782015-06-04 02:46:50 -0400772 options = parser.parse_args(argv)
David James8ece7ee2011-06-29 16:02:30 -0700773 if not options.upload and not options.skip_upload:
Mike Frysinger86509232014-05-24 13:18:37 -0400774 parser.error('you need to provide an upload location using -u')
David James8ece7ee2011-06-29 16:02:30 -0700775 if not options.set_version and options.skip_upload:
Mike Frysinger86509232014-05-24 13:18:37 -0400776 parser.error('If you are using --skip-upload, you must specify a '
777 'version number using --set-version.')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700778
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700779 target = None
780 if options.board:
781 target = BuildTarget(options.board, options.profile)
782
783 if target in options.slave_targets:
Mike Frysinger86509232014-05-24 13:18:37 -0400784 parser.error('--board/--profile must not also be a slave target.')
David Jamese2488642011-11-14 16:15:20 -0800785
David James4058b0d2011-12-08 21:24:50 -0800786 if len(set(options.slave_targets)) != len(options.slave_targets):
Mike Frysinger86509232014-05-24 13:18:37 -0400787 parser.error('--slave-boards must not have duplicates.')
David Jamese2488642011-11-14 16:15:20 -0800788
David James4058b0d2011-12-08 21:24:50 -0800789 if options.slave_targets and options.git_sync:
Mike Frysinger86509232014-05-24 13:18:37 -0400790 parser.error('--slave-boards is not compatible with --git-sync')
David Jamese2488642011-11-14 16:15:20 -0800791
David James8ece7ee2011-06-29 16:02:30 -0700792 if (options.upload_board_tarball and options.skip_upload and
793 options.board == 'amd64-host'):
Mike Frysinger86509232014-05-24 13:18:37 -0400794 parser.error('--skip-upload is not compatible with '
795 '--upload-board-tarball and --board=amd64-host')
David James8fa34ea2011-04-15 13:00:20 -0700796
David James8ece7ee2011-06-29 16:02:30 -0700797 if (options.upload_board_tarball and not options.skip_upload and
798 not options.upload.startswith('gs://')):
Mike Frysinger86509232014-05-24 13:18:37 -0400799 parser.error('--upload-board-tarball only works with gs:// URLs.\n'
800 '--upload must be a gs:// URL.')
David James8fa34ea2011-04-15 13:00:20 -0700801
Zdenek Behan86c15e92012-10-13 00:55:47 +0200802 if options.upload_board_tarball and options.prepackaged_tarball is None:
Mike Frysinger86509232014-05-24 13:18:37 -0400803 parser.error('--upload-board-tarball requires --prepackaged-tarball')
Zdenek Behan86c15e92012-10-13 00:55:47 +0200804
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700805 if options.private:
806 if options.sync_host:
Mike Frysinger86509232014-05-24 13:18:37 -0400807 parser.error('--private and --sync-host/-s cannot be specified '
808 'together; we do not support private host prebuilts')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700809
David James8ece7ee2011-06-29 16:02:30 -0700810 if not options.upload or not options.upload.startswith('gs://'):
Mike Frysinger86509232014-05-24 13:18:37 -0400811 parser.error('--private is only valid for gs:// URLs; '
812 '--upload must be a gs:// URL.')
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700813
814 if options.binhost_base_url != _BINHOST_BASE_URL:
Mike Frysinger86509232014-05-24 13:18:37 -0400815 parser.error('when using --private the --binhost-base-url '
816 'is automatically derived.')
David James27fa7d12011-06-29 17:24:14 -0700817
David Jamesc31168e2014-06-05 14:40:05 -0700818 if options.sync_binhost_conf and not options.binhost_conf_dir:
819 parser.error('--sync-binhost-conf requires --binhost-conf-dir')
820
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700821 if (options.toolchains_overlay_tarballs and
822 not options.toolchains_overlay_upload_path):
823 parser.error('--toolchains-overlay-tarball requires '
824 '--toolchains-overlay-upload-path')
Gilad Arnoldad333182015-05-27 15:50:41 -0700825
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700826 return options, target
David Jamesc0f158a2011-02-22 16:07:29 -0800827
Mike Frysingercc838832014-05-24 13:10:30 -0400828
Mike Frysinger86509232014-05-24 13:18:37 -0400829def main(argv):
David Jamesdb401072011-06-10 12:17:16 -0700830 # Set umask to a sane value so that files created as root are readable.
Mike Frysinger60ec1012013-10-21 00:11:10 -0400831 os.umask(0o22)
David Jamesdb401072011-06-10 12:17:16 -0700832
Mike Frysinger86509232014-05-24 13:18:37 -0400833 options, target = ParseOptions(argv)
David Jamesc0f158a2011-02-22 16:07:29 -0800834
David James05bcb2b2011-02-09 09:25:47 -0800835 # Calculate a list of Packages index files to compare against. Whenever we
836 # upload a package, we check to make sure it's not already stored in one of
837 # the packages files we uploaded. This list of packages files might contain
838 # both board and host packages.
David Jamesce093af2011-02-23 15:21:58 -0800839 pkg_indexes = _GrabAllRemotePackageIndexes(options.previous_binhost_url)
David James8c846492011-01-25 17:07:29 -0800840
David James8ece7ee2011-06-29 16:02:30 -0700841 if options.set_version:
842 version = options.set_version
843 else:
844 version = GetVersion()
Matt Tennante8179042013-10-01 15:47:32 -0700845
David Jamesc0f158a2011-02-22 16:07:29 -0800846 if options.prepend_version:
847 version = '%s-%s' % (options.prepend_version, version)
848
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700849 acl = 'public-read'
850 binhost_base_url = options.binhost_base_url
851
David Jamesadd21432013-05-21 10:04:07 -0700852 if options.private:
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700853 binhost_base_url = options.upload
David Jamesadd21432013-05-21 10:04:07 -0700854 if target:
Prathmesh Prabhu421eef22014-10-16 17:13:19 -0700855 acl = portage_util.FindOverlayFile(_GOOGLESTORAGE_GSUTIL_FILE,
856 board=target.board_variant,
857 buildroot=options.build_path)
Ben Chan067de492015-01-06 17:19:13 -0800858 if acl is None:
859 cros_build_lib.Die('No Google Storage ACL file %s found in %s overlay.',
860 _GOOGLESTORAGE_GSUTIL_FILE, target.board_variant)
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700861
David Jamesb26b9312014-12-15 11:26:46 -0800862 binhost_conf_dir = None
863 if options.binhost_conf_dir:
864 binhost_conf_dir = os.path.join(options.build_path,
865 options.binhost_conf_dir)
866
Scott Zawalskiab1bed32011-03-16 15:24:24 -0700867 uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
David James615e5b52011-06-03 11:10:15 -0700868 pkg_indexes, options.build_path,
David James27fa7d12011-06-29 17:24:14 -0700869 options.packages, options.skip_upload,
David Jamesb26b9312014-12-15 11:26:46 -0800870 binhost_conf_dir, options.dryrun,
Mike Frysinger8092a632014-05-24 13:25:46 -0400871 target, options.slave_targets, version)
David Jamesc0f158a2011-02-22 16:07:29 -0800872
David James8c846492011-01-25 17:07:29 -0800873 if options.sync_host:
Mike Frysinger8092a632014-05-24 13:25:46 -0400874 uploader.SyncHostPrebuilts(options.key, options.git_sync,
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700875 options.sync_binhost_conf)
David James8c846492011-01-25 17:07:29 -0800876
Chris Sosa62c8ff52012-06-04 15:03:12 -0700877 if options.board or options.slave_targets:
Mike Frysinger8092a632014-05-24 13:25:46 -0400878 uploader.SyncBoardPrebuilts(options.key, options.git_sync,
Chris Sosa6a5dceb2012-05-14 13:48:56 -0700879 options.sync_binhost_conf,
Zdenek Behan62a57792012-08-31 15:09:08 +0200880 options.upload_board_tarball,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500881 options.prepackaged_tarball,
Gilad Arnold2b79e2d2015-06-02 11:26:07 -0700882 options.toolchains_overlay_tarballs,
883 options.toolchains_overlay_upload_path,
Mike Frysinger9e979b92012-11-29 02:55:09 -0500884 options.toolchain_tarballs,
885 options.toolchain_upload_path)