blob: 95c8a4e8016018063c7f2e680104295b8974e6d4 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Ryan Cuia56a71e2012-10-18 18:40:35 -07005
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger1d4752b2014-11-08 04:00:18 -050019# pylint: disable=bad-continuation
20
Mike Frysinger383367e2014-09-16 15:06:17 -040021from __future__ import print_function
22
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080024import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070025import functools
Steve Funge984a532013-11-25 17:09:25 -080026import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import logging
David James88e6f032013-03-02 08:13:20 -080028import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070030import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070031import shlex
Steve Funge984a532013-11-25 17:09:25 -080032import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070033import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070034
Ryan Cuia56a71e2012-10-18 18:40:35 -070035
Don Garrett88b8d782014-05-13 17:30:55 -070036from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070037from chromite.cbuildbot import failures_lib
Ryan Cui686ec052013-02-12 16:39:41 -080038from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070039from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070041from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080042from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070043from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080044from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070045from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070046from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080047from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070048from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070049
50
Ryan Cuia56a71e2012-10-18 18:40:35 -070051_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
52
Ryan Cui3045c5d2012-07-13 18:00:33 -070053KERNEL_A_PARTITION = 2
54KERNEL_B_PARTITION = 4
55
56KILL_PROC_MAX_WAIT = 10
57POST_KILL_WAIT = 2
58
Ryan Cuie535b172012-10-19 18:25:03 -070059MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080060LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070061
Steve Funge984a532013-11-25 17:09:25 -080062_ANDROID_DIR = '/system/chrome'
63_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
64
David James2cb34002013-03-01 18:42:40 -080065_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070066_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080067
Thiago Goncales12793312013-05-23 11:26:17 -070068_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
69_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070070
Steve Funge984a532013-11-25 17:09:25 -080071DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080072
Ryan Cui3045c5d2012-07-13 18:00:33 -070073def _UrlBaseName(url):
74 """Return the last component of the URL."""
75 return url.rstrip('/').rpartition('/')[-1]
76
77
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070078class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080079 """Raised whenever the deploy fails."""
80
81
Ryan Cui7193a7e2013-04-26 14:15:19 -070082DeviceInfo = collections.namedtuple(
83 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
84
85
Ryan Cui3045c5d2012-07-13 18:00:33 -070086class DeployChrome(object):
87 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070088 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070089 """Initialize the class.
90
Mike Frysinger02e1e072013-11-10 22:11:34 -050091 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070092 options: Optparse result structure.
93 tempdir: Scratch space for the class. Caller has responsibility to clean
94 it up.
Steve Funge984a532013-11-25 17:09:25 -080095 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070096 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070097 self.tempdir = tempdir
98 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070099 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700100 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800101 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700102
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700103 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800104 self.chrome_dir = _CHROME_DIR
105
Ryan Cui7193a7e2013-04-26 14:15:19 -0700106 def _GetRemoteMountFree(self, remote_dir):
Daniel Erat1ae46382014-08-14 10:23:39 -0700107 result = self.host.RemoteSh(DF_COMMAND % remote_dir, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800109 value = line.split()[3]
110 multipliers = {
111 'G': 1024 * 1024 * 1024,
112 'M': 1024 * 1024,
113 'K': 1024,
114 }
115 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700116
117 def _GetRemoteDirSize(self, remote_dir):
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800118 result = self.host.RemoteSh('du -ks %s' % remote_dir, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119 return int(result.output.split()[0])
120
121 def _GetStagingDirSize(self):
122 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800123 redirect_stdout=True,
124 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700125 return int(result.output.split()[0])
126
Ryan Cui3045c5d2012-07-13 18:00:33 -0700127 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800128 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800129 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 return result.returncode == 0
131
132 def _DisableRootfsVerification(self):
133 if not self.options.force:
134 logging.error('Detected that the device has rootfs verification enabled.')
135 logging.info('This script can automatically remove the rootfs '
136 'verification, which requires that it reboot the device.')
137 logging.info('Make sure the device is in developer mode!')
138 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700139 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800140 # Since we stopped Chrome earlier, it's good form to start it up again.
141 if self.options.startui:
142 logging.info('Starting Chrome...')
143 self.host.RemoteSh('start ui')
144 raise DeployFailure('Need rootfs verification to be disabled. '
145 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700146
147 logging.info('Removing rootfs verification from %s', self.options.to)
148 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
149 # Use --force to bypass the checks.
150 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
151 '--remove_rootfs_verification --force')
152 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
153 self.host.RemoteSh(cmd % partition, error_code_ok=True)
154
155 # A reboot in developer mode takes a while (and has delays), so the user
156 # will have time to read and act on the USB boot instructions below.
157 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
158 self.host.RemoteReboot()
159
David James88e6f032013-03-02 08:13:20 -0800160 # Now that the machine has been rebooted, we need to kill Chrome again.
161 self._KillProcsIfNeeded()
162
163 # Make sure the rootfs is writable now.
164 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700165
166 def _CheckUiJobStarted(self):
167 # status output is in the format:
168 # <job_name> <status> ['process' <pid>].
169 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800170 try:
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800171 result = self.host.RemoteSh('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800172 except cros_build_lib.RunCommandError as e:
173 if 'Unknown job' in e.result.error:
174 return False
175 else:
176 raise e
177
Ryan Cui3045c5d2012-07-13 18:00:33 -0700178 return result.output.split()[1].split('/')[0] == 'start'
179
180 def _KillProcsIfNeeded(self):
181 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800182 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700183 self.host.RemoteSh('stop ui')
184
185 # Developers sometimes run session_manager manually, in which case we'll
186 # need to help shut the chrome processes down.
187 try:
David James3432acd2013-11-27 10:02:18 -0800188 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700189 while self._ChromeFileInUse():
190 logging.warning('The chrome binary on the device is in use.')
191 logging.warning('Killing chrome and session_manager processes...\n')
192
193 self.host.RemoteSh("pkill 'chrome|session_manager'",
194 error_code_ok=True)
195 # Wait for processes to actually terminate
196 time.sleep(POST_KILL_WAIT)
197 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800198 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800199 msg = ('Could not kill processes after %s seconds. Please exit any '
200 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
201 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700202
David James88e6f032013-03-02 08:13:20 -0800203 def _MountRootfsAsWritable(self, error_code_ok=True):
204 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
David James88e6f032013-03-02 08:13:20 -0800206 If the command fails, and error_code_ok is True, then this function sets
207 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Mike Frysinger02e1e072013-11-10 22:11:34 -0500209 Args:
David James88e6f032013-03-02 08:13:20 -0800210 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
211 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800212 # TODO: Should migrate to use the remount functions in remote_access.
Daniel Erat1ae46382014-08-14 10:23:39 -0700213 result = self.host.RemoteSh(MOUNT_RW_COMMAND,
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800214 error_code_ok=error_code_ok,
215 capture_output=True)
David James88e6f032013-03-02 08:13:20 -0800216 if result.returncode:
217 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700218
Ryan Cui7193a7e2013-04-26 14:15:19 -0700219 def _GetDeviceInfo(self):
220 steps = [
221 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
222 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
223 ]
224 return_values = parallel.RunParallelSteps(steps, return_values=True)
225 return DeviceInfo(*return_values)
226
227 def _CheckDeviceFreeSpace(self, device_info):
228 """See if target device has enough space for Chrome.
229
Mike Frysinger02e1e072013-11-10 22:11:34 -0500230 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700231 device_info: A DeviceInfo named tuple.
232 """
233 effective_free = device_info.target_dir_size + device_info.target_fs_free
234 staging_size = self._GetStagingDirSize()
235 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700236 raise DeployFailure(
237 'Not enough free space on the device. Required: %s MiB, '
238 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700239 if device_info.target_fs_free < (100 * 1024):
240 logging.warning('The device has less than 100MB free. deploy_chrome may '
241 'hang during the transfer.')
242
Ryan Cui3045c5d2012-07-13 18:00:33 -0700243 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800244 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700245 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800246 dest_path = _CHROME_DIR
Daniel Erat1ae46382014-08-14 10:23:39 -0700247 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
248 self.options.target_dir,
249 inplace=True, debug_level=logging.INFO,
250 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800251
252 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800253 if p.mode:
254 # Set mode if necessary.
255 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
256 p.src if not p.dest else p.dest))
257
258
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800259 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800260 logging.info('Starting UI...')
Daniel Erat1ae46382014-08-14 10:23:39 -0700261 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700262
David James88e6f032013-03-02 08:13:20 -0800263 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700264 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800265 logging.info('Testing connection to the device...')
Daniel Erat1ae46382014-08-14 10:23:39 -0700266 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800267 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700268 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800269 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270
Steve Funge984a532013-11-25 17:09:25 -0800271 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700272 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700273 def BinaryExists(filename):
274 """Checks if the passed-in file is present in the build directory."""
275 return os.path.exists(os.path.join(self.options.build_dir, filename))
276
Daniel Erat9813f0e2014-11-12 11:00:28 -0700277 # Handle non-Chrome deployments.
278 if not BinaryExists('chrome'):
279 if BinaryExists('envoy_shell'):
280 self.copy_paths = chrome_util.GetCopyPaths('envoy')
281 elif BinaryExists('app_shell'):
282 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
283
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700284 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700285 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700286 self.options.startui = False
287
David James88e6f032013-03-02 08:13:20 -0800288 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800289 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
290 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800291
Thiago Goncales12793312013-05-23 11:26:17 -0700292 def _MountTarget(self):
293 logging.info('Mounting Chrome...')
294
295 # Create directory if does not exist
296 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
297 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
298 self.options.mount_dir))
299 # Chrome needs partition to have exec and suid flags set
300 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
301
David James88e6f032013-03-02 08:13:20 -0800302 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800303 self._CheckDeployType()
304
David James88e6f032013-03-02 08:13:20 -0800305 # If requested, just do the staging step.
306 if self.options.staging_only:
307 self._PrepareStagingDir()
308 return 0
309
310 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800311 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800312 steps = [self._GetDeviceInfo, self._CheckConnection,
313 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
314 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700315 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
316 return_values=True)
317 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800318
319 # If we failed to mark the rootfs as writable, try disabling rootfs
320 # verification.
321 if self._rootfs_is_still_readonly.is_set():
322 self._DisableRootfsVerification()
323
Thiago Goncales12793312013-05-23 11:26:17 -0700324 if self.options.mount_dir is not None:
325 self._MountTarget()
326
David James88e6f032013-03-02 08:13:20 -0800327 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700328 self._Deploy()
329
330
Ryan Cuia56a71e2012-10-18 18:40:35 -0700331def ValidateGypDefines(_option, _opt, value):
332 """Convert GYP_DEFINES-formatted string to dictionary."""
333 return chrome_util.ProcessGypDefines(value)
334
335
336class CustomOption(commandline.Option):
337 """Subclass Option class to implement path evaluation."""
338 TYPES = commandline.Option.TYPES + ('gyp_defines',)
339 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
340 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
341
342
Ryan Cuie535b172012-10-19 18:25:03 -0700343def _CreateParser():
344 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800345 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
346 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700347
Ryan Cuia56a71e2012-10-18 18:40:35 -0700348 # TODO(rcui): Have this use the UI-V2 format of having source and target
349 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700350 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800351 help='Skip all prompts (i.e., for disabling of rootfs '
352 'verification). This may result in the target '
353 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800354 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
355 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800356 help="The board the Chrome build is targeted for. When in "
357 "a 'cros chrome-sdk' shell, defaults to the SDK "
358 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700359 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800360 help='The directory with Chrome build artifacts to deploy '
361 'from. Typically of format <chrome_root>/out/Debug. '
362 'When this option is used, the GYP_DEFINES '
363 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800364 parser.add_option('--target-dir', type='path',
365 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700366 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700367 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800368 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800369 parser.add_option('--nostartui', action='store_false', dest='startui',
370 default=True,
371 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800372 parser.add_option('--nostrip', action='store_false', dest='dostrip',
373 default=True,
374 help="Don't strip binaries during deployment. Warning: "
375 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700376 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800377 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700378 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800379 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700380 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800381 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700382 parser.add_option('--mount-dir', type='path', default=None,
Mike Frysingercd304692014-10-02 16:33:41 -0400383 help='Deploy Chrome in target directory and bind it '
384 'to the directory specified by this flag.')
Thiago Goncales12793312013-05-23 11:26:17 -0700385 parser.add_option('--mount', action='store_true', default=False,
Mike Frysingercd304692014-10-02 16:33:41 -0400386 help='Deploy Chrome to default target directory and bind '
387 'it to the default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700388
389 group = optparse.OptionGroup(parser, 'Advanced Options')
390 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800391 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800392 group.add_option('--sloppy', action='store_true', default=False,
393 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700394 group.add_option('--staging-flags', default=None, type='gyp_defines',
395 help='Extra flags to control staging. Valid flags are - %s'
396 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800397 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800398 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800399 'variable and --staging-flags, if set. Enforce that '
400 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700401 group.add_option('--strip-flags', default=None,
402 help="Flags to call the 'strip' binutil tool with. "
403 "Overrides the default arguments.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700404 parser.add_option_group(group)
405
Aviv Keshet1c986f32014-04-24 13:20:49 -0700406 group = optparse.OptionGroup(parser, 'Metadata Overrides (Advanced)',
407 description='Provide all of these overrides '
408 'in order to remove dependencies on '
409 'metadata.json existence.')
410 group.add_option('--target-tc', action='store', default=None,
411 help='Override target toolchain name, e.g. '
412 'x86_64-cros-linux-gnu')
413 group.add_option('--toolchain-url', action='store', default=None,
414 help='Override toolchain url format pattern, e.g. '
415 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
416 parser.add_option_group(group)
417
418
Ryan Cuif890a3e2013-03-07 18:57:06 -0800419 # GYP_DEFINES that Chrome was built with. Influences which files are staged
420 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
421 # enviroment variable.
422 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
423 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700424 # Path of an empty directory to stage chrome artifacts to. Defaults to a
425 # temporary directory that is removed when the script finishes. If the path
426 # is specified, then it will not be removed.
427 parser.add_option('--staging-dir', type='path', default=None,
428 help=optparse.SUPPRESS_HELP)
429 # Only prepare the staging directory, and skip deploying to the device.
430 parser.add_option('--staging-only', action='store_true', default=False,
431 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700432 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
433 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
434 # fetching the SDK toolchain.
435 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700436 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700437
Ryan Cuie535b172012-10-19 18:25:03 -0700438
439def _ParseCommandLine(argv):
440 """Parse args, and run environment-independent checks."""
441 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700442 (options, args) = parser.parse_args(argv)
443
Ryan Cuia56a71e2012-10-18 18:40:35 -0700444 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
445 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800446 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700447 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
448 parser.error('Cannot specify both --build_dir and '
449 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800450 if options.build_dir and not options.board:
451 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700452 if options.gs_path and options.local_pkg_path:
453 parser.error('Cannot specify both --gs-path and --local-pkg-path')
454 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700455 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800456 if (options.strict or options.staging_flags) and not options.build_dir:
457 parser.error('--strict and --staging-flags require --build-dir to be '
458 'set.')
459 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800460 parser.error('--staging-flags requires --strict to be set.')
461 if options.sloppy and options.strict:
462 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700463
464 if options.mount or options.mount_dir:
465 if not options.target_dir:
466 options.target_dir = _CHROME_DIR_MOUNT
467 else:
468 if not options.target_dir:
469 options.target_dir = _CHROME_DIR
470
471 if options.mount and not options.mount_dir:
472 options.mount_dir = _CHROME_DIR
473
Ryan Cui3045c5d2012-07-13 18:00:33 -0700474 return options, args
475
476
Ryan Cuie535b172012-10-19 18:25:03 -0700477def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800478 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700479
480 Args:
Steve Funge984a532013-11-25 17:09:25 -0800481 options: The options object returned by optparse.
482 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700483 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700484 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
485 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
486
Ryan Cuib623e7b2013-03-14 12:54:11 -0700487 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700488 gyp_env = os.getenv('GYP_DEFINES', None)
489 if gyp_env is not None:
490 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700491 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700492 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700493
494 if options.strict and not options.gyp_defines:
495 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800496 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700497
Ryan Cui3c183c22013-04-29 18:04:11 -0700498 if options.build_dir:
499 chrome_path = os.path.join(options.build_dir, 'chrome')
500 if os.path.isfile(chrome_path):
501 deps = lddtree.ParseELF(chrome_path)
502 if 'libbase.so' in deps['libs']:
503 cros_build_lib.Warning(
504 'Detected a component build of Chrome. component build is '
505 'not working properly for Chrome OS. See crbug.com/196317. '
506 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700507
Ryan Cuia56a71e2012-10-18 18:40:35 -0700508
Ryan Cui504db722013-01-22 11:48:01 -0800509def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700510 """Get the chrome prebuilt tarball from GS.
511
Mike Frysinger02e1e072013-11-10 22:11:34 -0500512 Returns:
513 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700514 """
David James9374aac2013-10-08 16:00:17 -0700515 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500516 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800517 files = [found for found in files if
518 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
519 if not files:
520 raise Exception('No chrome package found at %s' % gs_path)
521 elif len(files) > 1:
522 # - Users should provide us with a direct link to either a stripped or
523 # unstripped chrome package.
524 # - In the case of being provided with an archive directory, where both
525 # stripped and unstripped chrome available, use the stripped chrome
526 # package.
527 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
528 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
529 files = [f for f in files if not 'unstripped' in f]
530 assert len(files) == 1
531 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800532
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800533 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800534 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800535 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
536 chrome_path = os.path.join(tempdir, filename)
537 assert os.path.exists(chrome_path)
538 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700539
540
Ryan Cuif890a3e2013-03-07 18:57:06 -0800541@contextlib.contextmanager
542def _StripBinContext(options):
543 if not options.dostrip:
544 yield None
545 elif options.strip_bin:
546 yield options.strip_bin
547 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800548 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800549 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700550 with sdk.Prepare(components=components, target_tc=options.target_tc,
551 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800552 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
553 constants.CHROME_ENV_FILE)
554 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
555 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
556 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800557 yield strip_bin
558
559
Steve Funge984a532013-11-25 17:09:25 -0800560def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
561 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800562 """Place the necessary files in the staging directory.
563
564 The staging directory is the directory used to rsync the build artifacts over
565 to the device. Only the necessary Chrome build artifacts are put into the
566 staging directory.
567 """
Ryan Cui5866be02013-03-18 14:12:00 -0700568 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400569 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800570 if options.build_dir:
571 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700572 strip_flags = (None if options.strip_flags is None else
573 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800574 chrome_util.StageChromeFromBuildDir(
575 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800576 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700577 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800578 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700579 else:
580 pkg_path = options.local_pkg_path
581 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800582 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
583 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700584
585 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800586 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700587 # Extract only the ./opt/google/chrome contents, directly into the staging
588 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800589 if pkg_path[-4:] == '.zip':
590 cros_build_lib.DebugRunCommand(
591 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
592 staging_dir])
593 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
594 shutil.move(filename, staging_dir)
595 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
596 else:
597 cros_build_lib.DebugRunCommand(
598 ['tar', '--strip-components', '4', '--extract',
599 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
600 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700601
Ryan Cui71aa8de2013-04-19 16:12:55 -0700602
Ryan Cui3045c5d2012-07-13 18:00:33 -0700603def main(argv):
604 options, args = _ParseCommandLine(argv)
605 _PostParseCheck(options, args)
606
607 # Set cros_build_lib debug level to hide RunCommand spew.
608 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700609 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700610 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800611 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700612
Ryan Cui71aa8de2013-04-19 16:12:55 -0700613 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700614 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
615 if cmd_stats:
616 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700617
Ryan Cui71aa8de2013-04-19 16:12:55 -0700618 with osutils.TempDir(set_global=True) as tempdir:
619 staging_dir = options.staging_dir
620 if not staging_dir:
621 staging_dir = os.path.join(tempdir, 'chrome')
622
623 deploy = DeployChrome(options, tempdir, staging_dir)
624 try:
625 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700626 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700627 raise SystemExit(str(ex).strip())