blob: 36befec41c4659b8ebcbd8c66a07bf1f18482885 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Ryan Cuia56a71e2012-10-18 18:40:35 -07005
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger1d4752b2014-11-08 04:00:18 -050019# pylint: disable=bad-continuation
20
Mike Frysinger383367e2014-09-16 15:06:17 -040021from __future__ import print_function
22
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080024import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070025import functools
Steve Funge984a532013-11-25 17:09:25 -080026import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import logging
David James88e6f032013-03-02 08:13:20 -080028import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070030import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070031import shlex
Steve Funge984a532013-11-25 17:09:25 -080032import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070033import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070034
Ryan Cuia56a71e2012-10-18 18:40:35 -070035
Don Garrett88b8d782014-05-13 17:30:55 -070036from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070037from chromite.cbuildbot import failures_lib
Ryan Cui686ec052013-02-12 16:39:41 -080038from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070039from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070041from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080042from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070043from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080044from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070045from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070046from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080047from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070048from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070049
50
Ryan Cuia56a71e2012-10-18 18:40:35 -070051_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
52
Ryan Cui3045c5d2012-07-13 18:00:33 -070053KERNEL_A_PARTITION = 2
54KERNEL_B_PARTITION = 4
55
56KILL_PROC_MAX_WAIT = 10
57POST_KILL_WAIT = 2
58
Ryan Cuie535b172012-10-19 18:25:03 -070059MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080060LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070061
Steve Funge984a532013-11-25 17:09:25 -080062_ANDROID_DIR = '/system/chrome'
63_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
64
David James2cb34002013-03-01 18:42:40 -080065_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070066_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080067
Pawel Osciakfb200f52014-12-21 13:42:05 +090068_UMOUNT_DIR_IF_MOUNTPOINT_CMD = 'mountpoint -q %(dir)s && umount %(dir)s'
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080073
Ryan Cui3045c5d2012-07-13 18:00:33 -070074def _UrlBaseName(url):
75 """Return the last component of the URL."""
76 return url.rstrip('/').rpartition('/')[-1]
77
78
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070079class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080080 """Raised whenever the deploy fails."""
81
82
Ryan Cui7193a7e2013-04-26 14:15:19 -070083DeviceInfo = collections.namedtuple(
84 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
85
86
Ryan Cui3045c5d2012-07-13 18:00:33 -070087class DeployChrome(object):
88 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070089 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070090 """Initialize the class.
91
Mike Frysinger02e1e072013-11-10 22:11:34 -050092 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070093 options: Optparse result structure.
94 tempdir: Scratch space for the class. Caller has responsibility to clean
95 it up.
Steve Funge984a532013-11-25 17:09:25 -080096 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070097 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070098 self.tempdir = tempdir
99 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700100 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -0500101 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900102 self.device = remote.RemoteDevice(options.to, port=options.port,
103 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500104 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700105
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700106 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800107 self.chrome_dir = _CHROME_DIR
108
Ryan Cui7193a7e2013-04-26 14:15:19 -0700109 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500110 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700111 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800112 value = line.split()[3]
113 multipliers = {
114 'G': 1024 * 1024 * 1024,
115 'M': 1024 * 1024,
116 'K': 1024,
117 }
118 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119
120 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 result = self.device.RunCommand('du -ks %s' % remote_dir,
122 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700123 return int(result.output.split()[0])
124
125 def _GetStagingDirSize(self):
126 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800127 redirect_stdout=True,
128 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700129 return int(result.output.split()[0])
130
Ryan Cui3045c5d2012-07-13 18:00:33 -0700131 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500132 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
133 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700134 return result.returncode == 0
135
136 def _DisableRootfsVerification(self):
137 if not self.options.force:
138 logging.error('Detected that the device has rootfs verification enabled.')
139 logging.info('This script can automatically remove the rootfs '
140 'verification, which requires that it reboot the device.')
141 logging.info('Make sure the device is in developer mode!')
142 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700143 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800144 # Since we stopped Chrome earlier, it's good form to start it up again.
145 if self.options.startui:
146 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500147 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800148 raise DeployFailure('Need rootfs verification to be disabled. '
149 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700150
151 logging.info('Removing rootfs verification from %s', self.options.to)
152 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
153 # Use --force to bypass the checks.
154 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
155 '--remove_rootfs_verification --force')
156 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500157 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 # A reboot in developer mode takes a while (and has delays), so the user
160 # will have time to read and act on the USB boot instructions below.
161 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500162 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700163
David James88e6f032013-03-02 08:13:20 -0800164 # Now that the machine has been rebooted, we need to kill Chrome again.
165 self._KillProcsIfNeeded()
166
167 # Make sure the rootfs is writable now.
168 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700169
170 def _CheckUiJobStarted(self):
171 # status output is in the format:
172 # <job_name> <status> ['process' <pid>].
173 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500175 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800176 except cros_build_lib.RunCommandError as e:
177 if 'Unknown job' in e.result.error:
178 return False
179 else:
180 raise e
181
Ryan Cui3045c5d2012-07-13 18:00:33 -0700182 return result.output.split()[1].split('/')[0] == 'start'
183
184 def _KillProcsIfNeeded(self):
185 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800186 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500187 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188
189 # Developers sometimes run session_manager manually, in which case we'll
190 # need to help shut the chrome processes down.
191 try:
David James3432acd2013-11-27 10:02:18 -0800192 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700193 while self._ChromeFileInUse():
194 logging.warning('The chrome binary on the device is in use.')
195 logging.warning('Killing chrome and session_manager processes...\n')
196
Robert Flack1dc7ea82014-11-26 13:50:24 -0500197 self.device.RunCommand("pkill 'chrome|session_manager'",
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 error_code_ok=True)
199 # Wait for processes to actually terminate
200 time.sleep(POST_KILL_WAIT)
201 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800202 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800203 msg = ('Could not kill processes after %s seconds. Please exit any '
204 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
221 not self.device.IsPathWritable(self.options.target_dir)):
222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Ryan Cui3045c5d2012-07-13 18:00:33 -0700248 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800249 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700250 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800251 dest_path = _CHROME_DIR
Robert Flack1dc7ea82014-11-26 13:50:24 -0500252 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
253 self.options.target_dir,
254 inplace=True, debug_level=logging.INFO,
255 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800256
257 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800258 if p.mode:
259 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500260 self.device.RunCommand('chmod %o %s/%s' % (
261 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800262
263
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800264 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800265 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500266 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700267
David James88e6f032013-03-02 08:13:20 -0800268 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700269 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800270 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500271 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800272 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700273 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800274 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700275
Steve Funge984a532013-11-25 17:09:25 -0800276 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700277 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700278 def BinaryExists(filename):
279 """Checks if the passed-in file is present in the build directory."""
280 return os.path.exists(os.path.join(self.options.build_dir, filename))
281
Daniel Erat9813f0e2014-11-12 11:00:28 -0700282 # Handle non-Chrome deployments.
283 if not BinaryExists('chrome'):
284 if BinaryExists('envoy_shell'):
285 self.copy_paths = chrome_util.GetCopyPaths('envoy')
286 elif BinaryExists('app_shell'):
287 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
288
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700289 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700290 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700291 self.options.startui = False
292
David James88e6f032013-03-02 08:13:20 -0800293 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800294 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
295 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800296
Thiago Goncales12793312013-05-23 11:26:17 -0700297 def _MountTarget(self):
298 logging.info('Mounting Chrome...')
299
300 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500301 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
302 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900303 # Umount the existing mount on mount_dir if present first
304 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
305 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500306 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
307 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700308 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500309 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
310
311 def Cleanup(self):
312 """Clean up RemoteDevice."""
313 if not self.options.staging_only:
314 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700315
David James88e6f032013-03-02 08:13:20 -0800316 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800317 self._CheckDeployType()
318
David James88e6f032013-03-02 08:13:20 -0800319 # If requested, just do the staging step.
320 if self.options.staging_only:
321 self._PrepareStagingDir()
322 return 0
323
324 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800325 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800326 steps = [self._GetDeviceInfo, self._CheckConnection,
327 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
328 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700329 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
330 return_values=True)
331 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800332
Robert Flack1dc7ea82014-11-26 13:50:24 -0500333 # If we're trying to deploy to a dir which is not writable and we failed
334 # to mark the rootfs as writable, try disabling rootfs verification.
335 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800336 self._DisableRootfsVerification()
337
Thiago Goncales12793312013-05-23 11:26:17 -0700338 if self.options.mount_dir is not None:
339 self._MountTarget()
340
David James88e6f032013-03-02 08:13:20 -0800341 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700342 self._Deploy()
343
344
Ryan Cuia56a71e2012-10-18 18:40:35 -0700345def ValidateGypDefines(_option, _opt, value):
346 """Convert GYP_DEFINES-formatted string to dictionary."""
347 return chrome_util.ProcessGypDefines(value)
348
349
350class CustomOption(commandline.Option):
351 """Subclass Option class to implement path evaluation."""
352 TYPES = commandline.Option.TYPES + ('gyp_defines',)
353 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
354 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
355
356
Ryan Cuie535b172012-10-19 18:25:03 -0700357def _CreateParser():
358 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800359 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
360 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700361
Ryan Cuia56a71e2012-10-18 18:40:35 -0700362 # TODO(rcui): Have this use the UI-V2 format of having source and target
363 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700364 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800365 help='Skip all prompts (i.e., for disabling of rootfs '
366 'verification). This may result in the target '
367 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800368 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
369 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800370 help="The board the Chrome build is targeted for. When in "
371 "a 'cros chrome-sdk' shell, defaults to the SDK "
372 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700373 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800374 help='The directory with Chrome build artifacts to deploy '
375 'from. Typically of format <chrome_root>/out/Debug. '
376 'When this option is used, the GYP_DEFINES '
377 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800378 parser.add_option('--target-dir', type='path',
379 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700380 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700381 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800382 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800383 parser.add_option('--nostartui', action='store_false', dest='startui',
384 default=True,
385 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800386 parser.add_option('--nostrip', action='store_false', dest='dostrip',
387 default=True,
388 help="Don't strip binaries during deployment. Warning: "
389 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700390 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800391 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700392 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800393 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700394 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800395 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700396 parser.add_option('--mount-dir', type='path', default=None,
Mike Frysingercd304692014-10-02 16:33:41 -0400397 help='Deploy Chrome in target directory and bind it '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900398 'to the directory specified by this flag.'
399 'Any existing mount on this directory will be '
400 'umounted first.')
Thiago Goncales12793312013-05-23 11:26:17 -0700401 parser.add_option('--mount', action='store_true', default=False,
Mike Frysingercd304692014-10-02 16:33:41 -0400402 help='Deploy Chrome to default target directory and bind '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900403 'it to the default mount directory.'
404 'Any existing mount on this directory will be '
405 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700406
407 group = optparse.OptionGroup(parser, 'Advanced Options')
408 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800409 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800410 group.add_option('--sloppy', action='store_true', default=False,
411 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700412 group.add_option('--staging-flags', default=None, type='gyp_defines',
413 help='Extra flags to control staging. Valid flags are - %s'
414 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800415 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800416 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800417 'variable and --staging-flags, if set. Enforce that '
418 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700419 group.add_option('--strip-flags', default=None,
420 help="Flags to call the 'strip' binutil tool with. "
421 "Overrides the default arguments.")
Pawel Osciak141b2262014-12-21 10:27:05 +0900422 group.add_option('--ping', action='store_true', default=False,
423 help='Ping the device before connection attempt.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700424 parser.add_option_group(group)
425
Aviv Keshet1c986f32014-04-24 13:20:49 -0700426 group = optparse.OptionGroup(parser, 'Metadata Overrides (Advanced)',
427 description='Provide all of these overrides '
428 'in order to remove dependencies on '
429 'metadata.json existence.')
430 group.add_option('--target-tc', action='store', default=None,
431 help='Override target toolchain name, e.g. '
432 'x86_64-cros-linux-gnu')
433 group.add_option('--toolchain-url', action='store', default=None,
434 help='Override toolchain url format pattern, e.g. '
435 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
436 parser.add_option_group(group)
437
438
Ryan Cuif890a3e2013-03-07 18:57:06 -0800439 # GYP_DEFINES that Chrome was built with. Influences which files are staged
440 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
441 # enviroment variable.
442 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
443 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700444 # Path of an empty directory to stage chrome artifacts to. Defaults to a
445 # temporary directory that is removed when the script finishes. If the path
446 # is specified, then it will not be removed.
447 parser.add_option('--staging-dir', type='path', default=None,
448 help=optparse.SUPPRESS_HELP)
449 # Only prepare the staging directory, and skip deploying to the device.
450 parser.add_option('--staging-only', action='store_true', default=False,
451 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700452 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
453 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
454 # fetching the SDK toolchain.
455 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700456 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700457
Ryan Cuie535b172012-10-19 18:25:03 -0700458
459def _ParseCommandLine(argv):
460 """Parse args, and run environment-independent checks."""
461 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700462 (options, args) = parser.parse_args(argv)
463
Ryan Cuia56a71e2012-10-18 18:40:35 -0700464 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
465 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800466 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700467 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
468 parser.error('Cannot specify both --build_dir and '
469 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800470 if options.build_dir and not options.board:
471 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472 if options.gs_path and options.local_pkg_path:
473 parser.error('Cannot specify both --gs-path and --local-pkg-path')
474 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700475 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800476 if (options.strict or options.staging_flags) and not options.build_dir:
477 parser.error('--strict and --staging-flags require --build-dir to be '
478 'set.')
479 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800480 parser.error('--staging-flags requires --strict to be set.')
481 if options.sloppy and options.strict:
482 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700483
484 if options.mount or options.mount_dir:
485 if not options.target_dir:
486 options.target_dir = _CHROME_DIR_MOUNT
487 else:
488 if not options.target_dir:
489 options.target_dir = _CHROME_DIR
490
491 if options.mount and not options.mount_dir:
492 options.mount_dir = _CHROME_DIR
493
Ryan Cui3045c5d2012-07-13 18:00:33 -0700494 return options, args
495
496
Ryan Cuie535b172012-10-19 18:25:03 -0700497def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800498 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700499
500 Args:
Steve Funge984a532013-11-25 17:09:25 -0800501 options: The options object returned by optparse.
502 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700503 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700504 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
505 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
506
Ryan Cuib623e7b2013-03-14 12:54:11 -0700507 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700508 gyp_env = os.getenv('GYP_DEFINES', None)
509 if gyp_env is not None:
510 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700511 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700512 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700513
514 if options.strict and not options.gyp_defines:
515 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800516 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700517
Ryan Cui3c183c22013-04-29 18:04:11 -0700518 if options.build_dir:
519 chrome_path = os.path.join(options.build_dir, 'chrome')
520 if os.path.isfile(chrome_path):
521 deps = lddtree.ParseELF(chrome_path)
522 if 'libbase.so' in deps['libs']:
523 cros_build_lib.Warning(
524 'Detected a component build of Chrome. component build is '
525 'not working properly for Chrome OS. See crbug.com/196317. '
526 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700527
Ryan Cuia56a71e2012-10-18 18:40:35 -0700528
Ryan Cui504db722013-01-22 11:48:01 -0800529def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700530 """Get the chrome prebuilt tarball from GS.
531
Mike Frysinger02e1e072013-11-10 22:11:34 -0500532 Returns:
533 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700534 """
David James9374aac2013-10-08 16:00:17 -0700535 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500536 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800537 files = [found for found in files if
538 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
539 if not files:
540 raise Exception('No chrome package found at %s' % gs_path)
541 elif len(files) > 1:
542 # - Users should provide us with a direct link to either a stripped or
543 # unstripped chrome package.
544 # - In the case of being provided with an archive directory, where both
545 # stripped and unstripped chrome available, use the stripped chrome
546 # package.
547 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
548 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
549 files = [f for f in files if not 'unstripped' in f]
550 assert len(files) == 1
551 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800552
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800553 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800554 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800555 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
556 chrome_path = os.path.join(tempdir, filename)
557 assert os.path.exists(chrome_path)
558 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700559
560
Ryan Cuif890a3e2013-03-07 18:57:06 -0800561@contextlib.contextmanager
562def _StripBinContext(options):
563 if not options.dostrip:
564 yield None
565 elif options.strip_bin:
566 yield options.strip_bin
567 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800568 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800569 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700570 with sdk.Prepare(components=components, target_tc=options.target_tc,
571 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800572 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
573 constants.CHROME_ENV_FILE)
574 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
575 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
576 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800577 yield strip_bin
578
579
Steve Funge984a532013-11-25 17:09:25 -0800580def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
581 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800582 """Place the necessary files in the staging directory.
583
584 The staging directory is the directory used to rsync the build artifacts over
585 to the device. Only the necessary Chrome build artifacts are put into the
586 staging directory.
587 """
Ryan Cui5866be02013-03-18 14:12:00 -0700588 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400589 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800590 if options.build_dir:
591 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700592 strip_flags = (None if options.strip_flags is None else
593 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800594 chrome_util.StageChromeFromBuildDir(
595 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800596 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700597 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800598 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700599 else:
600 pkg_path = options.local_pkg_path
601 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800602 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
603 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700604
605 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800606 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700607 # Extract only the ./opt/google/chrome contents, directly into the staging
608 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800609 if pkg_path[-4:] == '.zip':
610 cros_build_lib.DebugRunCommand(
611 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
612 staging_dir])
613 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
614 shutil.move(filename, staging_dir)
615 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
616 else:
617 cros_build_lib.DebugRunCommand(
618 ['tar', '--strip-components', '4', '--extract',
619 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
620 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700621
Ryan Cui71aa8de2013-04-19 16:12:55 -0700622
Ryan Cui3045c5d2012-07-13 18:00:33 -0700623def main(argv):
624 options, args = _ParseCommandLine(argv)
625 _PostParseCheck(options, args)
626
627 # Set cros_build_lib debug level to hide RunCommand spew.
628 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700629 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700630 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800631 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700632
Ryan Cui71aa8de2013-04-19 16:12:55 -0700633 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700634 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
635 if cmd_stats:
636 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700637
Ryan Cui71aa8de2013-04-19 16:12:55 -0700638 with osutils.TempDir(set_global=True) as tempdir:
639 staging_dir = options.staging_dir
640 if not staging_dir:
641 staging_dir = os.path.join(tempdir, 'chrome')
642
643 deploy = DeployChrome(options, tempdir, staging_dir)
644 try:
645 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700646 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700647 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500648 deploy.Cleanup()