blob: 1cf01f4e98bffb9d2a906f7ee54e1ceefe43f8e9 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Ryan Cuia56a71e2012-10-18 18:40:35 -07005
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger1d4752b2014-11-08 04:00:18 -050019# pylint: disable=bad-continuation
20
Mike Frysinger383367e2014-09-16 15:06:17 -040021from __future__ import print_function
22
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080024import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070025import functools
Steve Funge984a532013-11-25 17:09:25 -080026import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import logging
David James88e6f032013-03-02 08:13:20 -080028import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070030import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070031import shlex
Steve Funge984a532013-11-25 17:09:25 -080032import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070033import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070034
Ryan Cuia56a71e2012-10-18 18:40:35 -070035
Don Garrett88b8d782014-05-13 17:30:55 -070036from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070037from chromite.cbuildbot import failures_lib
Ryan Cui686ec052013-02-12 16:39:41 -080038from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070039from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070041from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080042from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070043from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080044from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070045from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070046from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080047from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070048from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070049
50
Ryan Cuia56a71e2012-10-18 18:40:35 -070051_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
52
Ryan Cui3045c5d2012-07-13 18:00:33 -070053KERNEL_A_PARTITION = 2
54KERNEL_B_PARTITION = 4
55
56KILL_PROC_MAX_WAIT = 10
57POST_KILL_WAIT = 2
58
Ryan Cuie535b172012-10-19 18:25:03 -070059MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080060LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070061
Steve Funge984a532013-11-25 17:09:25 -080062_ANDROID_DIR = '/system/chrome'
63_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
64
David James2cb34002013-03-01 18:42:40 -080065_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070066_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080067
Pawel Osciakc1bb2742014-12-29 16:32:33 +090068_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
69 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070070_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
71_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070072
Steve Funge984a532013-11-25 17:09:25 -080073DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080074
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070080class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080081 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -0500102 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900103 self.device = remote.RemoteDevice(options.to, port=options.port,
104 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500105 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700106
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700107 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500111 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700112 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800113 value = line.split()[3]
114 multipliers = {
115 'G': 1024 * 1024 * 1024,
116 'M': 1024 * 1024,
117 'K': 1024,
118 }
119 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700120
121 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500122 result = self.device.RunCommand('du -ks %s' % remote_dir,
123 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700124 return int(result.output.split()[0])
125
126 def _GetStagingDirSize(self):
127 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800128 redirect_stdout=True,
129 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700130 return int(result.output.split()[0])
131
Ryan Cui3045c5d2012-07-13 18:00:33 -0700132 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500133 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
134 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700135 return result.returncode == 0
136
137 def _DisableRootfsVerification(self):
138 if not self.options.force:
139 logging.error('Detected that the device has rootfs verification enabled.')
140 logging.info('This script can automatically remove the rootfs '
141 'verification, which requires that it reboot the device.')
142 logging.info('Make sure the device is in developer mode!')
143 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700144 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800145 # Since we stopped Chrome earlier, it's good form to start it up again.
146 if self.options.startui:
147 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500148 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800149 raise DeployFailure('Need rootfs verification to be disabled. '
150 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700151
152 logging.info('Removing rootfs verification from %s', self.options.to)
153 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
154 # Use --force to bypass the checks.
155 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
156 '--remove_rootfs_verification --force')
157 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500158 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700159
160 # A reboot in developer mode takes a while (and has delays), so the user
161 # will have time to read and act on the USB boot instructions below.
162 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500163 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700164
David James88e6f032013-03-02 08:13:20 -0800165 # Now that the machine has been rebooted, we need to kill Chrome again.
166 self._KillProcsIfNeeded()
167
168 # Make sure the rootfs is writable now.
169 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700170
171 def _CheckUiJobStarted(self):
172 # status output is in the format:
173 # <job_name> <status> ['process' <pid>].
174 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500176 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800177 except cros_build_lib.RunCommandError as e:
178 if 'Unknown job' in e.result.error:
179 return False
180 else:
181 raise e
182
Ryan Cui3045c5d2012-07-13 18:00:33 -0700183 return result.output.split()[1].split('/')[0] == 'start'
184
185 def _KillProcsIfNeeded(self):
186 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800187 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500188 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700189
190 # Developers sometimes run session_manager manually, in which case we'll
191 # need to help shut the chrome processes down.
192 try:
David James3432acd2013-11-27 10:02:18 -0800193 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700194 while self._ChromeFileInUse():
195 logging.warning('The chrome binary on the device is in use.')
196 logging.warning('Killing chrome and session_manager processes...\n')
197
Robert Flack1dc7ea82014-11-26 13:50:24 -0500198 self.device.RunCommand("pkill 'chrome|session_manager'",
Ryan Cui3045c5d2012-07-13 18:00:33 -0700199 error_code_ok=True)
200 # Wait for processes to actually terminate
201 time.sleep(POST_KILL_WAIT)
202 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800203 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800204 msg = ('Could not kill processes after %s seconds. Please exit any '
205 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
206 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700207
David James88e6f032013-03-02 08:13:20 -0800208 def _MountRootfsAsWritable(self, error_code_ok=True):
209 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700210
Robert Flack1dc7ea82014-11-26 13:50:24 -0500211 If the command fails, and error_code_ok is True, and the target dir is not
212 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700213
Mike Frysinger02e1e072013-11-10 22:11:34 -0500214 Args:
David James88e6f032013-03-02 08:13:20 -0800215 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
216 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800217 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500218 result = self.device.RunCommand(MOUNT_RW_COMMAND,
219 error_code_ok=error_code_ok,
220 capture_output=True)
221 if (result.returncode and
222 not self.device.IsPathWritable(self.options.target_dir)):
223 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700224
Ryan Cui7193a7e2013-04-26 14:15:19 -0700225 def _GetDeviceInfo(self):
226 steps = [
227 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
228 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
229 ]
230 return_values = parallel.RunParallelSteps(steps, return_values=True)
231 return DeviceInfo(*return_values)
232
233 def _CheckDeviceFreeSpace(self, device_info):
234 """See if target device has enough space for Chrome.
235
Mike Frysinger02e1e072013-11-10 22:11:34 -0500236 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700237 device_info: A DeviceInfo named tuple.
238 """
239 effective_free = device_info.target_dir_size + device_info.target_fs_free
240 staging_size = self._GetStagingDirSize()
241 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700242 raise DeployFailure(
243 'Not enough free space on the device. Required: %s MiB, '
244 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700245 if device_info.target_fs_free < (100 * 1024):
246 logging.warning('The device has less than 100MB free. deploy_chrome may '
247 'hang during the transfer.')
248
Ryan Cui3045c5d2012-07-13 18:00:33 -0700249 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800250 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700251 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800252 dest_path = _CHROME_DIR
Robert Flack1dc7ea82014-11-26 13:50:24 -0500253 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
254 self.options.target_dir,
255 inplace=True, debug_level=logging.INFO,
256 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800257
258 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800259 if p.mode:
260 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500261 self.device.RunCommand('chmod %o %s/%s' % (
262 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800263
264
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800265 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800266 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500267 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700268
David James88e6f032013-03-02 08:13:20 -0800269 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800271 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500272 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800273 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700274 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800275 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700276
Steve Funge984a532013-11-25 17:09:25 -0800277 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700278 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700279 def BinaryExists(filename):
280 """Checks if the passed-in file is present in the build directory."""
281 return os.path.exists(os.path.join(self.options.build_dir, filename))
282
Daniel Erat9813f0e2014-11-12 11:00:28 -0700283 # Handle non-Chrome deployments.
284 if not BinaryExists('chrome'):
285 if BinaryExists('envoy_shell'):
286 self.copy_paths = chrome_util.GetCopyPaths('envoy')
287 elif BinaryExists('app_shell'):
288 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
289
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700290 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700291 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700292 self.options.startui = False
293
David James88e6f032013-03-02 08:13:20 -0800294 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800295 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
296 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800297
Thiago Goncales12793312013-05-23 11:26:17 -0700298 def _MountTarget(self):
299 logging.info('Mounting Chrome...')
300
301 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500302 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
303 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900304 # Umount the existing mount on mount_dir if present first
305 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
306 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500307 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
308 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700309 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500310 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
311
312 def Cleanup(self):
313 """Clean up RemoteDevice."""
314 if not self.options.staging_only:
315 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700316
David James88e6f032013-03-02 08:13:20 -0800317 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800318 self._CheckDeployType()
319
David James88e6f032013-03-02 08:13:20 -0800320 # If requested, just do the staging step.
321 if self.options.staging_only:
322 self._PrepareStagingDir()
323 return 0
324
325 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800326 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800327 steps = [self._GetDeviceInfo, self._CheckConnection,
328 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
329 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700330 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
331 return_values=True)
332 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800333
Robert Flack1dc7ea82014-11-26 13:50:24 -0500334 # If we're trying to deploy to a dir which is not writable and we failed
335 # to mark the rootfs as writable, try disabling rootfs verification.
336 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800337 self._DisableRootfsVerification()
338
Thiago Goncales12793312013-05-23 11:26:17 -0700339 if self.options.mount_dir is not None:
340 self._MountTarget()
341
David James88e6f032013-03-02 08:13:20 -0800342 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700343 self._Deploy()
344
345
Ryan Cuia56a71e2012-10-18 18:40:35 -0700346def ValidateGypDefines(_option, _opt, value):
347 """Convert GYP_DEFINES-formatted string to dictionary."""
348 return chrome_util.ProcessGypDefines(value)
349
350
351class CustomOption(commandline.Option):
352 """Subclass Option class to implement path evaluation."""
353 TYPES = commandline.Option.TYPES + ('gyp_defines',)
354 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
355 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
356
357
Ryan Cuie535b172012-10-19 18:25:03 -0700358def _CreateParser():
359 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800360 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
361 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700362
Ryan Cuia56a71e2012-10-18 18:40:35 -0700363 # TODO(rcui): Have this use the UI-V2 format of having source and target
364 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700365 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800366 help='Skip all prompts (i.e., for disabling of rootfs '
367 'verification). This may result in the target '
368 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800369 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
370 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800371 help="The board the Chrome build is targeted for. When in "
372 "a 'cros chrome-sdk' shell, defaults to the SDK "
373 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700374 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800375 help='The directory with Chrome build artifacts to deploy '
376 'from. Typically of format <chrome_root>/out/Debug. '
377 'When this option is used, the GYP_DEFINES '
378 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800379 parser.add_option('--target-dir', type='path',
380 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700381 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700382 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800383 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800384 parser.add_option('--nostartui', action='store_false', dest='startui',
385 default=True,
386 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800387 parser.add_option('--nostrip', action='store_false', dest='dostrip',
388 default=True,
389 help="Don't strip binaries during deployment. Warning: "
390 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700391 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800392 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700393 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800394 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700395 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800396 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700397 parser.add_option('--mount-dir', type='path', default=None,
Mike Frysingercd304692014-10-02 16:33:41 -0400398 help='Deploy Chrome in target directory and bind it '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900399 'to the directory specified by this flag.'
400 'Any existing mount on this directory will be '
401 'umounted first.')
Thiago Goncales12793312013-05-23 11:26:17 -0700402 parser.add_option('--mount', action='store_true', default=False,
Mike Frysingercd304692014-10-02 16:33:41 -0400403 help='Deploy Chrome to default target directory and bind '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900404 'it to the default mount directory.'
405 'Any existing mount on this directory will be '
406 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700407
408 group = optparse.OptionGroup(parser, 'Advanced Options')
409 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800410 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800411 group.add_option('--sloppy', action='store_true', default=False,
412 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700413 group.add_option('--staging-flags', default=None, type='gyp_defines',
414 help='Extra flags to control staging. Valid flags are - %s'
415 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800416 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800417 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800418 'variable and --staging-flags, if set. Enforce that '
419 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700420 group.add_option('--strip-flags', default=None,
421 help="Flags to call the 'strip' binutil tool with. "
422 "Overrides the default arguments.")
Pawel Osciak141b2262014-12-21 10:27:05 +0900423 group.add_option('--ping', action='store_true', default=False,
424 help='Ping the device before connection attempt.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700425 parser.add_option_group(group)
426
Aviv Keshet1c986f32014-04-24 13:20:49 -0700427 group = optparse.OptionGroup(parser, 'Metadata Overrides (Advanced)',
428 description='Provide all of these overrides '
429 'in order to remove dependencies on '
430 'metadata.json existence.')
431 group.add_option('--target-tc', action='store', default=None,
432 help='Override target toolchain name, e.g. '
433 'x86_64-cros-linux-gnu')
434 group.add_option('--toolchain-url', action='store', default=None,
435 help='Override toolchain url format pattern, e.g. '
436 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
437 parser.add_option_group(group)
438
439
Ryan Cuif890a3e2013-03-07 18:57:06 -0800440 # GYP_DEFINES that Chrome was built with. Influences which files are staged
441 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
442 # enviroment variable.
443 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
444 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700445 # Path of an empty directory to stage chrome artifacts to. Defaults to a
446 # temporary directory that is removed when the script finishes. If the path
447 # is specified, then it will not be removed.
448 parser.add_option('--staging-dir', type='path', default=None,
449 help=optparse.SUPPRESS_HELP)
450 # Only prepare the staging directory, and skip deploying to the device.
451 parser.add_option('--staging-only', action='store_true', default=False,
452 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700453 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
454 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
455 # fetching the SDK toolchain.
456 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700457 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700458
Ryan Cuie535b172012-10-19 18:25:03 -0700459
460def _ParseCommandLine(argv):
461 """Parse args, and run environment-independent checks."""
462 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700463 (options, args) = parser.parse_args(argv)
464
Ryan Cuia56a71e2012-10-18 18:40:35 -0700465 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
466 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800467 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700468 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
469 parser.error('Cannot specify both --build_dir and '
470 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800471 if options.build_dir and not options.board:
472 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700473 if options.gs_path and options.local_pkg_path:
474 parser.error('Cannot specify both --gs-path and --local-pkg-path')
475 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700476 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800477 if (options.strict or options.staging_flags) and not options.build_dir:
478 parser.error('--strict and --staging-flags require --build-dir to be '
479 'set.')
480 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800481 parser.error('--staging-flags requires --strict to be set.')
482 if options.sloppy and options.strict:
483 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700484
485 if options.mount or options.mount_dir:
486 if not options.target_dir:
487 options.target_dir = _CHROME_DIR_MOUNT
488 else:
489 if not options.target_dir:
490 options.target_dir = _CHROME_DIR
491
492 if options.mount and not options.mount_dir:
493 options.mount_dir = _CHROME_DIR
494
Ryan Cui3045c5d2012-07-13 18:00:33 -0700495 return options, args
496
497
Ryan Cuie535b172012-10-19 18:25:03 -0700498def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800499 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700500
501 Args:
Steve Funge984a532013-11-25 17:09:25 -0800502 options: The options object returned by optparse.
503 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700504 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700505 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
506 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
507
Ryan Cuib623e7b2013-03-14 12:54:11 -0700508 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700509 gyp_env = os.getenv('GYP_DEFINES', None)
510 if gyp_env is not None:
511 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700512 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700513 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700514
515 if options.strict and not options.gyp_defines:
516 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800517 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700518
Ryan Cui3c183c22013-04-29 18:04:11 -0700519 if options.build_dir:
520 chrome_path = os.path.join(options.build_dir, 'chrome')
521 if os.path.isfile(chrome_path):
522 deps = lddtree.ParseELF(chrome_path)
523 if 'libbase.so' in deps['libs']:
524 cros_build_lib.Warning(
525 'Detected a component build of Chrome. component build is '
526 'not working properly for Chrome OS. See crbug.com/196317. '
527 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700528
Ryan Cuia56a71e2012-10-18 18:40:35 -0700529
Ryan Cui504db722013-01-22 11:48:01 -0800530def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700531 """Get the chrome prebuilt tarball from GS.
532
Mike Frysinger02e1e072013-11-10 22:11:34 -0500533 Returns:
534 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700535 """
David James9374aac2013-10-08 16:00:17 -0700536 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500537 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800538 files = [found for found in files if
539 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
540 if not files:
541 raise Exception('No chrome package found at %s' % gs_path)
542 elif len(files) > 1:
543 # - Users should provide us with a direct link to either a stripped or
544 # unstripped chrome package.
545 # - In the case of being provided with an archive directory, where both
546 # stripped and unstripped chrome available, use the stripped chrome
547 # package.
548 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
549 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
550 files = [f for f in files if not 'unstripped' in f]
551 assert len(files) == 1
552 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800553
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800554 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800555 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800556 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
557 chrome_path = os.path.join(tempdir, filename)
558 assert os.path.exists(chrome_path)
559 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700560
561
Ryan Cuif890a3e2013-03-07 18:57:06 -0800562@contextlib.contextmanager
563def _StripBinContext(options):
564 if not options.dostrip:
565 yield None
566 elif options.strip_bin:
567 yield options.strip_bin
568 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800569 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800570 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700571 with sdk.Prepare(components=components, target_tc=options.target_tc,
572 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800573 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
574 constants.CHROME_ENV_FILE)
575 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
576 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
577 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800578 yield strip_bin
579
580
Steve Funge984a532013-11-25 17:09:25 -0800581def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
582 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800583 """Place the necessary files in the staging directory.
584
585 The staging directory is the directory used to rsync the build artifacts over
586 to the device. Only the necessary Chrome build artifacts are put into the
587 staging directory.
588 """
Ryan Cui5866be02013-03-18 14:12:00 -0700589 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400590 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800591 if options.build_dir:
592 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700593 strip_flags = (None if options.strip_flags is None else
594 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800595 chrome_util.StageChromeFromBuildDir(
596 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800597 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700598 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800599 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700600 else:
601 pkg_path = options.local_pkg_path
602 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800603 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
604 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700605
606 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800607 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700608 # Extract only the ./opt/google/chrome contents, directly into the staging
609 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800610 if pkg_path[-4:] == '.zip':
611 cros_build_lib.DebugRunCommand(
612 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
613 staging_dir])
614 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
615 shutil.move(filename, staging_dir)
616 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
617 else:
618 cros_build_lib.DebugRunCommand(
619 ['tar', '--strip-components', '4', '--extract',
620 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
621 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700622
Ryan Cui71aa8de2013-04-19 16:12:55 -0700623
Ryan Cui3045c5d2012-07-13 18:00:33 -0700624def main(argv):
625 options, args = _ParseCommandLine(argv)
626 _PostParseCheck(options, args)
627
628 # Set cros_build_lib debug level to hide RunCommand spew.
629 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700630 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700631 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800632 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700633
Ryan Cui71aa8de2013-04-19 16:12:55 -0700634 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700635 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
636 if cmd_stats:
637 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700638
Ryan Cui71aa8de2013-04-19 16:12:55 -0700639 with osutils.TempDir(set_global=True) as tempdir:
640 staging_dir = options.staging_dir
641 if not staging_dir:
642 staging_dir = os.path.join(tempdir, 'chrome')
643
644 deploy = DeployChrome(options, tempdir, staging_dir)
645 try:
646 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700647 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700648 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500649 deploy.Cleanup()