blob: c17aef22d340d060fa8817df9fec354f2250545d [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Ryan Cuia56a71e2012-10-18 18:40:35 -07005
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger1d4752b2014-11-08 04:00:18 -050019# pylint: disable=bad-continuation
20
Mike Frysinger383367e2014-09-16 15:06:17 -040021from __future__ import print_function
22
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080024import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070025import functools
Steve Funge984a532013-11-25 17:09:25 -080026import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import logging
David James88e6f032013-03-02 08:13:20 -080028import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070030import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070031import shlex
Steve Funge984a532013-11-25 17:09:25 -080032import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070033import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070034
Ryan Cuia56a71e2012-10-18 18:40:35 -070035
Don Garrett88b8d782014-05-13 17:30:55 -070036from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070037from chromite.cbuildbot import failures_lib
Ryan Cui686ec052013-02-12 16:39:41 -080038from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070039from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070041from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080042from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070043from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080044from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070045from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070046from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080047from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070048from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070049
50
Ryan Cuia56a71e2012-10-18 18:40:35 -070051_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
52
Ryan Cui3045c5d2012-07-13 18:00:33 -070053KERNEL_A_PARTITION = 2
54KERNEL_B_PARTITION = 4
55
56KILL_PROC_MAX_WAIT = 10
57POST_KILL_WAIT = 2
58
Ryan Cuie535b172012-10-19 18:25:03 -070059MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080060LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070061
Steve Funge984a532013-11-25 17:09:25 -080062_ANDROID_DIR = '/system/chrome'
63_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
64
David James2cb34002013-03-01 18:42:40 -080065_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070066_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080067
Pawel Osciakfb200f52014-12-21 13:42:05 +090068_UMOUNT_DIR_IF_MOUNTPOINT_CMD = 'mountpoint -q %(dir)s && umount %(dir)s'
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080073
Ryan Cui3045c5d2012-07-13 18:00:33 -070074def _UrlBaseName(url):
75 """Return the last component of the URL."""
76 return url.rstrip('/').rpartition('/')[-1]
77
78
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070079class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080080 """Raised whenever the deploy fails."""
81
82
Ryan Cui7193a7e2013-04-26 14:15:19 -070083DeviceInfo = collections.namedtuple(
84 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
85
86
Ryan Cui3045c5d2012-07-13 18:00:33 -070087class DeployChrome(object):
88 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070089 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070090 """Initialize the class.
91
Mike Frysinger02e1e072013-11-10 22:11:34 -050092 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070093 options: Optparse result structure.
94 tempdir: Scratch space for the class. Caller has responsibility to clean
95 it up.
Steve Funge984a532013-11-25 17:09:25 -080096 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070097 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070098 self.tempdir = tempdir
99 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700100 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -0500101 if not self.options.staging_only:
102 self.device = remote.RemoteDevice(options.to, port=options.port)
103 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700105 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800106 self.chrome_dir = _CHROME_DIR
107
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500109 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800111 value = line.split()[3]
112 multipliers = {
113 'G': 1024 * 1024 * 1024,
114 'M': 1024 * 1024,
115 'K': 1024,
116 }
117 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700118
119 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500120 result = self.device.RunCommand('du -ks %s' % remote_dir,
121 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122 return int(result.output.split()[0])
123
124 def _GetStagingDirSize(self):
125 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800126 redirect_stdout=True,
127 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700128 return int(result.output.split()[0])
129
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500131 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
132 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700133 return result.returncode == 0
134
135 def _DisableRootfsVerification(self):
136 if not self.options.force:
137 logging.error('Detected that the device has rootfs verification enabled.')
138 logging.info('This script can automatically remove the rootfs '
139 'verification, which requires that it reboot the device.')
140 logging.info('Make sure the device is in developer mode!')
141 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700142 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800143 # Since we stopped Chrome earlier, it's good form to start it up again.
144 if self.options.startui:
145 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500146 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800147 raise DeployFailure('Need rootfs verification to be disabled. '
148 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700149
150 logging.info('Removing rootfs verification from %s', self.options.to)
151 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
152 # Use --force to bypass the checks.
153 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
154 '--remove_rootfs_verification --force')
155 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500156 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700157
158 # A reboot in developer mode takes a while (and has delays), so the user
159 # will have time to read and act on the USB boot instructions below.
160 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500161 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
David James3432acd2013-11-27 10:02:18 -0800191 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Ryan Cui3045c5d2012-07-13 18:00:33 -0700197 error_code_ok=True)
198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
203 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
204 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
David James88e6f032013-03-02 08:13:20 -0800206 def _MountRootfsAsWritable(self, error_code_ok=True):
207 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Robert Flack1dc7ea82014-11-26 13:50:24 -0500209 If the command fails, and error_code_ok is True, and the target dir is not
210 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700211
Mike Frysinger02e1e072013-11-10 22:11:34 -0500212 Args:
David James88e6f032013-03-02 08:13:20 -0800213 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
214 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800215 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500216 result = self.device.RunCommand(MOUNT_RW_COMMAND,
217 error_code_ok=error_code_ok,
218 capture_output=True)
219 if (result.returncode and
220 not self.device.IsPathWritable(self.options.target_dir)):
221 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
Ryan Cui7193a7e2013-04-26 14:15:19 -0700223 def _GetDeviceInfo(self):
224 steps = [
225 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
226 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
227 ]
228 return_values = parallel.RunParallelSteps(steps, return_values=True)
229 return DeviceInfo(*return_values)
230
231 def _CheckDeviceFreeSpace(self, device_info):
232 """See if target device has enough space for Chrome.
233
Mike Frysinger02e1e072013-11-10 22:11:34 -0500234 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700235 device_info: A DeviceInfo named tuple.
236 """
237 effective_free = device_info.target_dir_size + device_info.target_fs_free
238 staging_size = self._GetStagingDirSize()
239 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700240 raise DeployFailure(
241 'Not enough free space on the device. Required: %s MiB, '
242 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700243 if device_info.target_fs_free < (100 * 1024):
244 logging.warning('The device has less than 100MB free. deploy_chrome may '
245 'hang during the transfer.')
246
Ryan Cui3045c5d2012-07-13 18:00:33 -0700247 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800248 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700249 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800250 dest_path = _CHROME_DIR
Robert Flack1dc7ea82014-11-26 13:50:24 -0500251 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
252 self.options.target_dir,
253 inplace=True, debug_level=logging.INFO,
254 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800255
256 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800257 if p.mode:
258 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500259 self.device.RunCommand('chmod %o %s/%s' % (
260 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800261
262
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800263 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800264 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500265 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700266
David James88e6f032013-03-02 08:13:20 -0800267 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700268 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800269 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500270 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800271 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800273 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700274
Steve Funge984a532013-11-25 17:09:25 -0800275 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700276 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700277 def BinaryExists(filename):
278 """Checks if the passed-in file is present in the build directory."""
279 return os.path.exists(os.path.join(self.options.build_dir, filename))
280
Daniel Erat9813f0e2014-11-12 11:00:28 -0700281 # Handle non-Chrome deployments.
282 if not BinaryExists('chrome'):
283 if BinaryExists('envoy_shell'):
284 self.copy_paths = chrome_util.GetCopyPaths('envoy')
285 elif BinaryExists('app_shell'):
286 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
287
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700288 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700289 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700290 self.options.startui = False
291
David James88e6f032013-03-02 08:13:20 -0800292 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800293 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
294 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800295
Thiago Goncales12793312013-05-23 11:26:17 -0700296 def _MountTarget(self):
297 logging.info('Mounting Chrome...')
298
299 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500300 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
301 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900302 # Umount the existing mount on mount_dir if present first
303 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
304 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500305 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
306 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700307 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500308 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
309
310 def Cleanup(self):
311 """Clean up RemoteDevice."""
312 if not self.options.staging_only:
313 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700314
David James88e6f032013-03-02 08:13:20 -0800315 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800316 self._CheckDeployType()
317
David James88e6f032013-03-02 08:13:20 -0800318 # If requested, just do the staging step.
319 if self.options.staging_only:
320 self._PrepareStagingDir()
321 return 0
322
323 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800324 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800325 steps = [self._GetDeviceInfo, self._CheckConnection,
326 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
327 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700328 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
329 return_values=True)
330 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800331
Robert Flack1dc7ea82014-11-26 13:50:24 -0500332 # If we're trying to deploy to a dir which is not writable and we failed
333 # to mark the rootfs as writable, try disabling rootfs verification.
334 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800335 self._DisableRootfsVerification()
336
Thiago Goncales12793312013-05-23 11:26:17 -0700337 if self.options.mount_dir is not None:
338 self._MountTarget()
339
David James88e6f032013-03-02 08:13:20 -0800340 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700341 self._Deploy()
342
343
Ryan Cuia56a71e2012-10-18 18:40:35 -0700344def ValidateGypDefines(_option, _opt, value):
345 """Convert GYP_DEFINES-formatted string to dictionary."""
346 return chrome_util.ProcessGypDefines(value)
347
348
349class CustomOption(commandline.Option):
350 """Subclass Option class to implement path evaluation."""
351 TYPES = commandline.Option.TYPES + ('gyp_defines',)
352 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
353 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
354
355
Ryan Cuie535b172012-10-19 18:25:03 -0700356def _CreateParser():
357 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800358 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
359 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700360
Ryan Cuia56a71e2012-10-18 18:40:35 -0700361 # TODO(rcui): Have this use the UI-V2 format of having source and target
362 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700363 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800364 help='Skip all prompts (i.e., for disabling of rootfs '
365 'verification). This may result in the target '
366 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800367 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
368 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800369 help="The board the Chrome build is targeted for. When in "
370 "a 'cros chrome-sdk' shell, defaults to the SDK "
371 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700372 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800373 help='The directory with Chrome build artifacts to deploy '
374 'from. Typically of format <chrome_root>/out/Debug. '
375 'When this option is used, the GYP_DEFINES '
376 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800377 parser.add_option('--target-dir', type='path',
378 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700379 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700380 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800381 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800382 parser.add_option('--nostartui', action='store_false', dest='startui',
383 default=True,
384 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800385 parser.add_option('--nostrip', action='store_false', dest='dostrip',
386 default=True,
387 help="Don't strip binaries during deployment. Warning: "
388 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700389 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800390 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700391 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800392 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700393 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800394 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700395 parser.add_option('--mount-dir', type='path', default=None,
Mike Frysingercd304692014-10-02 16:33:41 -0400396 help='Deploy Chrome in target directory and bind it '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900397 'to the directory specified by this flag.'
398 'Any existing mount on this directory will be '
399 'umounted first.')
Thiago Goncales12793312013-05-23 11:26:17 -0700400 parser.add_option('--mount', action='store_true', default=False,
Mike Frysingercd304692014-10-02 16:33:41 -0400401 help='Deploy Chrome to default target directory and bind '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900402 'it to the default mount directory.'
403 'Any existing mount on this directory will be '
404 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700405
406 group = optparse.OptionGroup(parser, 'Advanced Options')
407 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800408 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800409 group.add_option('--sloppy', action='store_true', default=False,
410 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700411 group.add_option('--staging-flags', default=None, type='gyp_defines',
412 help='Extra flags to control staging. Valid flags are - %s'
413 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800414 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800415 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800416 'variable and --staging-flags, if set. Enforce that '
417 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700418 group.add_option('--strip-flags', default=None,
419 help="Flags to call the 'strip' binutil tool with. "
420 "Overrides the default arguments.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700421 parser.add_option_group(group)
422
Aviv Keshet1c986f32014-04-24 13:20:49 -0700423 group = optparse.OptionGroup(parser, 'Metadata Overrides (Advanced)',
424 description='Provide all of these overrides '
425 'in order to remove dependencies on '
426 'metadata.json existence.')
427 group.add_option('--target-tc', action='store', default=None,
428 help='Override target toolchain name, e.g. '
429 'x86_64-cros-linux-gnu')
430 group.add_option('--toolchain-url', action='store', default=None,
431 help='Override toolchain url format pattern, e.g. '
432 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
433 parser.add_option_group(group)
434
435
Ryan Cuif890a3e2013-03-07 18:57:06 -0800436 # GYP_DEFINES that Chrome was built with. Influences which files are staged
437 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
438 # enviroment variable.
439 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
440 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700441 # Path of an empty directory to stage chrome artifacts to. Defaults to a
442 # temporary directory that is removed when the script finishes. If the path
443 # is specified, then it will not be removed.
444 parser.add_option('--staging-dir', type='path', default=None,
445 help=optparse.SUPPRESS_HELP)
446 # Only prepare the staging directory, and skip deploying to the device.
447 parser.add_option('--staging-only', action='store_true', default=False,
448 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700449 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
450 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
451 # fetching the SDK toolchain.
452 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700453 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700454
Ryan Cuie535b172012-10-19 18:25:03 -0700455
456def _ParseCommandLine(argv):
457 """Parse args, and run environment-independent checks."""
458 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700459 (options, args) = parser.parse_args(argv)
460
Ryan Cuia56a71e2012-10-18 18:40:35 -0700461 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
462 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800463 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700464 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
465 parser.error('Cannot specify both --build_dir and '
466 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800467 if options.build_dir and not options.board:
468 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700469 if options.gs_path and options.local_pkg_path:
470 parser.error('Cannot specify both --gs-path and --local-pkg-path')
471 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700472 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800473 if (options.strict or options.staging_flags) and not options.build_dir:
474 parser.error('--strict and --staging-flags require --build-dir to be '
475 'set.')
476 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800477 parser.error('--staging-flags requires --strict to be set.')
478 if options.sloppy and options.strict:
479 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700480
481 if options.mount or options.mount_dir:
482 if not options.target_dir:
483 options.target_dir = _CHROME_DIR_MOUNT
484 else:
485 if not options.target_dir:
486 options.target_dir = _CHROME_DIR
487
488 if options.mount and not options.mount_dir:
489 options.mount_dir = _CHROME_DIR
490
Ryan Cui3045c5d2012-07-13 18:00:33 -0700491 return options, args
492
493
Ryan Cuie535b172012-10-19 18:25:03 -0700494def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800495 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700496
497 Args:
Steve Funge984a532013-11-25 17:09:25 -0800498 options: The options object returned by optparse.
499 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700500 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700501 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
502 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
503
Ryan Cuib623e7b2013-03-14 12:54:11 -0700504 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700505 gyp_env = os.getenv('GYP_DEFINES', None)
506 if gyp_env is not None:
507 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700508 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700509 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700510
511 if options.strict and not options.gyp_defines:
512 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800513 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700514
Ryan Cui3c183c22013-04-29 18:04:11 -0700515 if options.build_dir:
516 chrome_path = os.path.join(options.build_dir, 'chrome')
517 if os.path.isfile(chrome_path):
518 deps = lddtree.ParseELF(chrome_path)
519 if 'libbase.so' in deps['libs']:
520 cros_build_lib.Warning(
521 'Detected a component build of Chrome. component build is '
522 'not working properly for Chrome OS. See crbug.com/196317. '
523 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700524
Ryan Cuia56a71e2012-10-18 18:40:35 -0700525
Ryan Cui504db722013-01-22 11:48:01 -0800526def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700527 """Get the chrome prebuilt tarball from GS.
528
Mike Frysinger02e1e072013-11-10 22:11:34 -0500529 Returns:
530 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700531 """
David James9374aac2013-10-08 16:00:17 -0700532 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500533 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800534 files = [found for found in files if
535 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
536 if not files:
537 raise Exception('No chrome package found at %s' % gs_path)
538 elif len(files) > 1:
539 # - Users should provide us with a direct link to either a stripped or
540 # unstripped chrome package.
541 # - In the case of being provided with an archive directory, where both
542 # stripped and unstripped chrome available, use the stripped chrome
543 # package.
544 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
545 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
546 files = [f for f in files if not 'unstripped' in f]
547 assert len(files) == 1
548 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800549
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800550 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800551 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800552 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
553 chrome_path = os.path.join(tempdir, filename)
554 assert os.path.exists(chrome_path)
555 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700556
557
Ryan Cuif890a3e2013-03-07 18:57:06 -0800558@contextlib.contextmanager
559def _StripBinContext(options):
560 if not options.dostrip:
561 yield None
562 elif options.strip_bin:
563 yield options.strip_bin
564 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800565 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800566 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700567 with sdk.Prepare(components=components, target_tc=options.target_tc,
568 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800569 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
570 constants.CHROME_ENV_FILE)
571 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
572 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
573 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800574 yield strip_bin
575
576
Steve Funge984a532013-11-25 17:09:25 -0800577def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
578 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800579 """Place the necessary files in the staging directory.
580
581 The staging directory is the directory used to rsync the build artifacts over
582 to the device. Only the necessary Chrome build artifacts are put into the
583 staging directory.
584 """
Ryan Cui5866be02013-03-18 14:12:00 -0700585 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400586 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800587 if options.build_dir:
588 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700589 strip_flags = (None if options.strip_flags is None else
590 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800591 chrome_util.StageChromeFromBuildDir(
592 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800593 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700594 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800595 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700596 else:
597 pkg_path = options.local_pkg_path
598 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800599 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
600 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700601
602 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800603 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700604 # Extract only the ./opt/google/chrome contents, directly into the staging
605 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800606 if pkg_path[-4:] == '.zip':
607 cros_build_lib.DebugRunCommand(
608 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
609 staging_dir])
610 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
611 shutil.move(filename, staging_dir)
612 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
613 else:
614 cros_build_lib.DebugRunCommand(
615 ['tar', '--strip-components', '4', '--extract',
616 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
617 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700618
Ryan Cui71aa8de2013-04-19 16:12:55 -0700619
Ryan Cui3045c5d2012-07-13 18:00:33 -0700620def main(argv):
621 options, args = _ParseCommandLine(argv)
622 _PostParseCheck(options, args)
623
624 # Set cros_build_lib debug level to hide RunCommand spew.
625 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700626 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700627 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800628 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700629
Ryan Cui71aa8de2013-04-19 16:12:55 -0700630 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700631 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
632 if cmd_stats:
633 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700634
Ryan Cui71aa8de2013-04-19 16:12:55 -0700635 with osutils.TempDir(set_global=True) as tempdir:
636 staging_dir = options.staging_dir
637 if not staging_dir:
638 staging_dir = os.path.join(tempdir, 'chrome')
639
640 deploy = DeployChrome(options, tempdir, staging_dir)
641 try:
642 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700643 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700644 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500645 deploy.Cleanup()