blob: 832eee5e1099734d390c886b741a8ca0e83e9d36 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
32from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080042from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070043from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070054
Steve Funge984a532013-11-25 17:09:25 -080055_ANDROID_DIR = '/system/chrome'
56_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
57
David James2cb34002013-03-01 18:42:40 -080058_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070059_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080060
Pawel Osciakc1bb2742014-12-29 16:32:33 +090061_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
62 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070063_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
64_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070065
Steve Funge984a532013-11-25 17:09:25 -080066DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080067
Mike Frysingere65f3752014-12-08 00:46:39 -050068
Ryan Cui3045c5d2012-07-13 18:00:33 -070069def _UrlBaseName(url):
70 """Return the last component of the URL."""
71 return url.rstrip('/').rpartition('/')[-1]
72
73
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070074class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080075 """Raised whenever the deploy fails."""
76
77
Ryan Cui7193a7e2013-04-26 14:15:19 -070078DeviceInfo = collections.namedtuple(
79 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
80
81
Ryan Cui3045c5d2012-07-13 18:00:33 -070082class DeployChrome(object):
83 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050084
Ryan Cuia56a71e2012-10-18 18:40:35 -070085 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070086 """Initialize the class.
87
Mike Frysinger02e1e072013-11-10 22:11:34 -050088 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040089 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070090 tempdir: Scratch space for the class. Caller has responsibility to clean
91 it up.
Steve Funge984a532013-11-25 17:09:25 -080092 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070093 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070094 self.tempdir = tempdir
95 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070096 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050097 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090098 self.device = remote.RemoteDevice(options.to, port=options.port,
99 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500100 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000101
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500102 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800103 self.chrome_dir = _CHROME_DIR
104
Ryan Cui7193a7e2013-04-26 14:15:19 -0700105 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500106 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800108 value = line.split()[3]
109 multipliers = {
110 'G': 1024 * 1024 * 1024,
111 'M': 1024 * 1024,
112 'K': 1024,
113 }
114 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700115
116 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500117 result = self.device.RunCommand('du -ks %s' % remote_dir,
118 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119 return int(result.output.split()[0])
120
121 def _GetStagingDirSize(self):
122 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800123 redirect_stdout=True,
124 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700125 return int(result.output.split()[0])
126
Ryan Cui3045c5d2012-07-13 18:00:33 -0700127 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500128 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
129 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 return result.returncode == 0
131
Justin TerAvestfac210e2017-04-13 11:39:00 -0600132 def _Reboot(self):
133 # A reboot in developer mode takes a while (and has delays), so the user
134 # will have time to read and act on the USB boot instructions below.
135 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
136 self.device.Reboot()
137
Ryan Cui3045c5d2012-07-13 18:00:33 -0700138 def _DisableRootfsVerification(self):
139 if not self.options.force:
140 logging.error('Detected that the device has rootfs verification enabled.')
141 logging.info('This script can automatically remove the rootfs '
142 'verification, which requires that it reboot the device.')
143 logging.info('Make sure the device is in developer mode!')
144 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700145 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800146 # Since we stopped Chrome earlier, it's good form to start it up again.
147 if self.options.startui:
148 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500149 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800150 raise DeployFailure('Need rootfs verification to be disabled. '
151 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700152
153 logging.info('Removing rootfs verification from %s', self.options.to)
154 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
155 # Use --force to bypass the checks.
156 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
157 '--remove_rootfs_verification --force')
158 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500159 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160
Justin TerAvestfac210e2017-04-13 11:39:00 -0600161 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700191 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700203 'running chrome processes and try again.'
204 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400221 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Satoru Takabayashif2893002017-06-20 14:52:48 +0900248 def _ShouldUseCompression(self):
249 """Checks if compression should be used for rsync."""
250 if self.options.compress == 'always':
251 return True
252 elif self.options.compress == 'never':
253 return False
254 elif self.options.compress == 'auto':
255 return not self.device.HasGigabitEthernet()
256
Ryan Cui3045c5d2012-07-13 18:00:33 -0700257 def _Deploy(self):
Justin TerAvestfac210e2017-04-13 11:39:00 -0600258 old_dbus_checksums = self._GetDBusChecksums()
259
Pawel Osciak577773a2013-03-05 10:52:12 -0800260 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700261 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
262 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900263 if not self.device.HasRsync():
264 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700265 'rsync is not found on the device.\n'
266 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500267 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
268 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700269 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900270 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700271 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500272 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800273
274 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800275 if p.mode:
276 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500277 self.device.RunCommand('chmod %o %s/%s' % (
Satoru Takabayashi4b36f1c2017-06-20 15:39:40 +0900278 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800279
Justin TerAvestfac210e2017-04-13 11:39:00 -0600280 new_dbus_checksums = self._GetDBusChecksums()
281 if old_dbus_checksums != new_dbus_checksums:
Justin TerAveste8c827b2017-05-19 12:58:35 -0600282 if self.options.target_dir == _CHROME_DIR:
283 logging.info('Detected change to D-Bus service files, rebooting.')
284 self._Reboot()
285 return
286 else:
287 logging.warn('Detected change in D-Bus service files, but target dir '
288 'is not %s. D-Bus changes will not be picked up by '
289 'dbus-daemon at boot time.', _CHROME_DIR)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600290
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800291 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800292 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500293 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700294
David James88e6f032013-03-02 08:13:20 -0800295 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700296 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800297 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500298 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800299 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700300 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800301 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700302
Steve Funge984a532013-11-25 17:09:25 -0800303 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700304 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700305 def BinaryExists(filename):
306 """Checks if the passed-in file is present in the build directory."""
307 return os.path.exists(os.path.join(self.options.build_dir, filename))
308
Daniel Erat9813f0e2014-11-12 11:00:28 -0700309 # Handle non-Chrome deployments.
310 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700311 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700312 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
313
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700314 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700315 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700316 self.options.startui = False
317
David James88e6f032013-03-02 08:13:20 -0800318 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800319 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
320 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800321
Thiago Goncales12793312013-05-23 11:26:17 -0700322 def _MountTarget(self):
323 logging.info('Mounting Chrome...')
324
325 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500326 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500327 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900328 # Umount the existing mount on mount_dir if present first
329 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
330 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500331 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
332 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700333 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500334 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
335
Justin TerAvestfac210e2017-04-13 11:39:00 -0600336 def _GetDBusChecksums(self):
337 """Returns Checksums for D-Bus files deployed with Chrome.
338
339 This is used to determine if a reboot is required after deploying Chrome.
340 """
Justin TerAveste8c827b2017-05-19 12:58:35 -0600341 path = os.path.join(_CHROME_DIR, 'dbus/*')
342 result = self.device.RunCommand('md5sum ' + path,
Justin TerAvestfac210e2017-04-13 11:39:00 -0600343 error_code_ok=True)
344 return result.output
345
Robert Flack1dc7ea82014-11-26 13:50:24 -0500346 def Cleanup(self):
347 """Clean up RemoteDevice."""
348 if not self.options.staging_only:
349 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700350
David James88e6f032013-03-02 08:13:20 -0800351 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800352 self._CheckDeployType()
353
David James88e6f032013-03-02 08:13:20 -0800354 # If requested, just do the staging step.
355 if self.options.staging_only:
356 self._PrepareStagingDir()
357 return 0
358
359 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800360 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800361 steps = [self._GetDeviceInfo, self._CheckConnection,
362 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
363 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700364 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
365 return_values=True)
366 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800367
Robert Flack1dc7ea82014-11-26 13:50:24 -0500368 # If we're trying to deploy to a dir which is not writable and we failed
369 # to mark the rootfs as writable, try disabling rootfs verification.
370 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800371 self._DisableRootfsVerification()
372
Thiago Goncales12793312013-05-23 11:26:17 -0700373 if self.options.mount_dir is not None:
374 self._MountTarget()
375
David James88e6f032013-03-02 08:13:20 -0800376 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700377 self._Deploy()
378
379
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700380def ValidateStagingFlags(value):
381 """Convert formatted string to dictionary."""
382 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700383
384
Steven Bennetts368c3e52016-09-23 13:05:21 -0700385def ValidateGnArgs(value):
386 """Convert GN_ARGS-formatted string to dictionary."""
387 return gn_helpers.FromGNArgs(value)
388
389
Ryan Cuie535b172012-10-19 18:25:03 -0700390def _CreateParser():
391 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400392 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700393
Ryan Cuia56a71e2012-10-18 18:40:35 -0700394 # TODO(rcui): Have this use the UI-V2 format of having source and target
395 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400396 parser.add_argument('--force', action='store_true', default=False,
397 help='Skip all prompts (i.e., for disabling of rootfs '
398 'verification). This may result in the target '
399 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800400 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400401 parser.add_argument('--board', default=sdk_board_env,
402 help="The board the Chrome build is targeted for. When "
403 "in a 'cros chrome-sdk' shell, defaults to the SDK "
404 "board.")
405 parser.add_argument('--build-dir', type='path',
406 help='The directory with Chrome build artifacts to '
407 'deploy from. Typically of format '
408 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700409 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400410 parser.add_argument('--target-dir', type='path',
411 default=None,
412 help='Target directory on device to deploy Chrome into.')
413 parser.add_argument('-g', '--gs-path', type='gs_path',
414 help='GS path that contains the chrome to deploy.')
415 parser.add_argument('--nostartui', action='store_false', dest='startui',
416 default=True,
417 help="Don't restart the ui daemon after deployment.")
418 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
419 default=True,
420 help="Don't strip binaries during deployment. Warning: "
421 'the resulting binaries will be very large!')
422 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
423 help='Port of the target device to connect to.')
424 parser.add_argument('-t', '--to',
425 help='The IP address of the CrOS device to deploy to.')
426 parser.add_argument('-v', '--verbose', action='store_true', default=False,
427 help='Show more debug output.')
428 parser.add_argument('--mount-dir', type='path', default=None,
429 help='Deploy Chrome in target directory and bind it '
430 'to the directory specified by this flag.'
431 'Any existing mount on this directory will be '
432 'umounted first.')
433 parser.add_argument('--mount', action='store_true', default=False,
434 help='Deploy Chrome to default target directory and bind '
435 'it to the default mount directory.'
436 'Any existing mount on this directory will be '
437 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700438
Mike Frysingerc3061a62015-06-04 04:16:18 -0400439 group = parser.add_argument_group('Advanced Options')
440 group.add_argument('-l', '--local-pkg-path', type='path',
441 help='Path to local chrome prebuilt package to deploy.')
442 group.add_argument('--sloppy', action='store_true', default=False,
443 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700444 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400445 help=('Extra flags to control staging. Valid flags are - '
446 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700447 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400448 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700449 help='Deprecated. Default behavior is "strict". Use '
450 '--sloppy to omit warnings for missing optional '
451 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400452 group.add_argument('--strip-flags', default=None,
453 help="Flags to call the 'strip' binutil tool with. "
454 "Overrides the default arguments.")
455 group.add_argument('--ping', action='store_true', default=False,
456 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700457 group.add_argument('--process-timeout', type=int,
458 default=KILL_PROC_MAX_WAIT,
459 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700460
Mike Frysingerc3061a62015-06-04 04:16:18 -0400461 group = parser.add_argument_group(
462 'Metadata Overrides (Advanced)',
463 description='Provide all of these overrides in order to remove '
464 'dependencies on metadata.json existence.')
465 group.add_argument('--target-tc', action='store', default=None,
466 help='Override target toolchain name, e.g. '
467 'x86_64-cros-linux-gnu')
468 group.add_argument('--toolchain-url', action='store', default=None,
469 help='Override toolchain url format pattern, e.g. '
470 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700471
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700472 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
473 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
474 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400475 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700476
477 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
478 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
479 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
480 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
481 help=argparse.SUPPRESS)
482
Ryan Cuia56a71e2012-10-18 18:40:35 -0700483 # Path of an empty directory to stage chrome artifacts to. Defaults to a
484 # temporary directory that is removed when the script finishes. If the path
485 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400486 parser.add_argument('--staging-dir', type='path', default=None,
487 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700488 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400489 parser.add_argument('--staging-only', action='store_true', default=False,
490 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700491 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
492 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
493 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400494 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900495 parser.add_argument('--compress', action='store', default='auto',
496 choices=('always', 'never', 'auto'),
497 help='Choose the data compression behavior. Default '
498 'is set to "auto", that disables compression if '
499 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700500 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700501
Ryan Cuie535b172012-10-19 18:25:03 -0700502
503def _ParseCommandLine(argv):
504 """Parse args, and run environment-independent checks."""
505 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400506 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700507
Ryan Cuia56a71e2012-10-18 18:40:35 -0700508 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
509 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800510 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700511 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
512 parser.error('Cannot specify both --build_dir and '
513 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800514 if options.build_dir and not options.board:
515 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700516 if options.gs_path and options.local_pkg_path:
517 parser.error('Cannot specify both --gs-path and --local-pkg-path')
518 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700519 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700520 if options.staging_flags and not options.build_dir:
521 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700522
Steven Bennetts46a84c32016-08-12 15:20:46 -0700523 if options.strict:
524 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700525 if options.gyp_defines:
526 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700527
528 if options.mount or options.mount_dir:
529 if not options.target_dir:
530 options.target_dir = _CHROME_DIR_MOUNT
531 else:
532 if not options.target_dir:
533 options.target_dir = _CHROME_DIR
534
535 if options.mount and not options.mount_dir:
536 options.mount_dir = _CHROME_DIR
537
Mike Frysingerc3061a62015-06-04 04:16:18 -0400538 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700539
540
Mike Frysingerc3061a62015-06-04 04:16:18 -0400541def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800542 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700543
544 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400545 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700546 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700547 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
548 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
549
Steven Bennetts368c3e52016-09-23 13:05:21 -0700550 if not options.gn_args:
551 gn_env = os.getenv('GN_ARGS')
552 if gn_env is not None:
553 options.gn_args = gn_helpers.FromGNArgs(gn_env)
554 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700555
Steven Bennetts60600462016-05-12 10:40:20 -0700556 if not options.staging_flags:
557 use_env = os.getenv('USE')
558 if use_env is not None:
559 options.staging_flags = ' '.join(set(use_env.split()).intersection(
560 chrome_util.STAGING_FLAGS))
561 logging.info('Staging flags taken from USE in environment: %s',
562 options.staging_flags)
563
Ryan Cuia56a71e2012-10-18 18:40:35 -0700564
Ryan Cui504db722013-01-22 11:48:01 -0800565def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700566 """Get the chrome prebuilt tarball from GS.
567
Mike Frysinger02e1e072013-11-10 22:11:34 -0500568 Returns:
569 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700570 """
David James9374aac2013-10-08 16:00:17 -0700571 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500572 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800573 files = [found for found in files if
574 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
575 if not files:
576 raise Exception('No chrome package found at %s' % gs_path)
577 elif len(files) > 1:
578 # - Users should provide us with a direct link to either a stripped or
579 # unstripped chrome package.
580 # - In the case of being provided with an archive directory, where both
581 # stripped and unstripped chrome available, use the stripped chrome
582 # package.
583 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
584 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
585 files = [f for f in files if not 'unstripped' in f]
586 assert len(files) == 1
587 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800588
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800589 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800590 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800591 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
592 chrome_path = os.path.join(tempdir, filename)
593 assert os.path.exists(chrome_path)
594 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700595
596
Ryan Cuif890a3e2013-03-07 18:57:06 -0800597@contextlib.contextmanager
598def _StripBinContext(options):
599 if not options.dostrip:
600 yield None
601 elif options.strip_bin:
602 yield options.strip_bin
603 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800604 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800605 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700606 with sdk.Prepare(components=components, target_tc=options.target_tc,
607 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800608 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
609 constants.CHROME_ENV_FILE)
610 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
611 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
612 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800613 yield strip_bin
614
615
Steve Funge984a532013-11-25 17:09:25 -0800616def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
617 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800618 """Place the necessary files in the staging directory.
619
620 The staging directory is the directory used to rsync the build artifacts over
621 to the device. Only the necessary Chrome build artifacts are put into the
622 staging directory.
623 """
Ryan Cui5866be02013-03-18 14:12:00 -0700624 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400625 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800626 if options.build_dir:
627 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700628 strip_flags = (None if options.strip_flags is None else
629 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800630 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700631 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700632 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700633 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800634 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700635 else:
636 pkg_path = options.local_pkg_path
637 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800638 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
639 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700640
641 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800642 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700643 # Extract only the ./opt/google/chrome contents, directly into the staging
644 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800645 if pkg_path[-4:] == '.zip':
646 cros_build_lib.DebugRunCommand(
647 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
648 staging_dir])
649 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
650 shutil.move(filename, staging_dir)
651 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
652 else:
653 cros_build_lib.DebugRunCommand(
654 ['tar', '--strip-components', '4', '--extract',
655 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
656 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700657
Ryan Cui71aa8de2013-04-19 16:12:55 -0700658
Ryan Cui3045c5d2012-07-13 18:00:33 -0700659def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400660 options = _ParseCommandLine(argv)
661 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700662
663 # Set cros_build_lib debug level to hide RunCommand spew.
664 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700665 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700666 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800667 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700668
Aviv Keshet01a82e92017-03-02 17:39:59 -0800669 with osutils.TempDir(set_global=True) as tempdir:
670 staging_dir = options.staging_dir
671 if not staging_dir:
672 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700673
Aviv Keshet01a82e92017-03-02 17:39:59 -0800674 deploy = DeployChrome(options, tempdir, staging_dir)
675 try:
676 deploy.Perform()
677 except failures_lib.StepFailure as ex:
678 raise SystemExit(str(ex).strip())
679 deploy.Cleanup()