blob: e9b3d32a349a2867c19ed3d44156b7a4558fcedc [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
32from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080042from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070043from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070054
Steve Funge984a532013-11-25 17:09:25 -080055_ANDROID_DIR = '/system/chrome'
56_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
57
David James2cb34002013-03-01 18:42:40 -080058_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070059_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080060
Pawel Osciakc1bb2742014-12-29 16:32:33 +090061_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
62 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070063_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
64_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070065
Steve Funge984a532013-11-25 17:09:25 -080066DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080067
Mike Frysingere65f3752014-12-08 00:46:39 -050068
Ryan Cui3045c5d2012-07-13 18:00:33 -070069def _UrlBaseName(url):
70 """Return the last component of the URL."""
71 return url.rstrip('/').rpartition('/')[-1]
72
73
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070074class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080075 """Raised whenever the deploy fails."""
76
77
Ryan Cui7193a7e2013-04-26 14:15:19 -070078DeviceInfo = collections.namedtuple(
79 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
80
81
Ryan Cui3045c5d2012-07-13 18:00:33 -070082class DeployChrome(object):
83 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050084
Ryan Cuia56a71e2012-10-18 18:40:35 -070085 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070086 """Initialize the class.
87
Mike Frysinger02e1e072013-11-10 22:11:34 -050088 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040089 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070090 tempdir: Scratch space for the class. Caller has responsibility to clean
91 it up.
Steve Funge984a532013-11-25 17:09:25 -080092 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070093 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070094 self.tempdir = tempdir
95 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070096 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050097 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090098 self.device = remote.RemoteDevice(options.to, port=options.port,
99 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500100 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000101
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500102 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800103 self.chrome_dir = _CHROME_DIR
104
Ryan Cui7193a7e2013-04-26 14:15:19 -0700105 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500106 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800108 value = line.split()[3]
109 multipliers = {
110 'G': 1024 * 1024 * 1024,
111 'M': 1024 * 1024,
112 'K': 1024,
113 }
114 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700115
116 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500117 result = self.device.RunCommand('du -ks %s' % remote_dir,
118 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119 return int(result.output.split()[0])
120
121 def _GetStagingDirSize(self):
122 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800123 redirect_stdout=True,
124 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700125 return int(result.output.split()[0])
126
Ryan Cui3045c5d2012-07-13 18:00:33 -0700127 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500128 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
129 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 return result.returncode == 0
131
Justin TerAvestfac210e2017-04-13 11:39:00 -0600132 def _Reboot(self):
133 # A reboot in developer mode takes a while (and has delays), so the user
134 # will have time to read and act on the USB boot instructions below.
135 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
136 self.device.Reboot()
137
Ryan Cui3045c5d2012-07-13 18:00:33 -0700138 def _DisableRootfsVerification(self):
139 if not self.options.force:
140 logging.error('Detected that the device has rootfs verification enabled.')
141 logging.info('This script can automatically remove the rootfs '
142 'verification, which requires that it reboot the device.')
143 logging.info('Make sure the device is in developer mode!')
144 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700145 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800146 # Since we stopped Chrome earlier, it's good form to start it up again.
147 if self.options.startui:
148 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500149 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800150 raise DeployFailure('Need rootfs verification to be disabled. '
151 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700152
153 logging.info('Removing rootfs verification from %s', self.options.to)
154 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
155 # Use --force to bypass the checks.
156 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
157 '--remove_rootfs_verification --force')
158 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500159 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160
Justin TerAvestfac210e2017-04-13 11:39:00 -0600161 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700191 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700203 'running chrome processes and try again.'
204 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400221 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Satoru Takabayashif2893002017-06-20 14:52:48 +0900248 def _ShouldUseCompression(self):
249 """Checks if compression should be used for rsync."""
250 if self.options.compress == 'always':
251 return True
252 elif self.options.compress == 'never':
253 return False
254 elif self.options.compress == 'auto':
255 return not self.device.HasGigabitEthernet()
256
Ryan Cui3045c5d2012-07-13 18:00:33 -0700257 def _Deploy(self):
Justin TerAvestfac210e2017-04-13 11:39:00 -0600258 old_dbus_checksums = self._GetDBusChecksums()
259
Pawel Osciak577773a2013-03-05 10:52:12 -0800260 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700261 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800262 dest_path = _CHROME_DIR
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700263 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
264 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900265 if not self.device.HasRsync():
266 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700267 'rsync is not found on the device.\n'
268 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500269 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
270 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700271 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900272 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700273 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500274 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800275
276 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800277 if p.mode:
278 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500279 self.device.RunCommand('chmod %o %s/%s' % (
280 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800281
Justin TerAvestfac210e2017-04-13 11:39:00 -0600282 new_dbus_checksums = self._GetDBusChecksums()
283 if old_dbus_checksums != new_dbus_checksums:
Justin TerAveste8c827b2017-05-19 12:58:35 -0600284 if self.options.target_dir == _CHROME_DIR:
285 logging.info('Detected change to D-Bus service files, rebooting.')
286 self._Reboot()
287 return
288 else:
289 logging.warn('Detected change in D-Bus service files, but target dir '
290 'is not %s. D-Bus changes will not be picked up by '
291 'dbus-daemon at boot time.', _CHROME_DIR)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600292
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800293 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800294 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500295 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700296
David James88e6f032013-03-02 08:13:20 -0800297 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700298 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800299 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500300 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800301 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700302 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800303 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700304
Steve Funge984a532013-11-25 17:09:25 -0800305 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700306 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700307 def BinaryExists(filename):
308 """Checks if the passed-in file is present in the build directory."""
309 return os.path.exists(os.path.join(self.options.build_dir, filename))
310
Daniel Erat9813f0e2014-11-12 11:00:28 -0700311 # Handle non-Chrome deployments.
312 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700313 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700314 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
315
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700316 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700317 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700318 self.options.startui = False
319
David James88e6f032013-03-02 08:13:20 -0800320 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800321 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
322 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800323
Thiago Goncales12793312013-05-23 11:26:17 -0700324 def _MountTarget(self):
325 logging.info('Mounting Chrome...')
326
327 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500328 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500329 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900330 # Umount the existing mount on mount_dir if present first
331 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
332 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500333 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
334 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700335 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500336 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
337
Justin TerAvestfac210e2017-04-13 11:39:00 -0600338 def _GetDBusChecksums(self):
339 """Returns Checksums for D-Bus files deployed with Chrome.
340
341 This is used to determine if a reboot is required after deploying Chrome.
342 """
Justin TerAveste8c827b2017-05-19 12:58:35 -0600343 path = os.path.join(_CHROME_DIR, 'dbus/*')
344 result = self.device.RunCommand('md5sum ' + path,
Justin TerAvestfac210e2017-04-13 11:39:00 -0600345 error_code_ok=True)
346 return result.output
347
Robert Flack1dc7ea82014-11-26 13:50:24 -0500348 def Cleanup(self):
349 """Clean up RemoteDevice."""
350 if not self.options.staging_only:
351 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700352
David James88e6f032013-03-02 08:13:20 -0800353 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800354 self._CheckDeployType()
355
David James88e6f032013-03-02 08:13:20 -0800356 # If requested, just do the staging step.
357 if self.options.staging_only:
358 self._PrepareStagingDir()
359 return 0
360
361 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800362 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800363 steps = [self._GetDeviceInfo, self._CheckConnection,
364 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
365 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700366 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
367 return_values=True)
368 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800369
Robert Flack1dc7ea82014-11-26 13:50:24 -0500370 # If we're trying to deploy to a dir which is not writable and we failed
371 # to mark the rootfs as writable, try disabling rootfs verification.
372 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800373 self._DisableRootfsVerification()
374
Thiago Goncales12793312013-05-23 11:26:17 -0700375 if self.options.mount_dir is not None:
376 self._MountTarget()
377
David James88e6f032013-03-02 08:13:20 -0800378 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700379 self._Deploy()
380
381
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700382def ValidateStagingFlags(value):
383 """Convert formatted string to dictionary."""
384 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700385
386
Steven Bennetts368c3e52016-09-23 13:05:21 -0700387def ValidateGnArgs(value):
388 """Convert GN_ARGS-formatted string to dictionary."""
389 return gn_helpers.FromGNArgs(value)
390
391
Ryan Cuie535b172012-10-19 18:25:03 -0700392def _CreateParser():
393 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400394 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700395
Ryan Cuia56a71e2012-10-18 18:40:35 -0700396 # TODO(rcui): Have this use the UI-V2 format of having source and target
397 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400398 parser.add_argument('--force', action='store_true', default=False,
399 help='Skip all prompts (i.e., for disabling of rootfs '
400 'verification). This may result in the target '
401 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800402 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400403 parser.add_argument('--board', default=sdk_board_env,
404 help="The board the Chrome build is targeted for. When "
405 "in a 'cros chrome-sdk' shell, defaults to the SDK "
406 "board.")
407 parser.add_argument('--build-dir', type='path',
408 help='The directory with Chrome build artifacts to '
409 'deploy from. Typically of format '
410 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700411 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400412 parser.add_argument('--target-dir', type='path',
413 default=None,
414 help='Target directory on device to deploy Chrome into.')
415 parser.add_argument('-g', '--gs-path', type='gs_path',
416 help='GS path that contains the chrome to deploy.')
417 parser.add_argument('--nostartui', action='store_false', dest='startui',
418 default=True,
419 help="Don't restart the ui daemon after deployment.")
420 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
421 default=True,
422 help="Don't strip binaries during deployment. Warning: "
423 'the resulting binaries will be very large!')
424 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
425 help='Port of the target device to connect to.')
426 parser.add_argument('-t', '--to',
427 help='The IP address of the CrOS device to deploy to.')
428 parser.add_argument('-v', '--verbose', action='store_true', default=False,
429 help='Show more debug output.')
430 parser.add_argument('--mount-dir', type='path', default=None,
431 help='Deploy Chrome in target directory and bind it '
432 'to the directory specified by this flag.'
433 'Any existing mount on this directory will be '
434 'umounted first.')
435 parser.add_argument('--mount', action='store_true', default=False,
436 help='Deploy Chrome to default target directory and bind '
437 'it to the default mount directory.'
438 'Any existing mount on this directory will be '
439 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700440
Mike Frysingerc3061a62015-06-04 04:16:18 -0400441 group = parser.add_argument_group('Advanced Options')
442 group.add_argument('-l', '--local-pkg-path', type='path',
443 help='Path to local chrome prebuilt package to deploy.')
444 group.add_argument('--sloppy', action='store_true', default=False,
445 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700446 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400447 help=('Extra flags to control staging. Valid flags are - '
448 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700449 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400450 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700451 help='Deprecated. Default behavior is "strict". Use '
452 '--sloppy to omit warnings for missing optional '
453 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400454 group.add_argument('--strip-flags', default=None,
455 help="Flags to call the 'strip' binutil tool with. "
456 "Overrides the default arguments.")
457 group.add_argument('--ping', action='store_true', default=False,
458 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700459 group.add_argument('--process-timeout', type=int,
460 default=KILL_PROC_MAX_WAIT,
461 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700462
Mike Frysingerc3061a62015-06-04 04:16:18 -0400463 group = parser.add_argument_group(
464 'Metadata Overrides (Advanced)',
465 description='Provide all of these overrides in order to remove '
466 'dependencies on metadata.json existence.')
467 group.add_argument('--target-tc', action='store', default=None,
468 help='Override target toolchain name, e.g. '
469 'x86_64-cros-linux-gnu')
470 group.add_argument('--toolchain-url', action='store', default=None,
471 help='Override toolchain url format pattern, e.g. '
472 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700473
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700474 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
475 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
476 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400477 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700478
479 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
480 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
481 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
482 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
483 help=argparse.SUPPRESS)
484
Ryan Cuia56a71e2012-10-18 18:40:35 -0700485 # Path of an empty directory to stage chrome artifacts to. Defaults to a
486 # temporary directory that is removed when the script finishes. If the path
487 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400488 parser.add_argument('--staging-dir', type='path', default=None,
489 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700490 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400491 parser.add_argument('--staging-only', action='store_true', default=False,
492 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700493 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
494 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
495 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400496 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900497 parser.add_argument('--compress', action='store', default='auto',
498 choices=('always', 'never', 'auto'),
499 help='Choose the data compression behavior. Default '
500 'is set to "auto", that disables compression if '
501 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700502 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700503
Ryan Cuie535b172012-10-19 18:25:03 -0700504
505def _ParseCommandLine(argv):
506 """Parse args, and run environment-independent checks."""
507 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400508 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700509
Ryan Cuia56a71e2012-10-18 18:40:35 -0700510 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
511 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800512 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700513 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
514 parser.error('Cannot specify both --build_dir and '
515 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800516 if options.build_dir and not options.board:
517 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700518 if options.gs_path and options.local_pkg_path:
519 parser.error('Cannot specify both --gs-path and --local-pkg-path')
520 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700521 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700522 if options.staging_flags and not options.build_dir:
523 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700524
Steven Bennetts46a84c32016-08-12 15:20:46 -0700525 if options.strict:
526 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700527 if options.gyp_defines:
528 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700529
530 if options.mount or options.mount_dir:
531 if not options.target_dir:
532 options.target_dir = _CHROME_DIR_MOUNT
533 else:
534 if not options.target_dir:
535 options.target_dir = _CHROME_DIR
536
537 if options.mount and not options.mount_dir:
538 options.mount_dir = _CHROME_DIR
539
Mike Frysingerc3061a62015-06-04 04:16:18 -0400540 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700541
542
Mike Frysingerc3061a62015-06-04 04:16:18 -0400543def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800544 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700545
546 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400547 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700548 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
550 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
551
Steven Bennetts368c3e52016-09-23 13:05:21 -0700552 if not options.gn_args:
553 gn_env = os.getenv('GN_ARGS')
554 if gn_env is not None:
555 options.gn_args = gn_helpers.FromGNArgs(gn_env)
556 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700557
Steven Bennetts60600462016-05-12 10:40:20 -0700558 if not options.staging_flags:
559 use_env = os.getenv('USE')
560 if use_env is not None:
561 options.staging_flags = ' '.join(set(use_env.split()).intersection(
562 chrome_util.STAGING_FLAGS))
563 logging.info('Staging flags taken from USE in environment: %s',
564 options.staging_flags)
565
Ryan Cuia56a71e2012-10-18 18:40:35 -0700566
Ryan Cui504db722013-01-22 11:48:01 -0800567def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700568 """Get the chrome prebuilt tarball from GS.
569
Mike Frysinger02e1e072013-11-10 22:11:34 -0500570 Returns:
571 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700572 """
David James9374aac2013-10-08 16:00:17 -0700573 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500574 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800575 files = [found for found in files if
576 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
577 if not files:
578 raise Exception('No chrome package found at %s' % gs_path)
579 elif len(files) > 1:
580 # - Users should provide us with a direct link to either a stripped or
581 # unstripped chrome package.
582 # - In the case of being provided with an archive directory, where both
583 # stripped and unstripped chrome available, use the stripped chrome
584 # package.
585 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
586 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
587 files = [f for f in files if not 'unstripped' in f]
588 assert len(files) == 1
589 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800590
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800591 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800592 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800593 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
594 chrome_path = os.path.join(tempdir, filename)
595 assert os.path.exists(chrome_path)
596 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700597
598
Ryan Cuif890a3e2013-03-07 18:57:06 -0800599@contextlib.contextmanager
600def _StripBinContext(options):
601 if not options.dostrip:
602 yield None
603 elif options.strip_bin:
604 yield options.strip_bin
605 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800606 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800607 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700608 with sdk.Prepare(components=components, target_tc=options.target_tc,
609 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800610 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
611 constants.CHROME_ENV_FILE)
612 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
613 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
614 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800615 yield strip_bin
616
617
Steve Funge984a532013-11-25 17:09:25 -0800618def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
619 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800620 """Place the necessary files in the staging directory.
621
622 The staging directory is the directory used to rsync the build artifacts over
623 to the device. Only the necessary Chrome build artifacts are put into the
624 staging directory.
625 """
Ryan Cui5866be02013-03-18 14:12:00 -0700626 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400627 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800628 if options.build_dir:
629 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700630 strip_flags = (None if options.strip_flags is None else
631 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800632 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700633 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700634 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700635 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800636 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700637 else:
638 pkg_path = options.local_pkg_path
639 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800640 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
641 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700642
643 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800644 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700645 # Extract only the ./opt/google/chrome contents, directly into the staging
646 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800647 if pkg_path[-4:] == '.zip':
648 cros_build_lib.DebugRunCommand(
649 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
650 staging_dir])
651 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
652 shutil.move(filename, staging_dir)
653 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
654 else:
655 cros_build_lib.DebugRunCommand(
656 ['tar', '--strip-components', '4', '--extract',
657 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
658 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700659
Ryan Cui71aa8de2013-04-19 16:12:55 -0700660
Ryan Cui3045c5d2012-07-13 18:00:33 -0700661def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400662 options = _ParseCommandLine(argv)
663 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700664
665 # Set cros_build_lib debug level to hide RunCommand spew.
666 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700667 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700668 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800669 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700670
Aviv Keshet01a82e92017-03-02 17:39:59 -0800671 with osutils.TempDir(set_global=True) as tempdir:
672 staging_dir = options.staging_dir
673 if not staging_dir:
674 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700675
Aviv Keshet01a82e92017-03-02 17:39:59 -0800676 deploy = DeployChrome(options, tempdir, staging_dir)
677 try:
678 deploy.Perform()
679 except failures_lib.StepFailure as ex:
680 raise SystemExit(str(ex).strip())
681 deploy.Cleanup()