blob: c4f4c3f6f077569e600bdd78b7b31a155c814cbe [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
32from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080042from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070043from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070054
Steve Funge984a532013-11-25 17:09:25 -080055_ANDROID_DIR = '/system/chrome'
56_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
57
David James2cb34002013-03-01 18:42:40 -080058_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070059_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080060
Pawel Osciakc1bb2742014-12-29 16:32:33 +090061_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
62 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070063_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
64_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070065
Steve Funge984a532013-11-25 17:09:25 -080066DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080067
Mike Frysingere65f3752014-12-08 00:46:39 -050068
Ryan Cui3045c5d2012-07-13 18:00:33 -070069def _UrlBaseName(url):
70 """Return the last component of the URL."""
71 return url.rstrip('/').rpartition('/')[-1]
72
73
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070074class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080075 """Raised whenever the deploy fails."""
76
77
Ryan Cui7193a7e2013-04-26 14:15:19 -070078DeviceInfo = collections.namedtuple(
79 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
80
81
Ryan Cui3045c5d2012-07-13 18:00:33 -070082class DeployChrome(object):
83 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050084
Ryan Cuia56a71e2012-10-18 18:40:35 -070085 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070086 """Initialize the class.
87
Mike Frysinger02e1e072013-11-10 22:11:34 -050088 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040089 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070090 tempdir: Scratch space for the class. Caller has responsibility to clean
91 it up.
Steve Funge984a532013-11-25 17:09:25 -080092 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070093 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070094 self.tempdir = tempdir
95 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070096 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050097 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090098 self.device = remote.RemoteDevice(options.to, port=options.port,
99 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500100 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000101
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500102 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800103 self.chrome_dir = _CHROME_DIR
104
Ryan Cui7193a7e2013-04-26 14:15:19 -0700105 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500106 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800108 value = line.split()[3]
109 multipliers = {
110 'G': 1024 * 1024 * 1024,
111 'M': 1024 * 1024,
112 'K': 1024,
113 }
114 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700115
116 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500117 result = self.device.RunCommand('du -ks %s' % remote_dir,
118 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119 return int(result.output.split()[0])
120
121 def _GetStagingDirSize(self):
122 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800123 redirect_stdout=True,
124 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700125 return int(result.output.split()[0])
126
Ryan Cui3045c5d2012-07-13 18:00:33 -0700127 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500128 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
129 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 return result.returncode == 0
131
Justin TerAvestfac210e2017-04-13 11:39:00 -0600132 def _Reboot(self):
133 # A reboot in developer mode takes a while (and has delays), so the user
134 # will have time to read and act on the USB boot instructions below.
135 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
136 self.device.Reboot()
137
Ryan Cui3045c5d2012-07-13 18:00:33 -0700138 def _DisableRootfsVerification(self):
139 if not self.options.force:
140 logging.error('Detected that the device has rootfs verification enabled.')
141 logging.info('This script can automatically remove the rootfs '
142 'verification, which requires that it reboot the device.')
143 logging.info('Make sure the device is in developer mode!')
144 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700145 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800146 # Since we stopped Chrome earlier, it's good form to start it up again.
147 if self.options.startui:
148 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500149 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800150 raise DeployFailure('Need rootfs verification to be disabled. '
151 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700152
153 logging.info('Removing rootfs verification from %s', self.options.to)
154 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
155 # Use --force to bypass the checks.
156 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
157 '--remove_rootfs_verification --force')
158 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500159 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160
Justin TerAvestfac210e2017-04-13 11:39:00 -0600161 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
David James3432acd2013-11-27 10:02:18 -0800191 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
203 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
204 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
David James88e6f032013-03-02 08:13:20 -0800206 def _MountRootfsAsWritable(self, error_code_ok=True):
207 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Robert Flack1dc7ea82014-11-26 13:50:24 -0500209 If the command fails, and error_code_ok is True, and the target dir is not
210 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700211
Mike Frysinger02e1e072013-11-10 22:11:34 -0500212 Args:
David James88e6f032013-03-02 08:13:20 -0800213 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
214 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800215 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500216 result = self.device.RunCommand(MOUNT_RW_COMMAND,
217 error_code_ok=error_code_ok,
218 capture_output=True)
219 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400220 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500221 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
Ryan Cui7193a7e2013-04-26 14:15:19 -0700223 def _GetDeviceInfo(self):
224 steps = [
225 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
226 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
227 ]
228 return_values = parallel.RunParallelSteps(steps, return_values=True)
229 return DeviceInfo(*return_values)
230
231 def _CheckDeviceFreeSpace(self, device_info):
232 """See if target device has enough space for Chrome.
233
Mike Frysinger02e1e072013-11-10 22:11:34 -0500234 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700235 device_info: A DeviceInfo named tuple.
236 """
237 effective_free = device_info.target_dir_size + device_info.target_fs_free
238 staging_size = self._GetStagingDirSize()
239 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700240 raise DeployFailure(
241 'Not enough free space on the device. Required: %s MiB, '
242 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700243 if device_info.target_fs_free < (100 * 1024):
244 logging.warning('The device has less than 100MB free. deploy_chrome may '
245 'hang during the transfer.')
246
Ryan Cui3045c5d2012-07-13 18:00:33 -0700247 def _Deploy(self):
Justin TerAvestfac210e2017-04-13 11:39:00 -0600248 old_dbus_checksums = self._GetDBusChecksums()
249
Pawel Osciak577773a2013-03-05 10:52:12 -0800250 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700251 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800252 dest_path = _CHROME_DIR
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700253 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
254 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900255 if not self.device.HasRsync():
256 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700257 'rsync is not found on the device.\n'
258 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500259 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
260 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700261 mode='rsync', inplace=True,
262 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500263 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800264
265 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800266 if p.mode:
267 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500268 self.device.RunCommand('chmod %o %s/%s' % (
269 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800270
Justin TerAvestfac210e2017-04-13 11:39:00 -0600271 new_dbus_checksums = self._GetDBusChecksums()
272 if old_dbus_checksums != new_dbus_checksums:
Justin TerAveste8c827b2017-05-19 12:58:35 -0600273 if self.options.target_dir == _CHROME_DIR:
274 logging.info('Detected change to D-Bus service files, rebooting.')
275 self._Reboot()
276 return
277 else:
278 logging.warn('Detected change in D-Bus service files, but target dir '
279 'is not %s. D-Bus changes will not be picked up by '
280 'dbus-daemon at boot time.', _CHROME_DIR)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600281
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800282 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800283 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500284 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700285
David James88e6f032013-03-02 08:13:20 -0800286 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700287 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800288 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500289 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800290 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700291 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800292 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700293
Steve Funge984a532013-11-25 17:09:25 -0800294 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700295 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700296 def BinaryExists(filename):
297 """Checks if the passed-in file is present in the build directory."""
298 return os.path.exists(os.path.join(self.options.build_dir, filename))
299
Daniel Erat9813f0e2014-11-12 11:00:28 -0700300 # Handle non-Chrome deployments.
301 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700302 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700303 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
304
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700305 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700306 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700307 self.options.startui = False
308
David James88e6f032013-03-02 08:13:20 -0800309 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800310 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
311 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800312
Thiago Goncales12793312013-05-23 11:26:17 -0700313 def _MountTarget(self):
314 logging.info('Mounting Chrome...')
315
316 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500317 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500318 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900319 # Umount the existing mount on mount_dir if present first
320 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
321 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500322 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
323 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700324 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500325 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
326
Justin TerAvestfac210e2017-04-13 11:39:00 -0600327 def _GetDBusChecksums(self):
328 """Returns Checksums for D-Bus files deployed with Chrome.
329
330 This is used to determine if a reboot is required after deploying Chrome.
331 """
Justin TerAveste8c827b2017-05-19 12:58:35 -0600332 path = os.path.join(_CHROME_DIR, 'dbus/*')
333 result = self.device.RunCommand('md5sum ' + path,
Justin TerAvestfac210e2017-04-13 11:39:00 -0600334 error_code_ok=True)
335 return result.output
336
Robert Flack1dc7ea82014-11-26 13:50:24 -0500337 def Cleanup(self):
338 """Clean up RemoteDevice."""
339 if not self.options.staging_only:
340 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700341
David James88e6f032013-03-02 08:13:20 -0800342 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800343 self._CheckDeployType()
344
David James88e6f032013-03-02 08:13:20 -0800345 # If requested, just do the staging step.
346 if self.options.staging_only:
347 self._PrepareStagingDir()
348 return 0
349
350 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800351 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800352 steps = [self._GetDeviceInfo, self._CheckConnection,
353 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
354 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700355 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
356 return_values=True)
357 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800358
Robert Flack1dc7ea82014-11-26 13:50:24 -0500359 # If we're trying to deploy to a dir which is not writable and we failed
360 # to mark the rootfs as writable, try disabling rootfs verification.
361 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800362 self._DisableRootfsVerification()
363
Thiago Goncales12793312013-05-23 11:26:17 -0700364 if self.options.mount_dir is not None:
365 self._MountTarget()
366
David James88e6f032013-03-02 08:13:20 -0800367 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700368 self._Deploy()
369
370
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700371def ValidateStagingFlags(value):
372 """Convert formatted string to dictionary."""
373 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700374
375
Steven Bennetts368c3e52016-09-23 13:05:21 -0700376def ValidateGnArgs(value):
377 """Convert GN_ARGS-formatted string to dictionary."""
378 return gn_helpers.FromGNArgs(value)
379
380
Ryan Cuie535b172012-10-19 18:25:03 -0700381def _CreateParser():
382 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400383 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700384
Ryan Cuia56a71e2012-10-18 18:40:35 -0700385 # TODO(rcui): Have this use the UI-V2 format of having source and target
386 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400387 parser.add_argument('--force', action='store_true', default=False,
388 help='Skip all prompts (i.e., for disabling of rootfs '
389 'verification). This may result in the target '
390 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800391 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400392 parser.add_argument('--board', default=sdk_board_env,
393 help="The board the Chrome build is targeted for. When "
394 "in a 'cros chrome-sdk' shell, defaults to the SDK "
395 "board.")
396 parser.add_argument('--build-dir', type='path',
397 help='The directory with Chrome build artifacts to '
398 'deploy from. Typically of format '
399 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700400 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400401 parser.add_argument('--target-dir', type='path',
402 default=None,
403 help='Target directory on device to deploy Chrome into.')
404 parser.add_argument('-g', '--gs-path', type='gs_path',
405 help='GS path that contains the chrome to deploy.')
406 parser.add_argument('--nostartui', action='store_false', dest='startui',
407 default=True,
408 help="Don't restart the ui daemon after deployment.")
409 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
410 default=True,
411 help="Don't strip binaries during deployment. Warning: "
412 'the resulting binaries will be very large!')
413 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
414 help='Port of the target device to connect to.')
415 parser.add_argument('-t', '--to',
416 help='The IP address of the CrOS device to deploy to.')
417 parser.add_argument('-v', '--verbose', action='store_true', default=False,
418 help='Show more debug output.')
419 parser.add_argument('--mount-dir', type='path', default=None,
420 help='Deploy Chrome in target directory and bind it '
421 'to the directory specified by this flag.'
422 'Any existing mount on this directory will be '
423 'umounted first.')
424 parser.add_argument('--mount', action='store_true', default=False,
425 help='Deploy Chrome to default target directory and bind '
426 'it to the default mount directory.'
427 'Any existing mount on this directory will be '
428 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700429
Mike Frysingerc3061a62015-06-04 04:16:18 -0400430 group = parser.add_argument_group('Advanced Options')
431 group.add_argument('-l', '--local-pkg-path', type='path',
432 help='Path to local chrome prebuilt package to deploy.')
433 group.add_argument('--sloppy', action='store_true', default=False,
434 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700435 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400436 help=('Extra flags to control staging. Valid flags are - '
437 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700438 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400439 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700440 help='Deprecated. Default behavior is "strict". Use '
441 '--sloppy to omit warnings for missing optional '
442 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400443 group.add_argument('--strip-flags', default=None,
444 help="Flags to call the 'strip' binutil tool with. "
445 "Overrides the default arguments.")
446 group.add_argument('--ping', action='store_true', default=False,
447 help='Ping the device before connection attempt.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700448
Mike Frysingerc3061a62015-06-04 04:16:18 -0400449 group = parser.add_argument_group(
450 'Metadata Overrides (Advanced)',
451 description='Provide all of these overrides in order to remove '
452 'dependencies on metadata.json existence.')
453 group.add_argument('--target-tc', action='store', default=None,
454 help='Override target toolchain name, e.g. '
455 'x86_64-cros-linux-gnu')
456 group.add_argument('--toolchain-url', action='store', default=None,
457 help='Override toolchain url format pattern, e.g. '
458 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700459
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700460 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
461 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
462 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400463 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700464
465 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
466 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
467 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
468 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
469 help=argparse.SUPPRESS)
470
Ryan Cuia56a71e2012-10-18 18:40:35 -0700471 # Path of an empty directory to stage chrome artifacts to. Defaults to a
472 # temporary directory that is removed when the script finishes. If the path
473 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400474 parser.add_argument('--staging-dir', type='path', default=None,
475 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700476 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400477 parser.add_argument('--staging-only', action='store_true', default=False,
478 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700479 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
480 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
481 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400482 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Ryan Cuie535b172012-10-19 18:25:03 -0700483 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700484
Ryan Cuie535b172012-10-19 18:25:03 -0700485
486def _ParseCommandLine(argv):
487 """Parse args, and run environment-independent checks."""
488 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400489 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700490
Ryan Cuia56a71e2012-10-18 18:40:35 -0700491 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
492 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800493 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700494 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
495 parser.error('Cannot specify both --build_dir and '
496 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800497 if options.build_dir and not options.board:
498 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700499 if options.gs_path and options.local_pkg_path:
500 parser.error('Cannot specify both --gs-path and --local-pkg-path')
501 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700502 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700503 if options.staging_flags and not options.build_dir:
504 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700505
Steven Bennetts46a84c32016-08-12 15:20:46 -0700506 if options.strict:
507 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700508 if options.gyp_defines:
509 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700510
511 if options.mount or options.mount_dir:
512 if not options.target_dir:
513 options.target_dir = _CHROME_DIR_MOUNT
514 else:
515 if not options.target_dir:
516 options.target_dir = _CHROME_DIR
517
518 if options.mount and not options.mount_dir:
519 options.mount_dir = _CHROME_DIR
520
Mike Frysingerc3061a62015-06-04 04:16:18 -0400521 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700522
523
Mike Frysingerc3061a62015-06-04 04:16:18 -0400524def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800525 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700526
527 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400528 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700529 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700530 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
531 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
532
Steven Bennetts368c3e52016-09-23 13:05:21 -0700533 if not options.gn_args:
534 gn_env = os.getenv('GN_ARGS')
535 if gn_env is not None:
536 options.gn_args = gn_helpers.FromGNArgs(gn_env)
537 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700538
Steven Bennetts60600462016-05-12 10:40:20 -0700539 if not options.staging_flags:
540 use_env = os.getenv('USE')
541 if use_env is not None:
542 options.staging_flags = ' '.join(set(use_env.split()).intersection(
543 chrome_util.STAGING_FLAGS))
544 logging.info('Staging flags taken from USE in environment: %s',
545 options.staging_flags)
546
Ryan Cuia56a71e2012-10-18 18:40:35 -0700547
Ryan Cui504db722013-01-22 11:48:01 -0800548def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549 """Get the chrome prebuilt tarball from GS.
550
Mike Frysinger02e1e072013-11-10 22:11:34 -0500551 Returns:
552 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700553 """
David James9374aac2013-10-08 16:00:17 -0700554 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500555 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800556 files = [found for found in files if
557 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
558 if not files:
559 raise Exception('No chrome package found at %s' % gs_path)
560 elif len(files) > 1:
561 # - Users should provide us with a direct link to either a stripped or
562 # unstripped chrome package.
563 # - In the case of being provided with an archive directory, where both
564 # stripped and unstripped chrome available, use the stripped chrome
565 # package.
566 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
567 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
568 files = [f for f in files if not 'unstripped' in f]
569 assert len(files) == 1
570 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800571
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800572 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800573 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800574 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
575 chrome_path = os.path.join(tempdir, filename)
576 assert os.path.exists(chrome_path)
577 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700578
579
Ryan Cuif890a3e2013-03-07 18:57:06 -0800580@contextlib.contextmanager
581def _StripBinContext(options):
582 if not options.dostrip:
583 yield None
584 elif options.strip_bin:
585 yield options.strip_bin
586 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800587 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800588 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700589 with sdk.Prepare(components=components, target_tc=options.target_tc,
590 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800591 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
592 constants.CHROME_ENV_FILE)
593 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
594 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
595 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800596 yield strip_bin
597
598
Steve Funge984a532013-11-25 17:09:25 -0800599def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
600 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800601 """Place the necessary files in the staging directory.
602
603 The staging directory is the directory used to rsync the build artifacts over
604 to the device. Only the necessary Chrome build artifacts are put into the
605 staging directory.
606 """
Ryan Cui5866be02013-03-18 14:12:00 -0700607 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400608 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800609 if options.build_dir:
610 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700611 strip_flags = (None if options.strip_flags is None else
612 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800613 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700614 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700615 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700616 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800617 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700618 else:
619 pkg_path = options.local_pkg_path
620 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800621 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
622 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700623
624 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800625 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700626 # Extract only the ./opt/google/chrome contents, directly into the staging
627 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800628 if pkg_path[-4:] == '.zip':
629 cros_build_lib.DebugRunCommand(
630 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
631 staging_dir])
632 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
633 shutil.move(filename, staging_dir)
634 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
635 else:
636 cros_build_lib.DebugRunCommand(
637 ['tar', '--strip-components', '4', '--extract',
638 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
639 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700640
Ryan Cui71aa8de2013-04-19 16:12:55 -0700641
Ryan Cui3045c5d2012-07-13 18:00:33 -0700642def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400643 options = _ParseCommandLine(argv)
644 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700645
646 # Set cros_build_lib debug level to hide RunCommand spew.
647 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700648 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700649 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800650 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700651
Aviv Keshet01a82e92017-03-02 17:39:59 -0800652 with osutils.TempDir(set_global=True) as tempdir:
653 staging_dir = options.staging_dir
654 if not staging_dir:
655 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700656
Aviv Keshet01a82e92017-03-02 17:39:59 -0800657 deploy = DeployChrome(options, tempdir, staging_dir)
658 try:
659 deploy.Perform()
660 except failures_lib.StepFailure as ex:
661 raise SystemExit(str(ex).strip())
662 deploy.Cleanup()