blob: e4f05b1e58ccd8befb55c17990d55e7c335d7f1a [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
32from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080042from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070043from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070054
Steve Funge984a532013-11-25 17:09:25 -080055_ANDROID_DIR = '/system/chrome'
56_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
57
David James2cb34002013-03-01 18:42:40 -080058_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070059_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080060
Pawel Osciakc1bb2742014-12-29 16:32:33 +090061_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
62 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070063_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
64_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070065
Steve Funge984a532013-11-25 17:09:25 -080066DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080067
Mike Frysingere65f3752014-12-08 00:46:39 -050068
Ryan Cui3045c5d2012-07-13 18:00:33 -070069def _UrlBaseName(url):
70 """Return the last component of the URL."""
71 return url.rstrip('/').rpartition('/')[-1]
72
73
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070074class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080075 """Raised whenever the deploy fails."""
76
77
Ryan Cui7193a7e2013-04-26 14:15:19 -070078DeviceInfo = collections.namedtuple(
79 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
80
81
Ryan Cui3045c5d2012-07-13 18:00:33 -070082class DeployChrome(object):
83 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050084
Ryan Cuia56a71e2012-10-18 18:40:35 -070085 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070086 """Initialize the class.
87
Mike Frysinger02e1e072013-11-10 22:11:34 -050088 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040089 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070090 tempdir: Scratch space for the class. Caller has responsibility to clean
91 it up.
Steve Funge984a532013-11-25 17:09:25 -080092 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070093 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070094 self.tempdir = tempdir
95 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070096 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050097 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090098 self.device = remote.RemoteDevice(options.to, port=options.port,
99 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500100 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000101
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500102 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800103 self.chrome_dir = _CHROME_DIR
104
Ryan Cui7193a7e2013-04-26 14:15:19 -0700105 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500106 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800108 value = line.split()[3]
109 multipliers = {
110 'G': 1024 * 1024 * 1024,
111 'M': 1024 * 1024,
112 'K': 1024,
113 }
114 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700115
116 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500117 result = self.device.RunCommand('du -ks %s' % remote_dir,
118 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119 return int(result.output.split()[0])
120
121 def _GetStagingDirSize(self):
122 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800123 redirect_stdout=True,
124 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700125 return int(result.output.split()[0])
126
Ryan Cui3045c5d2012-07-13 18:00:33 -0700127 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500128 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
129 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 return result.returncode == 0
131
Justin TerAvestfac210e2017-04-13 11:39:00 -0600132 def _Reboot(self):
133 # A reboot in developer mode takes a while (and has delays), so the user
134 # will have time to read and act on the USB boot instructions below.
135 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
136 self.device.Reboot()
137
Ryan Cui3045c5d2012-07-13 18:00:33 -0700138 def _DisableRootfsVerification(self):
139 if not self.options.force:
140 logging.error('Detected that the device has rootfs verification enabled.')
141 logging.info('This script can automatically remove the rootfs '
142 'verification, which requires that it reboot the device.')
143 logging.info('Make sure the device is in developer mode!')
144 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700145 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800146 # Since we stopped Chrome earlier, it's good form to start it up again.
147 if self.options.startui:
148 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500149 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800150 raise DeployFailure('Need rootfs verification to be disabled. '
151 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700152
153 logging.info('Removing rootfs verification from %s', self.options.to)
154 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
155 # Use --force to bypass the checks.
156 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
157 '--remove_rootfs_verification --force')
158 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500159 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160
Justin TerAvestfac210e2017-04-13 11:39:00 -0600161 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700191 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700203 'running chrome processes and try again.'
204 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400221 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Ryan Cui3045c5d2012-07-13 18:00:33 -0700248 def _Deploy(self):
Justin TerAvestfac210e2017-04-13 11:39:00 -0600249 old_dbus_checksums = self._GetDBusChecksums()
250
Pawel Osciak577773a2013-03-05 10:52:12 -0800251 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700252 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800253 dest_path = _CHROME_DIR
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700254 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
255 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900256 if not self.device.HasRsync():
257 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700258 'rsync is not found on the device.\n'
259 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500260 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
261 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700262 mode='rsync', inplace=True,
263 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500264 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800265
266 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800267 if p.mode:
268 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500269 self.device.RunCommand('chmod %o %s/%s' % (
270 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800271
Justin TerAvestfac210e2017-04-13 11:39:00 -0600272 new_dbus_checksums = self._GetDBusChecksums()
273 if old_dbus_checksums != new_dbus_checksums:
Justin TerAveste8c827b2017-05-19 12:58:35 -0600274 if self.options.target_dir == _CHROME_DIR:
275 logging.info('Detected change to D-Bus service files, rebooting.')
276 self._Reboot()
277 return
278 else:
279 logging.warn('Detected change in D-Bus service files, but target dir '
280 'is not %s. D-Bus changes will not be picked up by '
281 'dbus-daemon at boot time.', _CHROME_DIR)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600282
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800283 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800284 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500285 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700286
David James88e6f032013-03-02 08:13:20 -0800287 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700288 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800289 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500290 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800291 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700292 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800293 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700294
Steve Funge984a532013-11-25 17:09:25 -0800295 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700296 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700297 def BinaryExists(filename):
298 """Checks if the passed-in file is present in the build directory."""
299 return os.path.exists(os.path.join(self.options.build_dir, filename))
300
Daniel Erat9813f0e2014-11-12 11:00:28 -0700301 # Handle non-Chrome deployments.
302 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700303 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700304 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
305
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700306 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700307 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700308 self.options.startui = False
309
David James88e6f032013-03-02 08:13:20 -0800310 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800311 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
312 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800313
Thiago Goncales12793312013-05-23 11:26:17 -0700314 def _MountTarget(self):
315 logging.info('Mounting Chrome...')
316
317 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500318 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500319 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900320 # Umount the existing mount on mount_dir if present first
321 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
322 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500323 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
324 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700325 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500326 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
327
Justin TerAvestfac210e2017-04-13 11:39:00 -0600328 def _GetDBusChecksums(self):
329 """Returns Checksums for D-Bus files deployed with Chrome.
330
331 This is used to determine if a reboot is required after deploying Chrome.
332 """
Justin TerAveste8c827b2017-05-19 12:58:35 -0600333 path = os.path.join(_CHROME_DIR, 'dbus/*')
334 result = self.device.RunCommand('md5sum ' + path,
Justin TerAvestfac210e2017-04-13 11:39:00 -0600335 error_code_ok=True)
336 return result.output
337
Robert Flack1dc7ea82014-11-26 13:50:24 -0500338 def Cleanup(self):
339 """Clean up RemoteDevice."""
340 if not self.options.staging_only:
341 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700342
David James88e6f032013-03-02 08:13:20 -0800343 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800344 self._CheckDeployType()
345
David James88e6f032013-03-02 08:13:20 -0800346 # If requested, just do the staging step.
347 if self.options.staging_only:
348 self._PrepareStagingDir()
349 return 0
350
351 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800352 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800353 steps = [self._GetDeviceInfo, self._CheckConnection,
354 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
355 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700356 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
357 return_values=True)
358 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800359
Robert Flack1dc7ea82014-11-26 13:50:24 -0500360 # If we're trying to deploy to a dir which is not writable and we failed
361 # to mark the rootfs as writable, try disabling rootfs verification.
362 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800363 self._DisableRootfsVerification()
364
Thiago Goncales12793312013-05-23 11:26:17 -0700365 if self.options.mount_dir is not None:
366 self._MountTarget()
367
David James88e6f032013-03-02 08:13:20 -0800368 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700369 self._Deploy()
370
371
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700372def ValidateStagingFlags(value):
373 """Convert formatted string to dictionary."""
374 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700375
376
Steven Bennetts368c3e52016-09-23 13:05:21 -0700377def ValidateGnArgs(value):
378 """Convert GN_ARGS-formatted string to dictionary."""
379 return gn_helpers.FromGNArgs(value)
380
381
Ryan Cuie535b172012-10-19 18:25:03 -0700382def _CreateParser():
383 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400384 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700385
Ryan Cuia56a71e2012-10-18 18:40:35 -0700386 # TODO(rcui): Have this use the UI-V2 format of having source and target
387 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400388 parser.add_argument('--force', action='store_true', default=False,
389 help='Skip all prompts (i.e., for disabling of rootfs '
390 'verification). This may result in the target '
391 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800392 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400393 parser.add_argument('--board', default=sdk_board_env,
394 help="The board the Chrome build is targeted for. When "
395 "in a 'cros chrome-sdk' shell, defaults to the SDK "
396 "board.")
397 parser.add_argument('--build-dir', type='path',
398 help='The directory with Chrome build artifacts to '
399 'deploy from. Typically of format '
400 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700401 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400402 parser.add_argument('--target-dir', type='path',
403 default=None,
404 help='Target directory on device to deploy Chrome into.')
405 parser.add_argument('-g', '--gs-path', type='gs_path',
406 help='GS path that contains the chrome to deploy.')
407 parser.add_argument('--nostartui', action='store_false', dest='startui',
408 default=True,
409 help="Don't restart the ui daemon after deployment.")
410 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
411 default=True,
412 help="Don't strip binaries during deployment. Warning: "
413 'the resulting binaries will be very large!')
414 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
415 help='Port of the target device to connect to.')
416 parser.add_argument('-t', '--to',
417 help='The IP address of the CrOS device to deploy to.')
418 parser.add_argument('-v', '--verbose', action='store_true', default=False,
419 help='Show more debug output.')
420 parser.add_argument('--mount-dir', type='path', default=None,
421 help='Deploy Chrome in target directory and bind it '
422 'to the directory specified by this flag.'
423 'Any existing mount on this directory will be '
424 'umounted first.')
425 parser.add_argument('--mount', action='store_true', default=False,
426 help='Deploy Chrome to default target directory and bind '
427 'it to the default mount directory.'
428 'Any existing mount on this directory will be '
429 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700430
Mike Frysingerc3061a62015-06-04 04:16:18 -0400431 group = parser.add_argument_group('Advanced Options')
432 group.add_argument('-l', '--local-pkg-path', type='path',
433 help='Path to local chrome prebuilt package to deploy.')
434 group.add_argument('--sloppy', action='store_true', default=False,
435 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700436 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400437 help=('Extra flags to control staging. Valid flags are - '
438 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700439 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400440 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700441 help='Deprecated. Default behavior is "strict". Use '
442 '--sloppy to omit warnings for missing optional '
443 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400444 group.add_argument('--strip-flags', default=None,
445 help="Flags to call the 'strip' binutil tool with. "
446 "Overrides the default arguments.")
447 group.add_argument('--ping', action='store_true', default=False,
448 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700449 group.add_argument('--process-timeout', type=int,
450 default=KILL_PROC_MAX_WAIT,
451 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700452
Mike Frysingerc3061a62015-06-04 04:16:18 -0400453 group = parser.add_argument_group(
454 'Metadata Overrides (Advanced)',
455 description='Provide all of these overrides in order to remove '
456 'dependencies on metadata.json existence.')
457 group.add_argument('--target-tc', action='store', default=None,
458 help='Override target toolchain name, e.g. '
459 'x86_64-cros-linux-gnu')
460 group.add_argument('--toolchain-url', action='store', default=None,
461 help='Override toolchain url format pattern, e.g. '
462 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700463
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700464 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
465 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
466 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400467 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700468
469 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
470 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
471 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
472 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
473 help=argparse.SUPPRESS)
474
Ryan Cuia56a71e2012-10-18 18:40:35 -0700475 # Path of an empty directory to stage chrome artifacts to. Defaults to a
476 # temporary directory that is removed when the script finishes. If the path
477 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400478 parser.add_argument('--staging-dir', type='path', default=None,
479 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700480 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400481 parser.add_argument('--staging-only', action='store_true', default=False,
482 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700483 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
484 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
485 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400486 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Ryan Cuie535b172012-10-19 18:25:03 -0700487 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700488
Ryan Cuie535b172012-10-19 18:25:03 -0700489
490def _ParseCommandLine(argv):
491 """Parse args, and run environment-independent checks."""
492 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400493 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700494
Ryan Cuia56a71e2012-10-18 18:40:35 -0700495 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
496 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800497 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700498 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
499 parser.error('Cannot specify both --build_dir and '
500 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800501 if options.build_dir and not options.board:
502 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700503 if options.gs_path and options.local_pkg_path:
504 parser.error('Cannot specify both --gs-path and --local-pkg-path')
505 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700506 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700507 if options.staging_flags and not options.build_dir:
508 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700509
Steven Bennetts46a84c32016-08-12 15:20:46 -0700510 if options.strict:
511 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700512 if options.gyp_defines:
513 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700514
515 if options.mount or options.mount_dir:
516 if not options.target_dir:
517 options.target_dir = _CHROME_DIR_MOUNT
518 else:
519 if not options.target_dir:
520 options.target_dir = _CHROME_DIR
521
522 if options.mount and not options.mount_dir:
523 options.mount_dir = _CHROME_DIR
524
Mike Frysingerc3061a62015-06-04 04:16:18 -0400525 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700526
527
Mike Frysingerc3061a62015-06-04 04:16:18 -0400528def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800529 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700530
531 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400532 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700533 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700534 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
535 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
536
Steven Bennetts368c3e52016-09-23 13:05:21 -0700537 if not options.gn_args:
538 gn_env = os.getenv('GN_ARGS')
539 if gn_env is not None:
540 options.gn_args = gn_helpers.FromGNArgs(gn_env)
541 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700542
Steven Bennetts60600462016-05-12 10:40:20 -0700543 if not options.staging_flags:
544 use_env = os.getenv('USE')
545 if use_env is not None:
546 options.staging_flags = ' '.join(set(use_env.split()).intersection(
547 chrome_util.STAGING_FLAGS))
548 logging.info('Staging flags taken from USE in environment: %s',
549 options.staging_flags)
550
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551
Ryan Cui504db722013-01-22 11:48:01 -0800552def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700553 """Get the chrome prebuilt tarball from GS.
554
Mike Frysinger02e1e072013-11-10 22:11:34 -0500555 Returns:
556 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700557 """
David James9374aac2013-10-08 16:00:17 -0700558 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500559 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800560 files = [found for found in files if
561 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
562 if not files:
563 raise Exception('No chrome package found at %s' % gs_path)
564 elif len(files) > 1:
565 # - Users should provide us with a direct link to either a stripped or
566 # unstripped chrome package.
567 # - In the case of being provided with an archive directory, where both
568 # stripped and unstripped chrome available, use the stripped chrome
569 # package.
570 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
571 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
572 files = [f for f in files if not 'unstripped' in f]
573 assert len(files) == 1
574 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800575
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800576 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800577 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800578 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
579 chrome_path = os.path.join(tempdir, filename)
580 assert os.path.exists(chrome_path)
581 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700582
583
Ryan Cuif890a3e2013-03-07 18:57:06 -0800584@contextlib.contextmanager
585def _StripBinContext(options):
586 if not options.dostrip:
587 yield None
588 elif options.strip_bin:
589 yield options.strip_bin
590 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800591 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800592 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700593 with sdk.Prepare(components=components, target_tc=options.target_tc,
594 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800595 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
596 constants.CHROME_ENV_FILE)
597 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
598 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
599 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800600 yield strip_bin
601
602
Steve Funge984a532013-11-25 17:09:25 -0800603def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
604 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800605 """Place the necessary files in the staging directory.
606
607 The staging directory is the directory used to rsync the build artifacts over
608 to the device. Only the necessary Chrome build artifacts are put into the
609 staging directory.
610 """
Ryan Cui5866be02013-03-18 14:12:00 -0700611 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400612 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800613 if options.build_dir:
614 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700615 strip_flags = (None if options.strip_flags is None else
616 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800617 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700618 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700619 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700620 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800621 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700622 else:
623 pkg_path = options.local_pkg_path
624 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800625 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
626 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700627
628 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800629 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700630 # Extract only the ./opt/google/chrome contents, directly into the staging
631 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800632 if pkg_path[-4:] == '.zip':
633 cros_build_lib.DebugRunCommand(
634 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
635 staging_dir])
636 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
637 shutil.move(filename, staging_dir)
638 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
639 else:
640 cros_build_lib.DebugRunCommand(
641 ['tar', '--strip-components', '4', '--extract',
642 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
643 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700644
Ryan Cui71aa8de2013-04-19 16:12:55 -0700645
Ryan Cui3045c5d2012-07-13 18:00:33 -0700646def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400647 options = _ParseCommandLine(argv)
648 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700649
650 # Set cros_build_lib debug level to hide RunCommand spew.
651 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700652 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700653 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800654 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700655
Aviv Keshet01a82e92017-03-02 17:39:59 -0800656 with osutils.TempDir(set_global=True) as tempdir:
657 staging_dir = options.staging_dir
658 if not staging_dir:
659 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700660
Aviv Keshet01a82e92017-03-02 17:39:59 -0800661 deploy = DeployChrome(options, tempdir, staging_dir)
662 try:
663 deploy.Perform()
664 except failures_lib.StepFailure as ex:
665 raise SystemExit(str(ex).strip())
666 deploy.Cleanup()