blob: b4b19ffb724e98dcc00043763ee41a0e2e8997bd [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Aviv Keshetb7519e12016-10-04 00:50:00 -070031from chromite.lib import constants
32from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080042from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070043from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Daniel Erat0fce5bf2017-09-28 17:29:23 -070054DBUS_RELOAD_COMMAND = 'killall -HUP dbus-daemon'
Ryan Cui3045c5d2012-07-13 18:00:33 -070055
Steve Funge984a532013-11-25 17:09:25 -080056_ANDROID_DIR = '/system/chrome'
57_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
58
David James2cb34002013-03-01 18:42:40 -080059_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070060_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080061
Pawel Osciakc1bb2742014-12-29 16:32:33 +090062_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
63 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070064_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
65_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070066
Steve Funge984a532013-11-25 17:09:25 -080067DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080068
Mike Frysingere65f3752014-12-08 00:46:39 -050069
Ryan Cui3045c5d2012-07-13 18:00:33 -070070def _UrlBaseName(url):
71 """Return the last component of the URL."""
72 return url.rstrip('/').rpartition('/')[-1]
73
74
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070075class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080076 """Raised whenever the deploy fails."""
77
78
Ryan Cui7193a7e2013-04-26 14:15:19 -070079DeviceInfo = collections.namedtuple(
80 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
81
82
Ryan Cui3045c5d2012-07-13 18:00:33 -070083class DeployChrome(object):
84 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050085
Ryan Cuia56a71e2012-10-18 18:40:35 -070086 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070087 """Initialize the class.
88
Mike Frysinger02e1e072013-11-10 22:11:34 -050089 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040090 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070091 tempdir: Scratch space for the class. Caller has responsibility to clean
92 it up.
Steve Funge984a532013-11-25 17:09:25 -080093 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070094 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070095 self.tempdir = tempdir
96 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070097 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050098 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090099 self.device = remote.RemoteDevice(options.to, port=options.port,
100 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500101 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000102
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500103 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800104 self.chrome_dir = _CHROME_DIR
105
Ryan Cui7193a7e2013-04-26 14:15:19 -0700106 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500107 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800109 value = line.split()[3]
110 multipliers = {
111 'G': 1024 * 1024 * 1024,
112 'M': 1024 * 1024,
113 'K': 1024,
114 }
115 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700116
117 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500118 result = self.device.RunCommand('du -ks %s' % remote_dir,
119 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700120 return int(result.output.split()[0])
121
122 def _GetStagingDirSize(self):
123 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800124 redirect_stdout=True,
125 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700126 return int(result.output.split()[0])
127
Ryan Cui3045c5d2012-07-13 18:00:33 -0700128 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500129 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
130 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700131 return result.returncode == 0
132
Justin TerAvestfac210e2017-04-13 11:39:00 -0600133 def _Reboot(self):
134 # A reboot in developer mode takes a while (and has delays), so the user
135 # will have time to read and act on the USB boot instructions below.
136 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
137 self.device.Reboot()
138
Ryan Cui3045c5d2012-07-13 18:00:33 -0700139 def _DisableRootfsVerification(self):
140 if not self.options.force:
141 logging.error('Detected that the device has rootfs verification enabled.')
142 logging.info('This script can automatically remove the rootfs '
143 'verification, which requires that it reboot the device.')
144 logging.info('Make sure the device is in developer mode!')
145 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700146 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800147 # Since we stopped Chrome earlier, it's good form to start it up again.
148 if self.options.startui:
149 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500150 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800151 raise DeployFailure('Need rootfs verification to be disabled. '
152 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700153
154 logging.info('Removing rootfs verification from %s', self.options.to)
155 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
156 # Use --force to bypass the checks.
157 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
158 '--remove_rootfs_verification --force')
159 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500160 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700161
Justin TerAvestfac210e2017-04-13 11:39:00 -0600162 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700163
David James88e6f032013-03-02 08:13:20 -0800164 # Now that the machine has been rebooted, we need to kill Chrome again.
165 self._KillProcsIfNeeded()
166
167 # Make sure the rootfs is writable now.
168 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700169
170 def _CheckUiJobStarted(self):
171 # status output is in the format:
172 # <job_name> <status> ['process' <pid>].
173 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500175 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800176 except cros_build_lib.RunCommandError as e:
177 if 'Unknown job' in e.result.error:
178 return False
179 else:
180 raise e
181
Ryan Cui3045c5d2012-07-13 18:00:33 -0700182 return result.output.split()[1].split('/')[0] == 'start'
183
184 def _KillProcsIfNeeded(self):
185 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800186 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500187 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188
189 # Developers sometimes run session_manager manually, in which case we'll
190 # need to help shut the chrome processes down.
191 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700192 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700193 while self._ChromeFileInUse():
194 logging.warning('The chrome binary on the device is in use.')
195 logging.warning('Killing chrome and session_manager processes...\n')
196
Robert Flack1dc7ea82014-11-26 13:50:24 -0500197 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500198 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700199 # Wait for processes to actually terminate
200 time.sleep(POST_KILL_WAIT)
201 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800202 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800203 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700204 'running chrome processes and try again.'
205 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800206 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700207
David James88e6f032013-03-02 08:13:20 -0800208 def _MountRootfsAsWritable(self, error_code_ok=True):
209 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700210
Robert Flack1dc7ea82014-11-26 13:50:24 -0500211 If the command fails, and error_code_ok is True, and the target dir is not
212 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700213
Mike Frysinger02e1e072013-11-10 22:11:34 -0500214 Args:
David James88e6f032013-03-02 08:13:20 -0800215 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
216 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800217 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500218 result = self.device.RunCommand(MOUNT_RW_COMMAND,
219 error_code_ok=error_code_ok,
220 capture_output=True)
221 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400222 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500223 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700224
Ryan Cui7193a7e2013-04-26 14:15:19 -0700225 def _GetDeviceInfo(self):
226 steps = [
227 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
228 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
229 ]
230 return_values = parallel.RunParallelSteps(steps, return_values=True)
231 return DeviceInfo(*return_values)
232
233 def _CheckDeviceFreeSpace(self, device_info):
234 """See if target device has enough space for Chrome.
235
Mike Frysinger02e1e072013-11-10 22:11:34 -0500236 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700237 device_info: A DeviceInfo named tuple.
238 """
239 effective_free = device_info.target_dir_size + device_info.target_fs_free
240 staging_size = self._GetStagingDirSize()
241 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700242 raise DeployFailure(
243 'Not enough free space on the device. Required: %s MiB, '
244 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700245 if device_info.target_fs_free < (100 * 1024):
246 logging.warning('The device has less than 100MB free. deploy_chrome may '
247 'hang during the transfer.')
248
Satoru Takabayashif2893002017-06-20 14:52:48 +0900249 def _ShouldUseCompression(self):
250 """Checks if compression should be used for rsync."""
251 if self.options.compress == 'always':
252 return True
253 elif self.options.compress == 'never':
254 return False
255 elif self.options.compress == 'auto':
256 return not self.device.HasGigabitEthernet()
257
Ryan Cui3045c5d2012-07-13 18:00:33 -0700258 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800259 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700260 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
261 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900262 if not self.device.HasRsync():
263 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700264 'rsync is not found on the device.\n'
265 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500266 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
267 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700268 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900269 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700270 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500271 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800272
273 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800274 if p.mode:
275 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500276 self.device.RunCommand('chmod %o %s/%s' % (
Satoru Takabayashi4b36f1c2017-06-20 15:39:40 +0900277 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800278
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700279 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
280 # pick up major changes (bus type, logging, etc.), but all we care about is
281 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
282 # be authorized to take ownership of its service names.
283 self.device.RunCommand(DBUS_RELOAD_COMMAND, error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600284
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800285 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800286 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500287 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700288
David James88e6f032013-03-02 08:13:20 -0800289 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700290 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800291 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500292 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800293 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700294 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800295 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700296
Steve Funge984a532013-11-25 17:09:25 -0800297 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700298 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700299 def BinaryExists(filename):
300 """Checks if the passed-in file is present in the build directory."""
301 return os.path.exists(os.path.join(self.options.build_dir, filename))
302
Daniel Erat9813f0e2014-11-12 11:00:28 -0700303 # Handle non-Chrome deployments.
304 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700305 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700306 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
307
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700308 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700309 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700310 self.options.startui = False
311
David James88e6f032013-03-02 08:13:20 -0800312 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800313 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
314 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800315
Thiago Goncales12793312013-05-23 11:26:17 -0700316 def _MountTarget(self):
317 logging.info('Mounting Chrome...')
318
319 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500320 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500321 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900322 # Umount the existing mount on mount_dir if present first
323 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
324 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500325 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
326 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700327 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500328 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
329
330 def Cleanup(self):
331 """Clean up RemoteDevice."""
332 if not self.options.staging_only:
333 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700334
David James88e6f032013-03-02 08:13:20 -0800335 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800336 self._CheckDeployType()
337
David James88e6f032013-03-02 08:13:20 -0800338 # If requested, just do the staging step.
339 if self.options.staging_only:
340 self._PrepareStagingDir()
341 return 0
342
343 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800344 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800345 steps = [self._GetDeviceInfo, self._CheckConnection,
346 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
347 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700348 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
349 return_values=True)
350 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800351
Robert Flack1dc7ea82014-11-26 13:50:24 -0500352 # If we're trying to deploy to a dir which is not writable and we failed
353 # to mark the rootfs as writable, try disabling rootfs verification.
354 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800355 self._DisableRootfsVerification()
356
Thiago Goncales12793312013-05-23 11:26:17 -0700357 if self.options.mount_dir is not None:
358 self._MountTarget()
359
David James88e6f032013-03-02 08:13:20 -0800360 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700361 self._Deploy()
362
363
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700364def ValidateStagingFlags(value):
365 """Convert formatted string to dictionary."""
366 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700367
368
Steven Bennetts368c3e52016-09-23 13:05:21 -0700369def ValidateGnArgs(value):
370 """Convert GN_ARGS-formatted string to dictionary."""
371 return gn_helpers.FromGNArgs(value)
372
373
Ryan Cuie535b172012-10-19 18:25:03 -0700374def _CreateParser():
375 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400376 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700377
Ryan Cuia56a71e2012-10-18 18:40:35 -0700378 # TODO(rcui): Have this use the UI-V2 format of having source and target
379 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400380 parser.add_argument('--force', action='store_true', default=False,
381 help='Skip all prompts (i.e., for disabling of rootfs '
382 'verification). This may result in the target '
383 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800384 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400385 parser.add_argument('--board', default=sdk_board_env,
386 help="The board the Chrome build is targeted for. When "
387 "in a 'cros chrome-sdk' shell, defaults to the SDK "
388 "board.")
389 parser.add_argument('--build-dir', type='path',
390 help='The directory with Chrome build artifacts to '
391 'deploy from. Typically of format '
392 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700393 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400394 parser.add_argument('--target-dir', type='path',
395 default=None,
396 help='Target directory on device to deploy Chrome into.')
397 parser.add_argument('-g', '--gs-path', type='gs_path',
398 help='GS path that contains the chrome to deploy.')
399 parser.add_argument('--nostartui', action='store_false', dest='startui',
400 default=True,
401 help="Don't restart the ui daemon after deployment.")
402 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
403 default=True,
404 help="Don't strip binaries during deployment. Warning: "
405 'the resulting binaries will be very large!')
406 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
407 help='Port of the target device to connect to.')
408 parser.add_argument('-t', '--to',
409 help='The IP address of the CrOS device to deploy to.')
410 parser.add_argument('-v', '--verbose', action='store_true', default=False,
411 help='Show more debug output.')
412 parser.add_argument('--mount-dir', type='path', default=None,
413 help='Deploy Chrome in target directory and bind it '
414 'to the directory specified by this flag.'
415 'Any existing mount on this directory will be '
416 'umounted first.')
417 parser.add_argument('--mount', action='store_true', default=False,
418 help='Deploy Chrome to default target directory and bind '
419 'it to the default mount directory.'
420 'Any existing mount on this directory will be '
421 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700422
Mike Frysingerc3061a62015-06-04 04:16:18 -0400423 group = parser.add_argument_group('Advanced Options')
424 group.add_argument('-l', '--local-pkg-path', type='path',
425 help='Path to local chrome prebuilt package to deploy.')
426 group.add_argument('--sloppy', action='store_true', default=False,
427 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700428 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400429 help=('Extra flags to control staging. Valid flags are - '
430 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700431 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400432 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700433 help='Deprecated. Default behavior is "strict". Use '
434 '--sloppy to omit warnings for missing optional '
435 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400436 group.add_argument('--strip-flags', default=None,
437 help="Flags to call the 'strip' binutil tool with. "
438 "Overrides the default arguments.")
439 group.add_argument('--ping', action='store_true', default=False,
440 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700441 group.add_argument('--process-timeout', type=int,
442 default=KILL_PROC_MAX_WAIT,
443 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700444
Mike Frysingerc3061a62015-06-04 04:16:18 -0400445 group = parser.add_argument_group(
446 'Metadata Overrides (Advanced)',
447 description='Provide all of these overrides in order to remove '
448 'dependencies on metadata.json existence.')
449 group.add_argument('--target-tc', action='store', default=None,
450 help='Override target toolchain name, e.g. '
451 'x86_64-cros-linux-gnu')
452 group.add_argument('--toolchain-url', action='store', default=None,
453 help='Override toolchain url format pattern, e.g. '
454 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700455
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700456 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
457 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
458 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400459 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700460
461 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
462 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
463 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
464 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
465 help=argparse.SUPPRESS)
466
Ryan Cuia56a71e2012-10-18 18:40:35 -0700467 # Path of an empty directory to stage chrome artifacts to. Defaults to a
468 # temporary directory that is removed when the script finishes. If the path
469 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400470 parser.add_argument('--staging-dir', type='path', default=None,
471 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400473 parser.add_argument('--staging-only', action='store_true', default=False,
474 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700475 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
476 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
477 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400478 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900479 parser.add_argument('--compress', action='store', default='auto',
480 choices=('always', 'never', 'auto'),
481 help='Choose the data compression behavior. Default '
482 'is set to "auto", that disables compression if '
483 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700484 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700485
Ryan Cuie535b172012-10-19 18:25:03 -0700486
487def _ParseCommandLine(argv):
488 """Parse args, and run environment-independent checks."""
489 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400490 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700491
Ryan Cuia56a71e2012-10-18 18:40:35 -0700492 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
493 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800494 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700495 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
496 parser.error('Cannot specify both --build_dir and '
497 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800498 if options.build_dir and not options.board:
499 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700500 if options.gs_path and options.local_pkg_path:
501 parser.error('Cannot specify both --gs-path and --local-pkg-path')
502 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700503 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700504 if options.staging_flags and not options.build_dir:
505 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700506
Steven Bennetts46a84c32016-08-12 15:20:46 -0700507 if options.strict:
508 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700509 if options.gyp_defines:
510 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700511
512 if options.mount or options.mount_dir:
513 if not options.target_dir:
514 options.target_dir = _CHROME_DIR_MOUNT
515 else:
516 if not options.target_dir:
517 options.target_dir = _CHROME_DIR
518
519 if options.mount and not options.mount_dir:
520 options.mount_dir = _CHROME_DIR
521
Mike Frysingerc3061a62015-06-04 04:16:18 -0400522 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700523
524
Mike Frysingerc3061a62015-06-04 04:16:18 -0400525def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800526 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700527
528 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400529 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700530 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700531 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
532 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
533
Steven Bennetts368c3e52016-09-23 13:05:21 -0700534 if not options.gn_args:
535 gn_env = os.getenv('GN_ARGS')
536 if gn_env is not None:
537 options.gn_args = gn_helpers.FromGNArgs(gn_env)
538 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700539
Steven Bennetts60600462016-05-12 10:40:20 -0700540 if not options.staging_flags:
541 use_env = os.getenv('USE')
542 if use_env is not None:
543 options.staging_flags = ' '.join(set(use_env.split()).intersection(
544 chrome_util.STAGING_FLAGS))
545 logging.info('Staging flags taken from USE in environment: %s',
546 options.staging_flags)
547
Ryan Cuia56a71e2012-10-18 18:40:35 -0700548
Ryan Cui504db722013-01-22 11:48:01 -0800549def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700550 """Get the chrome prebuilt tarball from GS.
551
Mike Frysinger02e1e072013-11-10 22:11:34 -0500552 Returns:
553 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700554 """
David James9374aac2013-10-08 16:00:17 -0700555 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500556 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800557 files = [found for found in files if
558 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
559 if not files:
560 raise Exception('No chrome package found at %s' % gs_path)
561 elif len(files) > 1:
562 # - Users should provide us with a direct link to either a stripped or
563 # unstripped chrome package.
564 # - In the case of being provided with an archive directory, where both
565 # stripped and unstripped chrome available, use the stripped chrome
566 # package.
567 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
568 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
569 files = [f for f in files if not 'unstripped' in f]
570 assert len(files) == 1
571 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800572
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800573 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800574 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800575 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
576 chrome_path = os.path.join(tempdir, filename)
577 assert os.path.exists(chrome_path)
578 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700579
580
Ryan Cuif890a3e2013-03-07 18:57:06 -0800581@contextlib.contextmanager
582def _StripBinContext(options):
583 if not options.dostrip:
584 yield None
585 elif options.strip_bin:
586 yield options.strip_bin
587 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800588 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800589 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700590 with sdk.Prepare(components=components, target_tc=options.target_tc,
591 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800592 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
593 constants.CHROME_ENV_FILE)
594 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
595 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
596 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800597 yield strip_bin
598
599
Steve Funge984a532013-11-25 17:09:25 -0800600def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
601 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800602 """Place the necessary files in the staging directory.
603
604 The staging directory is the directory used to rsync the build artifacts over
605 to the device. Only the necessary Chrome build artifacts are put into the
606 staging directory.
607 """
Ryan Cui5866be02013-03-18 14:12:00 -0700608 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400609 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800610 if options.build_dir:
611 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700612 strip_flags = (None if options.strip_flags is None else
613 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800614 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700615 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700616 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700617 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800618 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700619 else:
620 pkg_path = options.local_pkg_path
621 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800622 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
623 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700624
625 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800626 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700627 # Extract only the ./opt/google/chrome contents, directly into the staging
628 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800629 if pkg_path[-4:] == '.zip':
630 cros_build_lib.DebugRunCommand(
631 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
632 staging_dir])
633 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
634 shutil.move(filename, staging_dir)
635 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
636 else:
637 cros_build_lib.DebugRunCommand(
638 ['tar', '--strip-components', '4', '--extract',
639 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
640 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700641
Ryan Cui71aa8de2013-04-19 16:12:55 -0700642
Ryan Cui3045c5d2012-07-13 18:00:33 -0700643def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400644 options = _ParseCommandLine(argv)
645 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700646
647 # Set cros_build_lib debug level to hide RunCommand spew.
648 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700649 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700650 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800651 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700652
Aviv Keshet01a82e92017-03-02 17:39:59 -0800653 with osutils.TempDir(set_global=True) as tempdir:
654 staging_dir = options.staging_dir
655 if not staging_dir:
656 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700657
Aviv Keshet01a82e92017-03-02 17:39:59 -0800658 deploy = DeployChrome(options, tempdir, staging_dir)
659 try:
660 deploy.Perform()
661 except failures_lib.StepFailure as ex:
662 raise SystemExit(str(ex).strip())
663 deploy.Cleanup()