blob: d913e73a0ea6747d394c8dfe589939d60472edf3 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
David Haddock3151d912017-10-24 03:50:32 +000053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
54LSOF_COMMAND = 'lsof %s/chrome'
55DBUS_RELOAD_COMMAND = 'killall -HUP dbus-daemon'
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
David Haddock3151d912017-10-24 03:50:32 +000063_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
64 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
65_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
66_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
67
68DF_COMMAND = 'df -k %s'
69
Mike Frysingere65f3752014-12-08 00:46:39 -050070
Ryan Cui3045c5d2012-07-13 18:00:33 -070071def _UrlBaseName(url):
72 """Return the last component of the URL."""
73 return url.rstrip('/').rpartition('/')[-1]
74
75
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070076class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080077 """Raised whenever the deploy fails."""
78
79
Ryan Cui7193a7e2013-04-26 14:15:19 -070080DeviceInfo = collections.namedtuple(
81 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
82
83
Ryan Cui3045c5d2012-07-13 18:00:33 -070084class DeployChrome(object):
85 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050086
Ryan Cuia56a71e2012-10-18 18:40:35 -070087 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070088 """Initialize the class.
89
Mike Frysinger02e1e072013-11-10 22:11:34 -050090 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040091 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070092 tempdir: Scratch space for the class. Caller has responsibility to clean
93 it up.
Steve Funge984a532013-11-25 17:09:25 -080094 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070095 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070096 self.tempdir = tempdir
97 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070098 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050099 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900100 self.device = remote.RemoteDevice(options.to, port=options.port,
101 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500102 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000103
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500104 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800105 self.chrome_dir = _CHROME_DIR
106
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 def _GetRemoteMountFree(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000108 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700109 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800110 value = line.split()[3]
111 multipliers = {
112 'G': 1024 * 1024 * 1024,
113 'M': 1024 * 1024,
114 'K': 1024,
115 }
116 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700117
118 def _GetRemoteDirSize(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000119 result = self.device.RunCommand('du -ks %s' % remote_dir,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500120 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700121 return int(result.output.split()[0])
122
123 def _GetStagingDirSize(self):
124 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800125 redirect_stdout=True,
126 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700127 return int(result.output.split()[0])
128
Ryan Cui3045c5d2012-07-13 18:00:33 -0700129 def _ChromeFileInUse(self):
David Haddock3151d912017-10-24 03:50:32 +0000130 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
131 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700132 return result.returncode == 0
133
Justin TerAvestfac210e2017-04-13 11:39:00 -0600134 def _Reboot(self):
135 # A reboot in developer mode takes a while (and has delays), so the user
136 # will have time to read and act on the USB boot instructions below.
137 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
138 self.device.Reboot()
139
Ryan Cui3045c5d2012-07-13 18:00:33 -0700140 def _DisableRootfsVerification(self):
141 if not self.options.force:
142 logging.error('Detected that the device has rootfs verification enabled.')
143 logging.info('This script can automatically remove the rootfs '
144 'verification, which requires that it reboot the device.')
145 logging.info('Make sure the device is in developer mode!')
146 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700147 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800148 # Since we stopped Chrome earlier, it's good form to start it up again.
149 if self.options.startui:
150 logging.info('Starting Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000151 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800152 raise DeployFailure('Need rootfs verification to be disabled. '
153 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700154
155 logging.info('Removing rootfs verification from %s', self.options.to)
156 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
157 # Use --force to bypass the checks.
David Haddock3151d912017-10-24 03:50:32 +0000158 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
159 '--remove_rootfs_verification --force')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
David Haddock3151d912017-10-24 03:50:32 +0000161 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
Justin TerAvestfac210e2017-04-13 11:39:00 -0600163 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700164
David James88e6f032013-03-02 08:13:20 -0800165 # Now that the machine has been rebooted, we need to kill Chrome again.
166 self._KillProcsIfNeeded()
167
168 # Make sure the rootfs is writable now.
169 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700170
171 def _CheckUiJobStarted(self):
172 # status output is in the format:
173 # <job_name> <status> ['process' <pid>].
174 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 try:
David Haddock3151d912017-10-24 03:50:32 +0000176 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800177 except cros_build_lib.RunCommandError as e:
178 if 'Unknown job' in e.result.error:
179 return False
180 else:
181 raise e
182
Ryan Cui3045c5d2012-07-13 18:00:33 -0700183 return result.output.split()[1].split('/')[0] == 'start'
184
185 def _KillProcsIfNeeded(self):
186 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800187 logging.info('Shutting down Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000188 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700189
190 # Developers sometimes run session_manager manually, in which case we'll
191 # need to help shut the chrome processes down.
192 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700193 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700194 while self._ChromeFileInUse():
195 logging.warning('The chrome binary on the device is in use.')
196 logging.warning('Killing chrome and session_manager processes...\n')
197
David Haddock3151d912017-10-24 03:50:32 +0000198 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500199 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700200 # Wait for processes to actually terminate
201 time.sleep(POST_KILL_WAIT)
202 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800203 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800204 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700205 'running chrome processes and try again.'
206 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800207 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
David James88e6f032013-03-02 08:13:20 -0800209 def _MountRootfsAsWritable(self, error_code_ok=True):
210 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700211
Robert Flack1dc7ea82014-11-26 13:50:24 -0500212 If the command fails, and error_code_ok is True, and the target dir is not
213 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700214
Mike Frysinger02e1e072013-11-10 22:11:34 -0500215 Args:
David James88e6f032013-03-02 08:13:20 -0800216 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
217 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800218 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500219 result = self.device.RunCommand(MOUNT_RW_COMMAND,
220 error_code_ok=error_code_ok,
221 capture_output=True)
222 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400223 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500224 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700225
Ryan Cui7193a7e2013-04-26 14:15:19 -0700226 def _GetDeviceInfo(self):
227 steps = [
228 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
229 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
230 ]
231 return_values = parallel.RunParallelSteps(steps, return_values=True)
232 return DeviceInfo(*return_values)
233
234 def _CheckDeviceFreeSpace(self, device_info):
235 """See if target device has enough space for Chrome.
236
Mike Frysinger02e1e072013-11-10 22:11:34 -0500237 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700238 device_info: A DeviceInfo named tuple.
239 """
240 effective_free = device_info.target_dir_size + device_info.target_fs_free
241 staging_size = self._GetStagingDirSize()
242 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700243 raise DeployFailure(
244 'Not enough free space on the device. Required: %s MiB, '
245 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700246 if device_info.target_fs_free < (100 * 1024):
247 logging.warning('The device has less than 100MB free. deploy_chrome may '
248 'hang during the transfer.')
249
Satoru Takabayashif2893002017-06-20 14:52:48 +0900250 def _ShouldUseCompression(self):
251 """Checks if compression should be used for rsync."""
252 if self.options.compress == 'always':
253 return True
254 elif self.options.compress == 'never':
255 return False
256 elif self.options.compress == 'auto':
257 return not self.device.HasGigabitEthernet()
258
Ryan Cui3045c5d2012-07-13 18:00:33 -0700259 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800260 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700261 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
262 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900263 if not self.device.HasRsync():
264 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700265 'rsync is not found on the device.\n'
266 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500267 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
268 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700269 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900270 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700271 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500272 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800273
274 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800275 if p.mode:
276 # Set mode if necessary.
David Haddock3151d912017-10-24 03:50:32 +0000277 self.device.RunCommand('chmod %o %s/%s' % (
278 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800279
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700280 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
281 # pick up major changes (bus type, logging, etc.), but all we care about is
282 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
283 # be authorized to take ownership of its service names.
David Haddock3151d912017-10-24 03:50:32 +0000284 self.device.RunCommand(DBUS_RELOAD_COMMAND, error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600285
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800286 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800287 logging.info('Starting UI...')
David Haddock3151d912017-10-24 03:50:32 +0000288 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700289
David James88e6f032013-03-02 08:13:20 -0800290 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700291 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800292 logging.info('Testing connection to the device...')
David Haddock3151d912017-10-24 03:50:32 +0000293 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800294 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700295 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800296 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700297
Steve Funge984a532013-11-25 17:09:25 -0800298 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700299 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700300 def BinaryExists(filename):
301 """Checks if the passed-in file is present in the build directory."""
302 return os.path.exists(os.path.join(self.options.build_dir, filename))
303
Daniel Erat9813f0e2014-11-12 11:00:28 -0700304 # Handle non-Chrome deployments.
305 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700306 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700307 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
308
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700309 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700310 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700311 self.options.startui = False
312
David James88e6f032013-03-02 08:13:20 -0800313 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800314 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
315 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800316
Thiago Goncales12793312013-05-23 11:26:17 -0700317 def _MountTarget(self):
318 logging.info('Mounting Chrome...')
319
320 # Create directory if does not exist
David Haddock3151d912017-10-24 03:50:32 +0000321 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
322 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900323 # Umount the existing mount on mount_dir if present first
David Haddock3151d912017-10-24 03:50:32 +0000324 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
325 {'dir': self.options.mount_dir})
326 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
327 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700328 # Chrome needs partition to have exec and suid flags set
David Haddock3151d912017-10-24 03:50:32 +0000329 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
Robert Flack1dc7ea82014-11-26 13:50:24 -0500330
331 def Cleanup(self):
332 """Clean up RemoteDevice."""
333 if not self.options.staging_only:
334 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700335
David James88e6f032013-03-02 08:13:20 -0800336 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800337 self._CheckDeployType()
338
David James88e6f032013-03-02 08:13:20 -0800339 # If requested, just do the staging step.
340 if self.options.staging_only:
341 self._PrepareStagingDir()
342 return 0
343
344 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800345 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800346 steps = [self._GetDeviceInfo, self._CheckConnection,
347 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
348 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700349 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
350 return_values=True)
351 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800352
Robert Flack1dc7ea82014-11-26 13:50:24 -0500353 # If we're trying to deploy to a dir which is not writable and we failed
354 # to mark the rootfs as writable, try disabling rootfs verification.
355 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800356 self._DisableRootfsVerification()
357
Thiago Goncales12793312013-05-23 11:26:17 -0700358 if self.options.mount_dir is not None:
359 self._MountTarget()
360
David James88e6f032013-03-02 08:13:20 -0800361 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700362 self._Deploy()
363
364
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700365def ValidateStagingFlags(value):
366 """Convert formatted string to dictionary."""
367 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700368
369
Steven Bennetts368c3e52016-09-23 13:05:21 -0700370def ValidateGnArgs(value):
371 """Convert GN_ARGS-formatted string to dictionary."""
372 return gn_helpers.FromGNArgs(value)
373
374
Ryan Cuie535b172012-10-19 18:25:03 -0700375def _CreateParser():
376 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400377 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700378
Ryan Cuia56a71e2012-10-18 18:40:35 -0700379 # TODO(rcui): Have this use the UI-V2 format of having source and target
380 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400381 parser.add_argument('--force', action='store_true', default=False,
382 help='Skip all prompts (i.e., for disabling of rootfs '
383 'verification). This may result in the target '
384 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800385 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400386 parser.add_argument('--board', default=sdk_board_env,
387 help="The board the Chrome build is targeted for. When "
388 "in a 'cros chrome-sdk' shell, defaults to the SDK "
389 "board.")
390 parser.add_argument('--build-dir', type='path',
391 help='The directory with Chrome build artifacts to '
392 'deploy from. Typically of format '
393 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700394 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400395 parser.add_argument('--target-dir', type='path',
396 default=None,
397 help='Target directory on device to deploy Chrome into.')
398 parser.add_argument('-g', '--gs-path', type='gs_path',
399 help='GS path that contains the chrome to deploy.')
400 parser.add_argument('--nostartui', action='store_false', dest='startui',
401 default=True,
402 help="Don't restart the ui daemon after deployment.")
403 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
404 default=True,
405 help="Don't strip binaries during deployment. Warning: "
406 'the resulting binaries will be very large!')
407 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
408 help='Port of the target device to connect to.')
409 parser.add_argument('-t', '--to',
410 help='The IP address of the CrOS device to deploy to.')
411 parser.add_argument('-v', '--verbose', action='store_true', default=False,
412 help='Show more debug output.')
413 parser.add_argument('--mount-dir', type='path', default=None,
414 help='Deploy Chrome in target directory and bind it '
415 'to the directory specified by this flag.'
416 'Any existing mount on this directory will be '
417 'umounted first.')
418 parser.add_argument('--mount', action='store_true', default=False,
419 help='Deploy Chrome to default target directory and bind '
420 'it to the default mount directory.'
421 'Any existing mount on this directory will be '
422 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700423
Mike Frysingerc3061a62015-06-04 04:16:18 -0400424 group = parser.add_argument_group('Advanced Options')
425 group.add_argument('-l', '--local-pkg-path', type='path',
426 help='Path to local chrome prebuilt package to deploy.')
427 group.add_argument('--sloppy', action='store_true', default=False,
428 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700429 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400430 help=('Extra flags to control staging. Valid flags are - '
431 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700432 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400433 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700434 help='Deprecated. Default behavior is "strict". Use '
435 '--sloppy to omit warnings for missing optional '
436 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400437 group.add_argument('--strip-flags', default=None,
438 help="Flags to call the 'strip' binutil tool with. "
439 "Overrides the default arguments.")
440 group.add_argument('--ping', action='store_true', default=False,
441 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700442 group.add_argument('--process-timeout', type=int,
443 default=KILL_PROC_MAX_WAIT,
444 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700445
Mike Frysingerc3061a62015-06-04 04:16:18 -0400446 group = parser.add_argument_group(
447 'Metadata Overrides (Advanced)',
448 description='Provide all of these overrides in order to remove '
449 'dependencies on metadata.json existence.')
450 group.add_argument('--target-tc', action='store', default=None,
451 help='Override target toolchain name, e.g. '
452 'x86_64-cros-linux-gnu')
453 group.add_argument('--toolchain-url', action='store', default=None,
454 help='Override toolchain url format pattern, e.g. '
455 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700456
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700457 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
458 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
459 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400460 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700461
462 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
463 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
464 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
465 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
466 help=argparse.SUPPRESS)
467
Ryan Cuia56a71e2012-10-18 18:40:35 -0700468 # Path of an empty directory to stage chrome artifacts to. Defaults to a
469 # temporary directory that is removed when the script finishes. If the path
470 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400471 parser.add_argument('--staging-dir', type='path', default=None,
472 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700473 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400474 parser.add_argument('--staging-only', action='store_true', default=False,
475 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700476 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
477 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
478 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400479 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900480 parser.add_argument('--compress', action='store', default='auto',
481 choices=('always', 'never', 'auto'),
482 help='Choose the data compression behavior. Default '
483 'is set to "auto", that disables compression if '
484 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700485 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700486
Ryan Cuie535b172012-10-19 18:25:03 -0700487
488def _ParseCommandLine(argv):
489 """Parse args, and run environment-independent checks."""
490 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400491 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700492
Ryan Cuia56a71e2012-10-18 18:40:35 -0700493 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
494 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800495 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700496 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
497 parser.error('Cannot specify both --build_dir and '
498 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800499 if options.build_dir and not options.board:
500 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700501 if options.gs_path and options.local_pkg_path:
502 parser.error('Cannot specify both --gs-path and --local-pkg-path')
503 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700504 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700505 if options.staging_flags and not options.build_dir:
506 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700507
Steven Bennetts46a84c32016-08-12 15:20:46 -0700508 if options.strict:
509 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700510 if options.gyp_defines:
511 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700512
513 if options.mount or options.mount_dir:
514 if not options.target_dir:
515 options.target_dir = _CHROME_DIR_MOUNT
516 else:
517 if not options.target_dir:
518 options.target_dir = _CHROME_DIR
519
520 if options.mount and not options.mount_dir:
521 options.mount_dir = _CHROME_DIR
522
Mike Frysingerc3061a62015-06-04 04:16:18 -0400523 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700524
525
Mike Frysingerc3061a62015-06-04 04:16:18 -0400526def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800527 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700528
529 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400530 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700531 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700532 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
533 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
534
Steven Bennetts368c3e52016-09-23 13:05:21 -0700535 if not options.gn_args:
536 gn_env = os.getenv('GN_ARGS')
537 if gn_env is not None:
538 options.gn_args = gn_helpers.FromGNArgs(gn_env)
539 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700540
Steven Bennetts60600462016-05-12 10:40:20 -0700541 if not options.staging_flags:
542 use_env = os.getenv('USE')
543 if use_env is not None:
544 options.staging_flags = ' '.join(set(use_env.split()).intersection(
545 chrome_util.STAGING_FLAGS))
546 logging.info('Staging flags taken from USE in environment: %s',
547 options.staging_flags)
548
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549
Ryan Cui504db722013-01-22 11:48:01 -0800550def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551 """Get the chrome prebuilt tarball from GS.
552
Mike Frysinger02e1e072013-11-10 22:11:34 -0500553 Returns:
554 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700555 """
David James9374aac2013-10-08 16:00:17 -0700556 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500557 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800558 files = [found for found in files if
559 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
560 if not files:
561 raise Exception('No chrome package found at %s' % gs_path)
562 elif len(files) > 1:
563 # - Users should provide us with a direct link to either a stripped or
564 # unstripped chrome package.
565 # - In the case of being provided with an archive directory, where both
566 # stripped and unstripped chrome available, use the stripped chrome
567 # package.
568 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
569 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
570 files = [f for f in files if not 'unstripped' in f]
571 assert len(files) == 1
572 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800573
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800574 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800575 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800576 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
577 chrome_path = os.path.join(tempdir, filename)
578 assert os.path.exists(chrome_path)
579 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700580
581
Ryan Cuif890a3e2013-03-07 18:57:06 -0800582@contextlib.contextmanager
583def _StripBinContext(options):
584 if not options.dostrip:
585 yield None
586 elif options.strip_bin:
587 yield options.strip_bin
588 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800589 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800590 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700591 with sdk.Prepare(components=components, target_tc=options.target_tc,
592 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800593 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
594 constants.CHROME_ENV_FILE)
595 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
596 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
597 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800598 yield strip_bin
599
600
Steve Funge984a532013-11-25 17:09:25 -0800601def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
602 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800603 """Place the necessary files in the staging directory.
604
605 The staging directory is the directory used to rsync the build artifacts over
606 to the device. Only the necessary Chrome build artifacts are put into the
607 staging directory.
608 """
Ryan Cui5866be02013-03-18 14:12:00 -0700609 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400610 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800611 if options.build_dir:
612 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700613 strip_flags = (None if options.strip_flags is None else
614 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800615 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700616 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700617 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700618 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800619 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700620 else:
621 pkg_path = options.local_pkg_path
622 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800623 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
624 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700625
626 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800627 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700628 # Extract only the ./opt/google/chrome contents, directly into the staging
629 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800630 if pkg_path[-4:] == '.zip':
631 cros_build_lib.DebugRunCommand(
632 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
633 staging_dir])
634 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
635 shutil.move(filename, staging_dir)
636 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
637 else:
638 cros_build_lib.DebugRunCommand(
639 ['tar', '--strip-components', '4', '--extract',
640 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
641 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700642
Ryan Cui71aa8de2013-04-19 16:12:55 -0700643
Ryan Cui3045c5d2012-07-13 18:00:33 -0700644def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400645 options = _ParseCommandLine(argv)
646 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700647
648 # Set cros_build_lib debug level to hide RunCommand spew.
649 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700650 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700651 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800652 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700653
Aviv Keshet01a82e92017-03-02 17:39:59 -0800654 with osutils.TempDir(set_global=True) as tempdir:
655 staging_dir = options.staging_dir
656 if not staging_dir:
657 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700658
Aviv Keshet01a82e92017-03-02 17:39:59 -0800659 deploy = DeployChrome(options, tempdir, staging_dir)
660 try:
661 deploy.Perform()
662 except failures_lib.StepFailure as ex:
663 raise SystemExit(str(ex).strip())
664 deploy.Cleanup()