blob: 9a6dd10b045e036bffeb0f03b86dd8877bbe3001 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
David Haddock3151d912017-10-24 03:50:32 +000053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
54LSOF_COMMAND = 'lsof %s/chrome'
55DBUS_RELOAD_COMMAND = 'killall -HUP dbus-daemon'
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
David Haddock3151d912017-10-24 03:50:32 +000063_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
64 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
65_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
66_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
67
68DF_COMMAND = 'df -k %s'
69
Mike Frysingere65f3752014-12-08 00:46:39 -050070
Ryan Cui3045c5d2012-07-13 18:00:33 -070071def _UrlBaseName(url):
72 """Return the last component of the URL."""
73 return url.rstrip('/').rpartition('/')[-1]
74
75
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070076class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080077 """Raised whenever the deploy fails."""
78
79
Ryan Cui7193a7e2013-04-26 14:15:19 -070080DeviceInfo = collections.namedtuple(
81 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
82
83
Ryan Cui3045c5d2012-07-13 18:00:33 -070084class DeployChrome(object):
85 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050086
Ryan Cuia56a71e2012-10-18 18:40:35 -070087 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070088 """Initialize the class.
89
Mike Frysinger02e1e072013-11-10 22:11:34 -050090 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040091 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070092 tempdir: Scratch space for the class. Caller has responsibility to clean
93 it up.
Steve Funge984a532013-11-25 17:09:25 -080094 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070095 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070096 self.tempdir = tempdir
97 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070098 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050099 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900100 self.device = remote.RemoteDevice(options.to, port=options.port,
Adrian Eldera2c548a2017-11-07 19:01:29 -0500101 ping=options.ping,
102 private_key=options.private_key)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500103 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000104
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500105 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800106 self.chrome_dir = _CHROME_DIR
107
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 def _GetRemoteMountFree(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000109 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800111 value = line.split()[3]
112 multipliers = {
113 'G': 1024 * 1024 * 1024,
114 'M': 1024 * 1024,
115 'K': 1024,
116 }
117 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700118
119 def _GetRemoteDirSize(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000120 result = self.device.RunCommand('du -ks %s' % remote_dir,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122 return int(result.output.split()[0])
123
124 def _GetStagingDirSize(self):
125 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800126 redirect_stdout=True,
127 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700128 return int(result.output.split()[0])
129
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 def _ChromeFileInUse(self):
David Haddock3151d912017-10-24 03:50:32 +0000131 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
132 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700133 return result.returncode == 0
134
Justin TerAvestfac210e2017-04-13 11:39:00 -0600135 def _Reboot(self):
136 # A reboot in developer mode takes a while (and has delays), so the user
137 # will have time to read and act on the USB boot instructions below.
138 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
139 self.device.Reboot()
140
Ryan Cui3045c5d2012-07-13 18:00:33 -0700141 def _DisableRootfsVerification(self):
142 if not self.options.force:
143 logging.error('Detected that the device has rootfs verification enabled.')
144 logging.info('This script can automatically remove the rootfs '
145 'verification, which requires that it reboot the device.')
146 logging.info('Make sure the device is in developer mode!')
147 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700148 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800149 # Since we stopped Chrome earlier, it's good form to start it up again.
150 if self.options.startui:
151 logging.info('Starting Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000152 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800153 raise DeployFailure('Need rootfs verification to be disabled. '
154 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700155
156 logging.info('Removing rootfs verification from %s', self.options.to)
157 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
158 # Use --force to bypass the checks.
David Haddock3151d912017-10-24 03:50:32 +0000159 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
160 '--remove_rootfs_verification --force')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700161 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
David Haddock3151d912017-10-24 03:50:32 +0000162 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700163
Justin TerAvestfac210e2017-04-13 11:39:00 -0600164 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700165
David James88e6f032013-03-02 08:13:20 -0800166 # Now that the machine has been rebooted, we need to kill Chrome again.
167 self._KillProcsIfNeeded()
168
169 # Make sure the rootfs is writable now.
170 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700171
172 def _CheckUiJobStarted(self):
173 # status output is in the format:
174 # <job_name> <status> ['process' <pid>].
175 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800176 try:
David Haddock3151d912017-10-24 03:50:32 +0000177 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800178 except cros_build_lib.RunCommandError as e:
179 if 'Unknown job' in e.result.error:
180 return False
181 else:
182 raise e
183
Ryan Cui3045c5d2012-07-13 18:00:33 -0700184 return result.output.split()[1].split('/')[0] == 'start'
185
186 def _KillProcsIfNeeded(self):
187 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800188 logging.info('Shutting down Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000189 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700190
191 # Developers sometimes run session_manager manually, in which case we'll
192 # need to help shut the chrome processes down.
193 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700194 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700195 while self._ChromeFileInUse():
196 logging.warning('The chrome binary on the device is in use.')
197 logging.warning('Killing chrome and session_manager processes...\n')
198
David Haddock3151d912017-10-24 03:50:32 +0000199 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500200 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700201 # Wait for processes to actually terminate
202 time.sleep(POST_KILL_WAIT)
203 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800204 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800205 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700206 'running chrome processes and try again.'
207 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800208 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
David James88e6f032013-03-02 08:13:20 -0800210 def _MountRootfsAsWritable(self, error_code_ok=True):
211 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Robert Flack1dc7ea82014-11-26 13:50:24 -0500213 If the command fails, and error_code_ok is True, and the target dir is not
214 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700215
Mike Frysinger02e1e072013-11-10 22:11:34 -0500216 Args:
David James88e6f032013-03-02 08:13:20 -0800217 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
218 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800219 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500220 result = self.device.RunCommand(MOUNT_RW_COMMAND,
221 error_code_ok=error_code_ok,
222 capture_output=True)
223 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400224 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500225 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700226
Ryan Cui7193a7e2013-04-26 14:15:19 -0700227 def _GetDeviceInfo(self):
228 steps = [
229 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
230 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
231 ]
232 return_values = parallel.RunParallelSteps(steps, return_values=True)
233 return DeviceInfo(*return_values)
234
235 def _CheckDeviceFreeSpace(self, device_info):
236 """See if target device has enough space for Chrome.
237
Mike Frysinger02e1e072013-11-10 22:11:34 -0500238 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700239 device_info: A DeviceInfo named tuple.
240 """
241 effective_free = device_info.target_dir_size + device_info.target_fs_free
242 staging_size = self._GetStagingDirSize()
243 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700244 raise DeployFailure(
245 'Not enough free space on the device. Required: %s MiB, '
246 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700247 if device_info.target_fs_free < (100 * 1024):
248 logging.warning('The device has less than 100MB free. deploy_chrome may '
249 'hang during the transfer.')
250
Satoru Takabayashif2893002017-06-20 14:52:48 +0900251 def _ShouldUseCompression(self):
252 """Checks if compression should be used for rsync."""
253 if self.options.compress == 'always':
254 return True
255 elif self.options.compress == 'never':
256 return False
257 elif self.options.compress == 'auto':
258 return not self.device.HasGigabitEthernet()
259
Ryan Cui3045c5d2012-07-13 18:00:33 -0700260 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800261 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700262 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
263 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900264 if not self.device.HasRsync():
265 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700266 'rsync is not found on the device.\n'
267 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500268 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
269 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700270 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900271 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700272 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500273 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800274
275 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800276 if p.mode:
277 # Set mode if necessary.
David Haddock3151d912017-10-24 03:50:32 +0000278 self.device.RunCommand('chmod %o %s/%s' % (
279 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800280
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700281 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
282 # pick up major changes (bus type, logging, etc.), but all we care about is
283 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
284 # be authorized to take ownership of its service names.
David Haddock3151d912017-10-24 03:50:32 +0000285 self.device.RunCommand(DBUS_RELOAD_COMMAND, error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600286
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800287 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800288 logging.info('Starting UI...')
David Haddock3151d912017-10-24 03:50:32 +0000289 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700290
David James88e6f032013-03-02 08:13:20 -0800291 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700292 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800293 logging.info('Testing connection to the device...')
David Haddock3151d912017-10-24 03:50:32 +0000294 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800295 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700296 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800297 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700298
Steve Funge984a532013-11-25 17:09:25 -0800299 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700300 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700301 def BinaryExists(filename):
302 """Checks if the passed-in file is present in the build directory."""
303 return os.path.exists(os.path.join(self.options.build_dir, filename))
304
Daniel Erat9813f0e2014-11-12 11:00:28 -0700305 # Handle non-Chrome deployments.
306 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700307 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700308 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
309
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700310 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700311 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700312 self.options.startui = False
313
David James88e6f032013-03-02 08:13:20 -0800314 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800315 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
316 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800317
Thiago Goncales12793312013-05-23 11:26:17 -0700318 def _MountTarget(self):
319 logging.info('Mounting Chrome...')
320
321 # Create directory if does not exist
David Haddock3151d912017-10-24 03:50:32 +0000322 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
323 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900324 # Umount the existing mount on mount_dir if present first
David Haddock3151d912017-10-24 03:50:32 +0000325 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
326 {'dir': self.options.mount_dir})
327 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
328 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700329 # Chrome needs partition to have exec and suid flags set
David Haddock3151d912017-10-24 03:50:32 +0000330 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
Robert Flack1dc7ea82014-11-26 13:50:24 -0500331
332 def Cleanup(self):
333 """Clean up RemoteDevice."""
334 if not self.options.staging_only:
335 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700336
David James88e6f032013-03-02 08:13:20 -0800337 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800338 self._CheckDeployType()
339
David James88e6f032013-03-02 08:13:20 -0800340 # If requested, just do the staging step.
341 if self.options.staging_only:
342 self._PrepareStagingDir()
343 return 0
344
345 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800346 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800347 steps = [self._GetDeviceInfo, self._CheckConnection,
348 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
349 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700350 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
351 return_values=True)
352 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800353
Robert Flack1dc7ea82014-11-26 13:50:24 -0500354 # If we're trying to deploy to a dir which is not writable and we failed
355 # to mark the rootfs as writable, try disabling rootfs verification.
356 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800357 self._DisableRootfsVerification()
358
Thiago Goncales12793312013-05-23 11:26:17 -0700359 if self.options.mount_dir is not None:
360 self._MountTarget()
361
David James88e6f032013-03-02 08:13:20 -0800362 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700363 self._Deploy()
364
365
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700366def ValidateStagingFlags(value):
367 """Convert formatted string to dictionary."""
368 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700369
370
Steven Bennetts368c3e52016-09-23 13:05:21 -0700371def ValidateGnArgs(value):
372 """Convert GN_ARGS-formatted string to dictionary."""
373 return gn_helpers.FromGNArgs(value)
374
375
Ryan Cuie535b172012-10-19 18:25:03 -0700376def _CreateParser():
377 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400378 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700379
Ryan Cuia56a71e2012-10-18 18:40:35 -0700380 # TODO(rcui): Have this use the UI-V2 format of having source and target
381 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400382 parser.add_argument('--force', action='store_true', default=False,
383 help='Skip all prompts (i.e., for disabling of rootfs '
384 'verification). This may result in the target '
385 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800386 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400387 parser.add_argument('--board', default=sdk_board_env,
388 help="The board the Chrome build is targeted for. When "
389 "in a 'cros chrome-sdk' shell, defaults to the SDK "
390 "board.")
391 parser.add_argument('--build-dir', type='path',
392 help='The directory with Chrome build artifacts to '
393 'deploy from. Typically of format '
394 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700395 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400396 parser.add_argument('--target-dir', type='path',
397 default=None,
398 help='Target directory on device to deploy Chrome into.')
399 parser.add_argument('-g', '--gs-path', type='gs_path',
400 help='GS path that contains the chrome to deploy.')
Adrian Eldera2c548a2017-11-07 19:01:29 -0500401 parser.add_argument('--private-key', type='path', default=None,
402 help='An ssh private key to use when deploying to '
403 'a CrOS device.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400404 parser.add_argument('--nostartui', action='store_false', dest='startui',
405 default=True,
406 help="Don't restart the ui daemon after deployment.")
407 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
408 default=True,
409 help="Don't strip binaries during deployment. Warning: "
410 'the resulting binaries will be very large!')
411 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
412 help='Port of the target device to connect to.')
413 parser.add_argument('-t', '--to',
414 help='The IP address of the CrOS device to deploy to.')
415 parser.add_argument('-v', '--verbose', action='store_true', default=False,
416 help='Show more debug output.')
417 parser.add_argument('--mount-dir', type='path', default=None,
418 help='Deploy Chrome in target directory and bind it '
419 'to the directory specified by this flag.'
420 'Any existing mount on this directory will be '
421 'umounted first.')
422 parser.add_argument('--mount', action='store_true', default=False,
423 help='Deploy Chrome to default target directory and bind '
424 'it to the default mount directory.'
425 'Any existing mount on this directory will be '
426 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700427
Mike Frysingerc3061a62015-06-04 04:16:18 -0400428 group = parser.add_argument_group('Advanced Options')
429 group.add_argument('-l', '--local-pkg-path', type='path',
430 help='Path to local chrome prebuilt package to deploy.')
431 group.add_argument('--sloppy', action='store_true', default=False,
432 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700433 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400434 help=('Extra flags to control staging. Valid flags are - '
435 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700436 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400437 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700438 help='Deprecated. Default behavior is "strict". Use '
439 '--sloppy to omit warnings for missing optional '
440 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400441 group.add_argument('--strip-flags', default=None,
442 help="Flags to call the 'strip' binutil tool with. "
443 "Overrides the default arguments.")
444 group.add_argument('--ping', action='store_true', default=False,
445 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700446 group.add_argument('--process-timeout', type=int,
447 default=KILL_PROC_MAX_WAIT,
448 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700449
Mike Frysingerc3061a62015-06-04 04:16:18 -0400450 group = parser.add_argument_group(
451 'Metadata Overrides (Advanced)',
452 description='Provide all of these overrides in order to remove '
453 'dependencies on metadata.json existence.')
454 group.add_argument('--target-tc', action='store', default=None,
455 help='Override target toolchain name, e.g. '
456 'x86_64-cros-linux-gnu')
457 group.add_argument('--toolchain-url', action='store', default=None,
458 help='Override toolchain url format pattern, e.g. '
459 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700460
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700461 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
462 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
463 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400464 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700465
466 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
467 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
468 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
469 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
470 help=argparse.SUPPRESS)
471
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472 # Path of an empty directory to stage chrome artifacts to. Defaults to a
473 # temporary directory that is removed when the script finishes. If the path
474 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400475 parser.add_argument('--staging-dir', type='path', default=None,
476 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700477 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400478 parser.add_argument('--staging-only', action='store_true', default=False,
479 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700480 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
481 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
482 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400483 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900484 parser.add_argument('--compress', action='store', default='auto',
485 choices=('always', 'never', 'auto'),
486 help='Choose the data compression behavior. Default '
487 'is set to "auto", that disables compression if '
488 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700489 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700490
Ryan Cuie535b172012-10-19 18:25:03 -0700491
492def _ParseCommandLine(argv):
493 """Parse args, and run environment-independent checks."""
494 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400495 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700496
Ryan Cuia56a71e2012-10-18 18:40:35 -0700497 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
498 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800499 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700500 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
501 parser.error('Cannot specify both --build_dir and '
502 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800503 if options.build_dir and not options.board:
504 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700505 if options.gs_path and options.local_pkg_path:
506 parser.error('Cannot specify both --gs-path and --local-pkg-path')
507 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700508 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700509 if options.staging_flags and not options.build_dir:
510 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700511
Steven Bennetts46a84c32016-08-12 15:20:46 -0700512 if options.strict:
513 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700514 if options.gyp_defines:
515 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700516
517 if options.mount or options.mount_dir:
518 if not options.target_dir:
519 options.target_dir = _CHROME_DIR_MOUNT
520 else:
521 if not options.target_dir:
522 options.target_dir = _CHROME_DIR
523
524 if options.mount and not options.mount_dir:
525 options.mount_dir = _CHROME_DIR
526
Mike Frysingerc3061a62015-06-04 04:16:18 -0400527 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700528
529
Mike Frysingerc3061a62015-06-04 04:16:18 -0400530def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800531 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700532
533 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400534 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700535 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700536 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
537 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
538
Steven Bennetts368c3e52016-09-23 13:05:21 -0700539 if not options.gn_args:
540 gn_env = os.getenv('GN_ARGS')
541 if gn_env is not None:
542 options.gn_args = gn_helpers.FromGNArgs(gn_env)
543 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700544
Steven Bennetts60600462016-05-12 10:40:20 -0700545 if not options.staging_flags:
546 use_env = os.getenv('USE')
547 if use_env is not None:
548 options.staging_flags = ' '.join(set(use_env.split()).intersection(
549 chrome_util.STAGING_FLAGS))
550 logging.info('Staging flags taken from USE in environment: %s',
551 options.staging_flags)
552
Ryan Cuia56a71e2012-10-18 18:40:35 -0700553
Ryan Cui504db722013-01-22 11:48:01 -0800554def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700555 """Get the chrome prebuilt tarball from GS.
556
Mike Frysinger02e1e072013-11-10 22:11:34 -0500557 Returns:
558 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700559 """
David James9374aac2013-10-08 16:00:17 -0700560 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500561 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800562 files = [found for found in files if
563 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
564 if not files:
565 raise Exception('No chrome package found at %s' % gs_path)
566 elif len(files) > 1:
567 # - Users should provide us with a direct link to either a stripped or
568 # unstripped chrome package.
569 # - In the case of being provided with an archive directory, where both
570 # stripped and unstripped chrome available, use the stripped chrome
571 # package.
572 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
573 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
574 files = [f for f in files if not 'unstripped' in f]
575 assert len(files) == 1
576 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800577
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800578 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800579 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800580 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
581 chrome_path = os.path.join(tempdir, filename)
582 assert os.path.exists(chrome_path)
583 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700584
585
Ryan Cuif890a3e2013-03-07 18:57:06 -0800586@contextlib.contextmanager
587def _StripBinContext(options):
588 if not options.dostrip:
589 yield None
590 elif options.strip_bin:
591 yield options.strip_bin
592 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800593 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800594 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700595 with sdk.Prepare(components=components, target_tc=options.target_tc,
596 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800597 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
598 constants.CHROME_ENV_FILE)
599 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
600 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
601 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800602 yield strip_bin
603
604
Steve Funge984a532013-11-25 17:09:25 -0800605def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
606 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800607 """Place the necessary files in the staging directory.
608
609 The staging directory is the directory used to rsync the build artifacts over
610 to the device. Only the necessary Chrome build artifacts are put into the
611 staging directory.
612 """
Ryan Cui5866be02013-03-18 14:12:00 -0700613 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400614 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800615 if options.build_dir:
616 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700617 strip_flags = (None if options.strip_flags is None else
618 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800619 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700620 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700621 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700622 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800623 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700624 else:
625 pkg_path = options.local_pkg_path
626 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800627 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
628 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700629
630 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800631 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700632 # Extract only the ./opt/google/chrome contents, directly into the staging
633 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800634 if pkg_path[-4:] == '.zip':
635 cros_build_lib.DebugRunCommand(
636 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
637 staging_dir])
638 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
639 shutil.move(filename, staging_dir)
640 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
641 else:
642 cros_build_lib.DebugRunCommand(
643 ['tar', '--strip-components', '4', '--extract',
644 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
645 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700646
Ryan Cui71aa8de2013-04-19 16:12:55 -0700647
Ryan Cui3045c5d2012-07-13 18:00:33 -0700648def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400649 options = _ParseCommandLine(argv)
650 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700651
652 # Set cros_build_lib debug level to hide RunCommand spew.
653 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700654 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700655 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800656 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700657
Aviv Keshet01a82e92017-03-02 17:39:59 -0800658 with osutils.TempDir(set_global=True) as tempdir:
659 staging_dir = options.staging_dir
660 if not staging_dir:
661 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700662
Aviv Keshet01a82e92017-03-02 17:39:59 -0800663 deploy = DeployChrome(options, tempdir, staging_dir)
664 try:
665 deploy.Perform()
666 except failures_lib.StepFailure as ex:
667 raise SystemExit(str(ex).strip())
668 deploy.Cleanup()