blob: 8bec7de8d49295e0770ed596b811e017f8104441 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
David Haddock3151d912017-10-24 03:50:32 +000053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
54LSOF_COMMAND = 'lsof %s/chrome'
55DBUS_RELOAD_COMMAND = 'killall -HUP dbus-daemon'
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
David Haddock3151d912017-10-24 03:50:32 +000063_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
64 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
65_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
66_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
67
68DF_COMMAND = 'df -k %s'
69
Mike Frysingere65f3752014-12-08 00:46:39 -050070
Ryan Cui3045c5d2012-07-13 18:00:33 -070071def _UrlBaseName(url):
72 """Return the last component of the URL."""
73 return url.rstrip('/').rpartition('/')[-1]
74
75
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070076class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080077 """Raised whenever the deploy fails."""
78
79
Ryan Cui7193a7e2013-04-26 14:15:19 -070080DeviceInfo = collections.namedtuple(
81 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
82
83
Ryan Cui3045c5d2012-07-13 18:00:33 -070084class DeployChrome(object):
85 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050086
Ryan Cuia56a71e2012-10-18 18:40:35 -070087 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070088 """Initialize the class.
89
Mike Frysinger02e1e072013-11-10 22:11:34 -050090 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040091 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070092 tempdir: Scratch space for the class. Caller has responsibility to clean
93 it up.
Steve Funge984a532013-11-25 17:09:25 -080094 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070095 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070096 self.tempdir = tempdir
97 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070098 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050099 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900100 self.device = remote.RemoteDevice(options.to, port=options.port,
Adrian Eldera2c548a2017-11-07 19:01:29 -0500101 ping=options.ping,
102 private_key=options.private_key)
Steven Bennettsca73efa2018-07-10 13:36:56 -0700103 self._root_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000104
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500105 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800106 self.chrome_dir = _CHROME_DIR
107
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 def _GetRemoteMountFree(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000109 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800111 value = line.split()[3]
112 multipliers = {
113 'G': 1024 * 1024 * 1024,
114 'M': 1024 * 1024,
115 'K': 1024,
116 }
117 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700118
119 def _GetRemoteDirSize(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000120 result = self.device.RunCommand('du -ks %s' % remote_dir,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122 return int(result.output.split()[0])
123
124 def _GetStagingDirSize(self):
125 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800126 redirect_stdout=True,
127 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700128 return int(result.output.split()[0])
129
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 def _ChromeFileInUse(self):
David Haddock3151d912017-10-24 03:50:32 +0000131 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
132 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700133 return result.returncode == 0
134
Justin TerAvestfac210e2017-04-13 11:39:00 -0600135 def _Reboot(self):
136 # A reboot in developer mode takes a while (and has delays), so the user
137 # will have time to read and act on the USB boot instructions below.
138 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
139 self.device.Reboot()
140
Ryan Cui3045c5d2012-07-13 18:00:33 -0700141 def _DisableRootfsVerification(self):
142 if not self.options.force:
143 logging.error('Detected that the device has rootfs verification enabled.')
144 logging.info('This script can automatically remove the rootfs '
145 'verification, which requires that it reboot the device.')
146 logging.info('Make sure the device is in developer mode!')
147 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700148 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
Steven Bennettsca73efa2018-07-10 13:36:56 -0700149 return False
Ryan Cui3045c5d2012-07-13 18:00:33 -0700150
151 logging.info('Removing rootfs verification from %s', self.options.to)
152 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
153 # Use --force to bypass the checks.
David Haddock3151d912017-10-24 03:50:32 +0000154 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
155 '--remove_rootfs_verification --force')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
David Haddock3151d912017-10-24 03:50:32 +0000157 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
Justin TerAvestfac210e2017-04-13 11:39:00 -0600159 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700160
David James88e6f032013-03-02 08:13:20 -0800161 # Now that the machine has been rebooted, we need to kill Chrome again.
162 self._KillProcsIfNeeded()
163
164 # Make sure the rootfs is writable now.
165 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700166
Steven Bennettsca73efa2018-07-10 13:36:56 -0700167 return True
168
Ryan Cui3045c5d2012-07-13 18:00:33 -0700169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
David Haddock3151d912017-10-24 03:50:32 +0000174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700191 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
David Haddock3151d912017-10-24 03:50:32 +0000196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700203 'running chrome processes and try again.'
204 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800208 """Mounts the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Steven Bennettsca73efa2018-07-10 13:36:56 -0700210 If the command fails and the root dir is not writable then this function
211 sets self._root_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
Steven Bennettsca73efa2018-07-10 13:36:56 -0700220 if result.returncode and not self.device.IsDirWritable('/'):
221 self._root_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800223 def _EnsureTargetDir(self):
224 """Ensures that the target directory exists on the remote device."""
225 target_dir = self.options.target_dir
226 # Any valid /opt directory should already exist so avoid the remote call.
227 if os.path.commonprefix([target_dir, '/opt']) == '/opt':
228 return
229 self.device.RunCommand(['mkdir', '-p', '--mode', '0775', target_dir])
230
Ryan Cui7193a7e2013-04-26 14:15:19 -0700231 def _GetDeviceInfo(self):
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800232 """Returns the disk space used and available for the target diectory."""
Ryan Cui7193a7e2013-04-26 14:15:19 -0700233 steps = [
234 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
235 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
236 ]
237 return_values = parallel.RunParallelSteps(steps, return_values=True)
238 return DeviceInfo(*return_values)
239
240 def _CheckDeviceFreeSpace(self, device_info):
241 """See if target device has enough space for Chrome.
242
Mike Frysinger02e1e072013-11-10 22:11:34 -0500243 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 device_info: A DeviceInfo named tuple.
245 """
246 effective_free = device_info.target_dir_size + device_info.target_fs_free
247 staging_size = self._GetStagingDirSize()
248 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700249 raise DeployFailure(
250 'Not enough free space on the device. Required: %s MiB, '
251 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700252 if device_info.target_fs_free < (100 * 1024):
253 logging.warning('The device has less than 100MB free. deploy_chrome may '
254 'hang during the transfer.')
255
Satoru Takabayashif2893002017-06-20 14:52:48 +0900256 def _ShouldUseCompression(self):
257 """Checks if compression should be used for rsync."""
258 if self.options.compress == 'always':
259 return True
260 elif self.options.compress == 'never':
261 return False
262 elif self.options.compress == 'auto':
263 return not self.device.HasGigabitEthernet()
264
Ryan Cui3045c5d2012-07-13 18:00:33 -0700265 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800266 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700267 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
268 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900269 if not self.device.HasRsync():
270 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700271 'rsync is not found on the device.\n'
272 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500273 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
274 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700275 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900276 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700277 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500278 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800279
280 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800281 if p.mode:
282 # Set mode if necessary.
David Haddock3151d912017-10-24 03:50:32 +0000283 self.device.RunCommand('chmod %o %s/%s' % (
284 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800285
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700286 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
287 # pick up major changes (bus type, logging, etc.), but all we care about is
288 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
289 # be authorized to take ownership of its service names.
David Haddock3151d912017-10-24 03:50:32 +0000290 self.device.RunCommand(DBUS_RELOAD_COMMAND, error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600291
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800292 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800293 logging.info('Starting UI...')
David Haddock3151d912017-10-24 03:50:32 +0000294 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700295
David James88e6f032013-03-02 08:13:20 -0800296 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700297 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800298 logging.info('Testing connection to the device...')
David Haddock3151d912017-10-24 03:50:32 +0000299 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800300 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700301 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800302 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700303
Steve Funge984a532013-11-25 17:09:25 -0800304 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700305 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700306 def BinaryExists(filename):
307 """Checks if the passed-in file is present in the build directory."""
308 return os.path.exists(os.path.join(self.options.build_dir, filename))
309
Daniel Erat9813f0e2014-11-12 11:00:28 -0700310 # Handle non-Chrome deployments.
311 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700312 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700313 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
314
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700315 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700316 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700317 self.options.startui = False
318
David James88e6f032013-03-02 08:13:20 -0800319 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800320 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
321 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800322
Thiago Goncales12793312013-05-23 11:26:17 -0700323 def _MountTarget(self):
324 logging.info('Mounting Chrome...')
325
326 # Create directory if does not exist
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800327 self.device.RunCommand(['mkdir', '-p', '--mode', '0775',
328 self.options.mount_dir])
Pawel Osciakfb200f52014-12-21 13:42:05 +0900329 # Umount the existing mount on mount_dir if present first
David Haddock3151d912017-10-24 03:50:32 +0000330 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
331 {'dir': self.options.mount_dir})
332 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
333 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700334 # Chrome needs partition to have exec and suid flags set
David Haddock3151d912017-10-24 03:50:32 +0000335 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
Robert Flack1dc7ea82014-11-26 13:50:24 -0500336
337 def Cleanup(self):
338 """Clean up RemoteDevice."""
339 if not self.options.staging_only:
340 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700341
David James88e6f032013-03-02 08:13:20 -0800342 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800343 self._CheckDeployType()
344
David James88e6f032013-03-02 08:13:20 -0800345 # If requested, just do the staging step.
346 if self.options.staging_only:
347 self._PrepareStagingDir()
348 return 0
349
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800350 # Ensure that the target directory exists before running parallel steps.
351 self._EnsureTargetDir()
352
David James88e6f032013-03-02 08:13:20 -0800353 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800354 # stop printing output at that point, and halt any running steps.
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800355 logging.info('Preparing device')
Steve Funge984a532013-11-25 17:09:25 -0800356 steps = [self._GetDeviceInfo, self._CheckConnection,
357 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
358 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700359 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
360 return_values=True)
361 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800362
Steven Bennettsca73efa2018-07-10 13:36:56 -0700363 # If the root dir is not writable, try disabling rootfs verification.
364 # (We always do this by default so that developers can write to
365 # /etc/chriome_dev.conf and other directories in the rootfs).
366 if self._root_dir_is_still_readonly.is_set():
367 if self.options.noremove_rootfs_verification:
368 logging.warning('Skipping disable rootfs verification.')
369 elif not self._DisableRootfsVerification():
370 logging.warning('Failed to disable rootfs verification.')
371
372 # If the target dir is still not writable (i.e. the user opted out or the
373 # command failed), abort.
374 if not self.device.IsDirWritable(self.options.target_dir):
375 if self.options.startui:
376 logging.info('Restarting Chrome...')
377 self.device.RunCommand('start ui')
378 raise DeployFailure('Target location is not writable. Aborting.')
David James88e6f032013-03-02 08:13:20 -0800379
Thiago Goncales12793312013-05-23 11:26:17 -0700380 if self.options.mount_dir is not None:
381 self._MountTarget()
382
David James88e6f032013-03-02 08:13:20 -0800383 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700384 self._Deploy()
385
386
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700387def ValidateStagingFlags(value):
388 """Convert formatted string to dictionary."""
389 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700390
391
Steven Bennetts368c3e52016-09-23 13:05:21 -0700392def ValidateGnArgs(value):
393 """Convert GN_ARGS-formatted string to dictionary."""
394 return gn_helpers.FromGNArgs(value)
395
396
Ryan Cuie535b172012-10-19 18:25:03 -0700397def _CreateParser():
398 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400399 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700400
Ryan Cuia56a71e2012-10-18 18:40:35 -0700401 # TODO(rcui): Have this use the UI-V2 format of having source and target
402 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400403 parser.add_argument('--force', action='store_true', default=False,
404 help='Skip all prompts (i.e., for disabling of rootfs '
405 'verification). This may result in the target '
406 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800407 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400408 parser.add_argument('--board', default=sdk_board_env,
409 help="The board the Chrome build is targeted for. When "
410 "in a 'cros chrome-sdk' shell, defaults to the SDK "
411 "board.")
412 parser.add_argument('--build-dir', type='path',
413 help='The directory with Chrome build artifacts to '
414 'deploy from. Typically of format '
415 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700416 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400417 parser.add_argument('--target-dir', type='path',
418 default=None,
419 help='Target directory on device to deploy Chrome into.')
420 parser.add_argument('-g', '--gs-path', type='gs_path',
421 help='GS path that contains the chrome to deploy.')
Adrian Eldera2c548a2017-11-07 19:01:29 -0500422 parser.add_argument('--private-key', type='path', default=None,
423 help='An ssh private key to use when deploying to '
424 'a CrOS device.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400425 parser.add_argument('--nostartui', action='store_false', dest='startui',
426 default=True,
427 help="Don't restart the ui daemon after deployment.")
428 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
429 default=True,
430 help="Don't strip binaries during deployment. Warning: "
431 'the resulting binaries will be very large!')
432 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
433 help='Port of the target device to connect to.')
434 parser.add_argument('-t', '--to',
435 help='The IP address of the CrOS device to deploy to.')
436 parser.add_argument('-v', '--verbose', action='store_true', default=False,
437 help='Show more debug output.')
438 parser.add_argument('--mount-dir', type='path', default=None,
439 help='Deploy Chrome in target directory and bind it '
440 'to the directory specified by this flag.'
441 'Any existing mount on this directory will be '
442 'umounted first.')
443 parser.add_argument('--mount', action='store_true', default=False,
444 help='Deploy Chrome to default target directory and bind '
445 'it to the default mount directory.'
446 'Any existing mount on this directory will be '
447 'umounted first.')
Steven Bennettsca73efa2018-07-10 13:36:56 -0700448 parser.add_argument('--noremove-rootfs-verification', action='store_true',
449 default=False, help='Never remove rootfs verification.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700450
Mike Frysingerc3061a62015-06-04 04:16:18 -0400451 group = parser.add_argument_group('Advanced Options')
452 group.add_argument('-l', '--local-pkg-path', type='path',
453 help='Path to local chrome prebuilt package to deploy.')
454 group.add_argument('--sloppy', action='store_true', default=False,
455 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700456 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400457 help=('Extra flags to control staging. Valid flags are - '
458 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700459 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400460 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700461 help='Deprecated. Default behavior is "strict". Use '
462 '--sloppy to omit warnings for missing optional '
463 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400464 group.add_argument('--strip-flags', default=None,
465 help="Flags to call the 'strip' binutil tool with. "
466 "Overrides the default arguments.")
467 group.add_argument('--ping', action='store_true', default=False,
468 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700469 group.add_argument('--process-timeout', type=int,
470 default=KILL_PROC_MAX_WAIT,
471 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472
Mike Frysingerc3061a62015-06-04 04:16:18 -0400473 group = parser.add_argument_group(
474 'Metadata Overrides (Advanced)',
475 description='Provide all of these overrides in order to remove '
476 'dependencies on metadata.json existence.')
477 group.add_argument('--target-tc', action='store', default=None,
478 help='Override target toolchain name, e.g. '
479 'x86_64-cros-linux-gnu')
480 group.add_argument('--toolchain-url', action='store', default=None,
481 help='Override toolchain url format pattern, e.g. '
482 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700483
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700484 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
485 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
486 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400487 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700488
489 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
490 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
491 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
492 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
493 help=argparse.SUPPRESS)
494
Ryan Cuia56a71e2012-10-18 18:40:35 -0700495 # Path of an empty directory to stage chrome artifacts to. Defaults to a
496 # temporary directory that is removed when the script finishes. If the path
497 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400498 parser.add_argument('--staging-dir', type='path', default=None,
499 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700500 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400501 parser.add_argument('--staging-only', action='store_true', default=False,
502 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700503 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
504 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
505 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400506 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900507 parser.add_argument('--compress', action='store', default='auto',
508 choices=('always', 'never', 'auto'),
509 help='Choose the data compression behavior. Default '
510 'is set to "auto", that disables compression if '
511 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700512 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700513
Ryan Cuie535b172012-10-19 18:25:03 -0700514
515def _ParseCommandLine(argv):
516 """Parse args, and run environment-independent checks."""
517 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400518 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700519
Ryan Cuia56a71e2012-10-18 18:40:35 -0700520 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
521 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800522 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700523 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
524 parser.error('Cannot specify both --build_dir and '
525 '--gs-path/--local-pkg-patch')
Achuith Bhandarkar31a3eb02018-03-22 16:33:48 -0700526 if (not options.board and options.build_dir and options.dostrip and
527 not options.strip_bin):
528 parser.error('--board is required for stripping.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700529 if options.gs_path and options.local_pkg_path:
530 parser.error('Cannot specify both --gs-path and --local-pkg-path')
531 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700532 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700533 if options.staging_flags and not options.build_dir:
534 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700535
Steven Bennetts46a84c32016-08-12 15:20:46 -0700536 if options.strict:
537 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700538 if options.gyp_defines:
539 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700540
541 if options.mount or options.mount_dir:
542 if not options.target_dir:
543 options.target_dir = _CHROME_DIR_MOUNT
544 else:
545 if not options.target_dir:
546 options.target_dir = _CHROME_DIR
547
548 if options.mount and not options.mount_dir:
549 options.mount_dir = _CHROME_DIR
550
Mike Frysingerc3061a62015-06-04 04:16:18 -0400551 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700552
553
Mike Frysingerc3061a62015-06-04 04:16:18 -0400554def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800555 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700556
557 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400558 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700559 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700560 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
561 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
562
Steven Bennetts368c3e52016-09-23 13:05:21 -0700563 if not options.gn_args:
564 gn_env = os.getenv('GN_ARGS')
565 if gn_env is not None:
566 options.gn_args = gn_helpers.FromGNArgs(gn_env)
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800567 logging.debug('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700568
Steven Bennetts60600462016-05-12 10:40:20 -0700569 if not options.staging_flags:
570 use_env = os.getenv('USE')
571 if use_env is not None:
572 options.staging_flags = ' '.join(set(use_env.split()).intersection(
573 chrome_util.STAGING_FLAGS))
574 logging.info('Staging flags taken from USE in environment: %s',
575 options.staging_flags)
576
Ryan Cuia56a71e2012-10-18 18:40:35 -0700577
Ryan Cui504db722013-01-22 11:48:01 -0800578def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700579 """Get the chrome prebuilt tarball from GS.
580
Mike Frysinger02e1e072013-11-10 22:11:34 -0500581 Returns:
582 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700583 """
David James9374aac2013-10-08 16:00:17 -0700584 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500585 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800586 files = [found for found in files if
587 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
588 if not files:
589 raise Exception('No chrome package found at %s' % gs_path)
590 elif len(files) > 1:
591 # - Users should provide us with a direct link to either a stripped or
592 # unstripped chrome package.
593 # - In the case of being provided with an archive directory, where both
594 # stripped and unstripped chrome available, use the stripped chrome
595 # package.
596 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
597 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
598 files = [f for f in files if not 'unstripped' in f]
599 assert len(files) == 1
600 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800601
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800602 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800603 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800604 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
605 chrome_path = os.path.join(tempdir, filename)
606 assert os.path.exists(chrome_path)
607 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700608
609
Ryan Cuif890a3e2013-03-07 18:57:06 -0800610@contextlib.contextmanager
611def _StripBinContext(options):
612 if not options.dostrip:
613 yield None
614 elif options.strip_bin:
615 yield options.strip_bin
616 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800617 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800618 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700619 with sdk.Prepare(components=components, target_tc=options.target_tc,
620 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800621 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
622 constants.CHROME_ENV_FILE)
623 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
624 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
625 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800626 yield strip_bin
627
628
Steve Funge984a532013-11-25 17:09:25 -0800629def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
630 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800631 """Place the necessary files in the staging directory.
632
633 The staging directory is the directory used to rsync the build artifacts over
634 to the device. Only the necessary Chrome build artifacts are put into the
635 staging directory.
636 """
Ryan Cui5866be02013-03-18 14:12:00 -0700637 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400638 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800639 if options.build_dir:
640 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700641 strip_flags = (None if options.strip_flags is None else
642 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800643 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700644 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700645 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700646 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800647 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700648 else:
649 pkg_path = options.local_pkg_path
650 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800651 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
652 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700653
654 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800655 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700656 # Extract only the ./opt/google/chrome contents, directly into the staging
657 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800658 if pkg_path[-4:] == '.zip':
659 cros_build_lib.DebugRunCommand(
660 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
661 staging_dir])
662 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
663 shutil.move(filename, staging_dir)
664 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
665 else:
666 cros_build_lib.DebugRunCommand(
667 ['tar', '--strip-components', '4', '--extract',
668 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
669 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700670
Ryan Cui71aa8de2013-04-19 16:12:55 -0700671
Ryan Cui3045c5d2012-07-13 18:00:33 -0700672def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400673 options = _ParseCommandLine(argv)
674 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700675
676 # Set cros_build_lib debug level to hide RunCommand spew.
677 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700678 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700679 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800680 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700681
Aviv Keshet01a82e92017-03-02 17:39:59 -0800682 with osutils.TempDir(set_global=True) as tempdir:
683 staging_dir = options.staging_dir
684 if not staging_dir:
685 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700686
Aviv Keshet01a82e92017-03-02 17:39:59 -0800687 deploy = DeployChrome(options, tempdir, staging_dir)
688 try:
689 deploy.Perform()
690 except failures_lib.StepFailure as ex:
691 raise SystemExit(str(ex).strip())
692 deploy.Cleanup()