blob: b67a33136722234a47db1e37d7d7beb362d39411 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
David Haddock3151d912017-10-24 03:50:32 +000053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
54LSOF_COMMAND = 'lsof %s/chrome'
55DBUS_RELOAD_COMMAND = 'killall -HUP dbus-daemon'
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
David Haddock3151d912017-10-24 03:50:32 +000063_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
64 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
65_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
66_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
67
68DF_COMMAND = 'df -k %s'
69
Mike Frysingere65f3752014-12-08 00:46:39 -050070
Ryan Cui3045c5d2012-07-13 18:00:33 -070071def _UrlBaseName(url):
72 """Return the last component of the URL."""
73 return url.rstrip('/').rpartition('/')[-1]
74
75
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070076class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080077 """Raised whenever the deploy fails."""
78
79
Ryan Cui7193a7e2013-04-26 14:15:19 -070080DeviceInfo = collections.namedtuple(
81 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
82
83
Ryan Cui3045c5d2012-07-13 18:00:33 -070084class DeployChrome(object):
85 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050086
Ryan Cuia56a71e2012-10-18 18:40:35 -070087 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070088 """Initialize the class.
89
Mike Frysinger02e1e072013-11-10 22:11:34 -050090 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040091 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070092 tempdir: Scratch space for the class. Caller has responsibility to clean
93 it up.
Steve Funge984a532013-11-25 17:09:25 -080094 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070095 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070096 self.tempdir = tempdir
97 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070098 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050099 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900100 self.device = remote.RemoteDevice(options.to, port=options.port,
Adrian Eldera2c548a2017-11-07 19:01:29 -0500101 ping=options.ping,
102 private_key=options.private_key)
Steven Bennettsca73efa2018-07-10 13:36:56 -0700103 self._root_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +0000104
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -0500105 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800106 self.chrome_dir = _CHROME_DIR
107
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 def _GetRemoteMountFree(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000109 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800111 value = line.split()[3]
112 multipliers = {
113 'G': 1024 * 1024 * 1024,
114 'M': 1024 * 1024,
115 'K': 1024,
116 }
117 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700118
119 def _GetRemoteDirSize(self, remote_dir):
David Haddock3151d912017-10-24 03:50:32 +0000120 result = self.device.RunCommand('du -ks %s' % remote_dir,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122 return int(result.output.split()[0])
123
124 def _GetStagingDirSize(self):
Mike Frysinger45602c72019-09-22 02:15:11 -0400125 result = cros_build_lib.dbg_run(['du', '-ks', self.staging_dir],
126 redirect_stdout=True, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700127 return int(result.output.split()[0])
128
Ryan Cui3045c5d2012-07-13 18:00:33 -0700129 def _ChromeFileInUse(self):
David Haddock3151d912017-10-24 03:50:32 +0000130 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
131 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700132 return result.returncode == 0
133
Justin TerAvestfac210e2017-04-13 11:39:00 -0600134 def _Reboot(self):
135 # A reboot in developer mode takes a while (and has delays), so the user
136 # will have time to read and act on the USB boot instructions below.
137 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
138 self.device.Reboot()
139
Ryan Cui3045c5d2012-07-13 18:00:33 -0700140 def _DisableRootfsVerification(self):
141 if not self.options.force:
142 logging.error('Detected that the device has rootfs verification enabled.')
143 logging.info('This script can automatically remove the rootfs '
144 'verification, which requires that it reboot the device.')
145 logging.info('Make sure the device is in developer mode!')
146 logging.info('Skip this prompt by specifying --force.')
Ben Chan37d64e52018-10-05 15:35:02 -0700147 if not cros_build_lib.BooleanPrompt('Remove rootfs verification?', False):
Steven Bennettsca73efa2018-07-10 13:36:56 -0700148 return False
Ryan Cui3045c5d2012-07-13 18:00:33 -0700149
150 logging.info('Removing rootfs verification from %s', self.options.to)
151 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
152 # Use --force to bypass the checks.
David Haddock3151d912017-10-24 03:50:32 +0000153 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
154 '--remove_rootfs_verification --force')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700155 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
David Haddock3151d912017-10-24 03:50:32 +0000156 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700157
Justin TerAvestfac210e2017-04-13 11:39:00 -0600158 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700159
David James88e6f032013-03-02 08:13:20 -0800160 # Now that the machine has been rebooted, we need to kill Chrome again.
161 self._KillProcsIfNeeded()
162
163 # Make sure the rootfs is writable now.
164 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700165
Steven Bennettsca73efa2018-07-10 13:36:56 -0700166 return True
167
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168 def _CheckUiJobStarted(self):
169 # status output is in the format:
170 # <job_name> <status> ['process' <pid>].
171 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800172 try:
David Haddock3151d912017-10-24 03:50:32 +0000173 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 except cros_build_lib.RunCommandError as e:
175 if 'Unknown job' in e.result.error:
176 return False
177 else:
178 raise e
179
Ryan Cui3045c5d2012-07-13 18:00:33 -0700180 return result.output.split()[1].split('/')[0] == 'start'
181
182 def _KillProcsIfNeeded(self):
183 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800184 logging.info('Shutting down Chrome...')
David Haddock3151d912017-10-24 03:50:32 +0000185 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700186
187 # Developers sometimes run session_manager manually, in which case we'll
188 # need to help shut the chrome processes down.
189 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700190 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700191 while self._ChromeFileInUse():
192 logging.warning('The chrome binary on the device is in use.')
193 logging.warning('Killing chrome and session_manager processes...\n')
194
David Haddock3151d912017-10-24 03:50:32 +0000195 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500196 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700197 # Wait for processes to actually terminate
198 time.sleep(POST_KILL_WAIT)
199 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800200 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800201 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700202 'running chrome processes and try again.'
203 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800204 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
David James88e6f032013-03-02 08:13:20 -0800206 def _MountRootfsAsWritable(self, error_code_ok=True):
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800207 """Mounts the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Steven Bennettsca73efa2018-07-10 13:36:56 -0700209 If the command fails and the root dir is not writable then this function
210 sets self._root_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700211
Mike Frysinger02e1e072013-11-10 22:11:34 -0500212 Args:
David James88e6f032013-03-02 08:13:20 -0800213 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
214 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800215 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500216 result = self.device.RunCommand(MOUNT_RW_COMMAND,
217 error_code_ok=error_code_ok,
218 capture_output=True)
Steven Bennettsca73efa2018-07-10 13:36:56 -0700219 if result.returncode and not self.device.IsDirWritable('/'):
220 self._root_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700221
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800222 def _EnsureTargetDir(self):
223 """Ensures that the target directory exists on the remote device."""
224 target_dir = self.options.target_dir
225 # Any valid /opt directory should already exist so avoid the remote call.
226 if os.path.commonprefix([target_dir, '/opt']) == '/opt':
227 return
228 self.device.RunCommand(['mkdir', '-p', '--mode', '0775', target_dir])
229
Ryan Cui7193a7e2013-04-26 14:15:19 -0700230 def _GetDeviceInfo(self):
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800231 """Returns the disk space used and available for the target diectory."""
Ryan Cui7193a7e2013-04-26 14:15:19 -0700232 steps = [
233 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
234 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
235 ]
236 return_values = parallel.RunParallelSteps(steps, return_values=True)
237 return DeviceInfo(*return_values)
238
239 def _CheckDeviceFreeSpace(self, device_info):
240 """See if target device has enough space for Chrome.
241
Mike Frysinger02e1e072013-11-10 22:11:34 -0500242 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700243 device_info: A DeviceInfo named tuple.
244 """
245 effective_free = device_info.target_dir_size + device_info.target_fs_free
246 staging_size = self._GetStagingDirSize()
247 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700248 raise DeployFailure(
249 'Not enough free space on the device. Required: %s MiB, '
Mike Frysinger93e8ffa2019-07-03 20:24:18 -0400250 'actual: %s MiB.' % (staging_size // 1024, effective_free // 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700251 if device_info.target_fs_free < (100 * 1024):
252 logging.warning('The device has less than 100MB free. deploy_chrome may '
253 'hang during the transfer.')
254
Satoru Takabayashif2893002017-06-20 14:52:48 +0900255 def _ShouldUseCompression(self):
256 """Checks if compression should be used for rsync."""
257 if self.options.compress == 'always':
258 return True
259 elif self.options.compress == 'never':
260 return False
261 elif self.options.compress == 'auto':
262 return not self.device.HasGigabitEthernet()
263
Ryan Cui3045c5d2012-07-13 18:00:33 -0700264 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800265 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700266 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
267 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900268 if not self.device.HasRsync():
269 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700270 'rsync is not found on the device.\n'
Jorge Lucangeli Obesea3742f2019-02-07 15:29:09 -0500271 'Run dev_install on the device to get rsync installed.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500272 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
273 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700274 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900275 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700276 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500277 verbose=self.options.verbose)
Ben Pastene5f03b052019-08-12 18:03:24 -0700278
279 # Set the security context on the default Chrome dir if that's where it's
280 # getting deployed, and only on SELinux supported devices.
281 if (self.device.IsSELinuxAvailable() and
282 _CHROME_DIR in (self.options.target_dir, self.options.mount_dir)):
Miah Sanchez769bb1b2019-07-26 13:02:00 -0700283 self.device.RunCommand(['restorecon', '-R', _CHROME_DIR])
Steve Funge984a532013-11-25 17:09:25 -0800284
285 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800286 if p.mode:
287 # Set mode if necessary.
David Haddock3151d912017-10-24 03:50:32 +0000288 self.device.RunCommand('chmod %o %s/%s' % (
289 p.mode, self.options.target_dir, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800290
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700291 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
292 # pick up major changes (bus type, logging, etc.), but all we care about is
293 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
294 # be authorized to take ownership of its service names.
David Haddock3151d912017-10-24 03:50:32 +0000295 self.device.RunCommand(DBUS_RELOAD_COMMAND, error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600296
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800297 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800298 logging.info('Starting UI...')
David Haddock3151d912017-10-24 03:50:32 +0000299 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700300
David James88e6f032013-03-02 08:13:20 -0800301 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700302 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800303 logging.info('Testing connection to the device...')
David Haddock3151d912017-10-24 03:50:32 +0000304 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800305 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700306 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800307 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700308
Steve Funge984a532013-11-25 17:09:25 -0800309 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700310 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700311 def BinaryExists(filename):
312 """Checks if the passed-in file is present in the build directory."""
313 return os.path.exists(os.path.join(self.options.build_dir, filename))
314
Daniel Erat9813f0e2014-11-12 11:00:28 -0700315 # Handle non-Chrome deployments.
316 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700317 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700318 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
319
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700320 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700321 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700322 self.options.startui = False
323
David James88e6f032013-03-02 08:13:20 -0800324 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800325 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
326 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800327
Thiago Goncales12793312013-05-23 11:26:17 -0700328 def _MountTarget(self):
329 logging.info('Mounting Chrome...')
330
331 # Create directory if does not exist
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800332 self.device.RunCommand(['mkdir', '-p', '--mode', '0775',
333 self.options.mount_dir])
Pawel Osciakfb200f52014-12-21 13:42:05 +0900334 # Umount the existing mount on mount_dir if present first
David Haddock3151d912017-10-24 03:50:32 +0000335 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
336 {'dir': self.options.mount_dir})
337 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
338 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700339 # Chrome needs partition to have exec and suid flags set
David Haddock3151d912017-10-24 03:50:32 +0000340 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
Robert Flack1dc7ea82014-11-26 13:50:24 -0500341
342 def Cleanup(self):
343 """Clean up RemoteDevice."""
344 if not self.options.staging_only:
345 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700346
David James88e6f032013-03-02 08:13:20 -0800347 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800348 self._CheckDeployType()
349
David James88e6f032013-03-02 08:13:20 -0800350 # If requested, just do the staging step.
351 if self.options.staging_only:
352 self._PrepareStagingDir()
353 return 0
354
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800355 # Ensure that the target directory exists before running parallel steps.
356 self._EnsureTargetDir()
357
David James88e6f032013-03-02 08:13:20 -0800358 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800359 # stop printing output at that point, and halt any running steps.
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800360 logging.info('Preparing device')
Steve Funge984a532013-11-25 17:09:25 -0800361 steps = [self._GetDeviceInfo, self._CheckConnection,
362 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
363 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700364 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
365 return_values=True)
366 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800367
Steven Bennettsca73efa2018-07-10 13:36:56 -0700368 # If the root dir is not writable, try disabling rootfs verification.
369 # (We always do this by default so that developers can write to
370 # /etc/chriome_dev.conf and other directories in the rootfs).
371 if self._root_dir_is_still_readonly.is_set():
372 if self.options.noremove_rootfs_verification:
373 logging.warning('Skipping disable rootfs verification.')
374 elif not self._DisableRootfsVerification():
375 logging.warning('Failed to disable rootfs verification.')
376
377 # If the target dir is still not writable (i.e. the user opted out or the
378 # command failed), abort.
379 if not self.device.IsDirWritable(self.options.target_dir):
380 if self.options.startui:
381 logging.info('Restarting Chrome...')
382 self.device.RunCommand('start ui')
383 raise DeployFailure('Target location is not writable. Aborting.')
David James88e6f032013-03-02 08:13:20 -0800384
Thiago Goncales12793312013-05-23 11:26:17 -0700385 if self.options.mount_dir is not None:
386 self._MountTarget()
387
David James88e6f032013-03-02 08:13:20 -0800388 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700389 self._Deploy()
390
391
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700392def ValidateStagingFlags(value):
393 """Convert formatted string to dictionary."""
394 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700395
396
Steven Bennetts368c3e52016-09-23 13:05:21 -0700397def ValidateGnArgs(value):
398 """Convert GN_ARGS-formatted string to dictionary."""
399 return gn_helpers.FromGNArgs(value)
400
401
Ryan Cuie535b172012-10-19 18:25:03 -0700402def _CreateParser():
403 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400404 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700405
Ryan Cuia56a71e2012-10-18 18:40:35 -0700406 # TODO(rcui): Have this use the UI-V2 format of having source and target
407 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400408 parser.add_argument('--force', action='store_true', default=False,
409 help='Skip all prompts (i.e., for disabling of rootfs '
410 'verification). This may result in the target '
411 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800412 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400413 parser.add_argument('--board', default=sdk_board_env,
Mike Frysinger80de5012019-08-01 14:10:53 -0400414 help='The board the Chrome build is targeted for. When '
Mike Frysingerc3061a62015-06-04 04:16:18 -0400415 "in a 'cros chrome-sdk' shell, defaults to the SDK "
Mike Frysinger80de5012019-08-01 14:10:53 -0400416 'board.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400417 parser.add_argument('--build-dir', type='path',
418 help='The directory with Chrome build artifacts to '
419 'deploy from. Typically of format '
420 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700421 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400422 parser.add_argument('--target-dir', type='path',
423 default=None,
424 help='Target directory on device to deploy Chrome into.')
425 parser.add_argument('-g', '--gs-path', type='gs_path',
426 help='GS path that contains the chrome to deploy.')
Adrian Eldera2c548a2017-11-07 19:01:29 -0500427 parser.add_argument('--private-key', type='path', default=None,
428 help='An ssh private key to use when deploying to '
429 'a CrOS device.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400430 parser.add_argument('--nostartui', action='store_false', dest='startui',
431 default=True,
432 help="Don't restart the ui daemon after deployment.")
433 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
434 default=True,
435 help="Don't strip binaries during deployment. Warning: "
436 'the resulting binaries will be very large!')
437 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
438 help='Port of the target device to connect to.')
439 parser.add_argument('-t', '--to',
440 help='The IP address of the CrOS device to deploy to.')
441 parser.add_argument('-v', '--verbose', action='store_true', default=False,
442 help='Show more debug output.')
443 parser.add_argument('--mount-dir', type='path', default=None,
444 help='Deploy Chrome in target directory and bind it '
445 'to the directory specified by this flag.'
446 'Any existing mount on this directory will be '
447 'umounted first.')
448 parser.add_argument('--mount', action='store_true', default=False,
449 help='Deploy Chrome to default target directory and bind '
450 'it to the default mount directory.'
451 'Any existing mount on this directory will be '
452 'umounted first.')
Steven Bennettsca73efa2018-07-10 13:36:56 -0700453 parser.add_argument('--noremove-rootfs-verification', action='store_true',
454 default=False, help='Never remove rootfs verification.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700455
Mike Frysingerc3061a62015-06-04 04:16:18 -0400456 group = parser.add_argument_group('Advanced Options')
457 group.add_argument('-l', '--local-pkg-path', type='path',
458 help='Path to local chrome prebuilt package to deploy.')
459 group.add_argument('--sloppy', action='store_true', default=False,
460 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700461 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400462 help=('Extra flags to control staging. Valid flags are - '
463 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700464 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400465 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700466 help='Deprecated. Default behavior is "strict". Use '
467 '--sloppy to omit warnings for missing optional '
468 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400469 group.add_argument('--strip-flags', default=None,
470 help="Flags to call the 'strip' binutil tool with. "
Mike Frysinger80de5012019-08-01 14:10:53 -0400471 'Overrides the default arguments.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400472 group.add_argument('--ping', action='store_true', default=False,
473 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700474 group.add_argument('--process-timeout', type=int,
475 default=KILL_PROC_MAX_WAIT,
476 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700477
Mike Frysingerc3061a62015-06-04 04:16:18 -0400478 group = parser.add_argument_group(
479 'Metadata Overrides (Advanced)',
480 description='Provide all of these overrides in order to remove '
481 'dependencies on metadata.json existence.')
482 group.add_argument('--target-tc', action='store', default=None,
483 help='Override target toolchain name, e.g. '
484 'x86_64-cros-linux-gnu')
485 group.add_argument('--toolchain-url', action='store', default=None,
486 help='Override toolchain url format pattern, e.g. '
487 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700488
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700489 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
490 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
491 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400492 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700493
494 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
495 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
496 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
497 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
498 help=argparse.SUPPRESS)
499
Ryan Cuia56a71e2012-10-18 18:40:35 -0700500 # Path of an empty directory to stage chrome artifacts to. Defaults to a
501 # temporary directory that is removed when the script finishes. If the path
502 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400503 parser.add_argument('--staging-dir', type='path', default=None,
504 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700505 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400506 parser.add_argument('--staging-only', action='store_true', default=False,
507 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700508 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
509 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
510 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400511 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900512 parser.add_argument('--compress', action='store', default='auto',
513 choices=('always', 'never', 'auto'),
514 help='Choose the data compression behavior. Default '
515 'is set to "auto", that disables compression if '
516 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700517 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700518
Ryan Cuie535b172012-10-19 18:25:03 -0700519
520def _ParseCommandLine(argv):
521 """Parse args, and run environment-independent checks."""
522 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400523 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700524
Ryan Cuia56a71e2012-10-18 18:40:35 -0700525 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
526 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800527 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700528 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
529 parser.error('Cannot specify both --build_dir and '
530 '--gs-path/--local-pkg-patch')
Achuith Bhandarkar31a3eb02018-03-22 16:33:48 -0700531 if (not options.board and options.build_dir and options.dostrip and
532 not options.strip_bin):
533 parser.error('--board is required for stripping.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700534 if options.gs_path and options.local_pkg_path:
535 parser.error('Cannot specify both --gs-path and --local-pkg-path')
536 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700537 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700538 if options.staging_flags and not options.build_dir:
539 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700540
Steven Bennetts46a84c32016-08-12 15:20:46 -0700541 if options.strict:
542 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700543 if options.gyp_defines:
544 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700545
546 if options.mount or options.mount_dir:
547 if not options.target_dir:
548 options.target_dir = _CHROME_DIR_MOUNT
549 else:
550 if not options.target_dir:
551 options.target_dir = _CHROME_DIR
552
553 if options.mount and not options.mount_dir:
554 options.mount_dir = _CHROME_DIR
555
Mike Frysingerc3061a62015-06-04 04:16:18 -0400556 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700557
558
Mike Frysingerc3061a62015-06-04 04:16:18 -0400559def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800560 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700561
562 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400563 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700564 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700565 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
566 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
567
Steven Bennetts368c3e52016-09-23 13:05:21 -0700568 if not options.gn_args:
569 gn_env = os.getenv('GN_ARGS')
570 if gn_env is not None:
571 options.gn_args = gn_helpers.FromGNArgs(gn_env)
Steven Bennetts2ae83c72017-12-04 16:34:24 -0800572 logging.debug('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700573
Steven Bennetts60600462016-05-12 10:40:20 -0700574 if not options.staging_flags:
575 use_env = os.getenv('USE')
576 if use_env is not None:
577 options.staging_flags = ' '.join(set(use_env.split()).intersection(
578 chrome_util.STAGING_FLAGS))
579 logging.info('Staging flags taken from USE in environment: %s',
580 options.staging_flags)
581
Ryan Cuia56a71e2012-10-18 18:40:35 -0700582
Ryan Cui504db722013-01-22 11:48:01 -0800583def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700584 """Get the chrome prebuilt tarball from GS.
585
Mike Frysinger02e1e072013-11-10 22:11:34 -0500586 Returns:
587 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700588 """
David James9374aac2013-10-08 16:00:17 -0700589 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500590 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800591 files = [found for found in files if
592 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
593 if not files:
594 raise Exception('No chrome package found at %s' % gs_path)
595 elif len(files) > 1:
596 # - Users should provide us with a direct link to either a stripped or
597 # unstripped chrome package.
598 # - In the case of being provided with an archive directory, where both
599 # stripped and unstripped chrome available, use the stripped chrome
600 # package.
601 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
602 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
603 files = [f for f in files if not 'unstripped' in f]
604 assert len(files) == 1
605 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800606
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800607 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800608 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800609 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
610 chrome_path = os.path.join(tempdir, filename)
611 assert os.path.exists(chrome_path)
612 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700613
614
Ryan Cuif890a3e2013-03-07 18:57:06 -0800615@contextlib.contextmanager
616def _StripBinContext(options):
617 if not options.dostrip:
618 yield None
619 elif options.strip_bin:
620 yield options.strip_bin
621 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800622 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800623 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700624 with sdk.Prepare(components=components, target_tc=options.target_tc,
625 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800626 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
627 constants.CHROME_ENV_FILE)
628 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
629 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
630 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800631 yield strip_bin
632
633
Steve Funge984a532013-11-25 17:09:25 -0800634def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
635 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800636 """Place the necessary files in the staging directory.
637
638 The staging directory is the directory used to rsync the build artifacts over
639 to the device. Only the necessary Chrome build artifacts are put into the
640 staging directory.
641 """
Ryan Cui5866be02013-03-18 14:12:00 -0700642 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400643 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800644 if options.build_dir:
645 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700646 strip_flags = (None if options.strip_flags is None else
647 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800648 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700649 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700650 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700651 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800652 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700653 else:
654 pkg_path = options.local_pkg_path
655 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800656 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
657 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700658
659 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800660 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700661 # Extract only the ./opt/google/chrome contents, directly into the staging
662 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800663 if pkg_path[-4:] == '.zip':
Mike Frysinger45602c72019-09-22 02:15:11 -0400664 cros_build_lib.dbg_run(
Steve Funge984a532013-11-25 17:09:25 -0800665 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
666 staging_dir])
667 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
668 shutil.move(filename, staging_dir)
669 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
670 else:
Mike Frysinger45602c72019-09-22 02:15:11 -0400671 cros_build_lib.dbg_run(
Steve Funge984a532013-11-25 17:09:25 -0800672 ['tar', '--strip-components', '4', '--extract',
673 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
674 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700675
Ryan Cui71aa8de2013-04-19 16:12:55 -0700676
Ryan Cui3045c5d2012-07-13 18:00:33 -0700677def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400678 options = _ParseCommandLine(argv)
679 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700680
681 # Set cros_build_lib debug level to hide RunCommand spew.
682 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700683 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700684 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800685 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700686
Aviv Keshet01a82e92017-03-02 17:39:59 -0800687 with osutils.TempDir(set_global=True) as tempdir:
688 staging_dir = options.staging_dir
689 if not staging_dir:
690 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700691
Aviv Keshet01a82e92017-03-02 17:39:59 -0800692 deploy = DeployChrome(options, tempdir, staging_dir)
693 try:
694 deploy.Perform()
695 except failures_lib.StepFailure as ex:
696 raise SystemExit(str(ex).strip())
697 deploy.Cleanup()