blob: ef112c72182c3a4076b1d811c06f9aefe0b3c8e5 [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
Mike Frysinger24151262017-10-03 02:50:01 -040053# Complicated commands that unittests want to mock out.
54# All others should be inlined.
55MOUNT_RW_COMMAND = ['mount', '-o', 'remount,rw', '/']
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
Mike Frysingere65f3752014-12-08 00:46:39 -050063
Ryan Cui3045c5d2012-07-13 18:00:33 -070064def _UrlBaseName(url):
65 """Return the last component of the URL."""
66 return url.rstrip('/').rpartition('/')[-1]
67
68
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070069class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080070 """Raised whenever the deploy fails."""
71
72
Ryan Cui7193a7e2013-04-26 14:15:19 -070073DeviceInfo = collections.namedtuple(
74 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
75
76
Ryan Cui3045c5d2012-07-13 18:00:33 -070077class DeployChrome(object):
78 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050079
Ryan Cuia56a71e2012-10-18 18:40:35 -070080 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070081 """Initialize the class.
82
Mike Frysinger02e1e072013-11-10 22:11:34 -050083 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040084 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070085 tempdir: Scratch space for the class. Caller has responsibility to clean
86 it up.
Steve Funge984a532013-11-25 17:09:25 -080087 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070088 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070089 self.tempdir = tempdir
90 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070091 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050092 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090093 self.device = remote.RemoteDevice(options.to, port=options.port,
94 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -050095 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +000096
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -050097 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -080098 self.chrome_dir = _CHROME_DIR
99
Ryan Cui7193a7e2013-04-26 14:15:19 -0700100 def _GetRemoteMountFree(self, remote_dir):
Mike Frysinger24151262017-10-03 02:50:01 -0400101 result = self.device.RunCommand(['df', '-k', remote_dir])
Ryan Cui7193a7e2013-04-26 14:15:19 -0700102 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800103 value = line.split()[3]
104 multipliers = {
105 'G': 1024 * 1024 * 1024,
106 'M': 1024 * 1024,
107 'K': 1024,
108 }
109 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110
111 def _GetRemoteDirSize(self, remote_dir):
Mike Frysinger24151262017-10-03 02:50:01 -0400112 result = self.device.RunCommand(['du', '-ks', remote_dir],
Robert Flack1dc7ea82014-11-26 13:50:24 -0500113 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700114 return int(result.output.split()[0])
115
116 def _GetStagingDirSize(self):
117 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800118 redirect_stdout=True,
119 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700120 return int(result.output.split()[0])
121
Ryan Cui3045c5d2012-07-13 18:00:33 -0700122 def _ChromeFileInUse(self):
Mike Frysinger24151262017-10-03 02:50:01 -0400123 result = self.device.RunCommand(
124 ['lsof', os.path.join(self.options.target_dir, 'chrome')],
125 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700126 return result.returncode == 0
127
Justin TerAvestfac210e2017-04-13 11:39:00 -0600128 def _Reboot(self):
129 # A reboot in developer mode takes a while (and has delays), so the user
130 # will have time to read and act on the USB boot instructions below.
131 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
132 self.device.Reboot()
133
Ryan Cui3045c5d2012-07-13 18:00:33 -0700134 def _DisableRootfsVerification(self):
135 if not self.options.force:
136 logging.error('Detected that the device has rootfs verification enabled.')
137 logging.info('This script can automatically remove the rootfs '
138 'verification, which requires that it reboot the device.')
139 logging.info('Make sure the device is in developer mode!')
140 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700141 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800142 # Since we stopped Chrome earlier, it's good form to start it up again.
143 if self.options.startui:
144 logging.info('Starting Chrome...')
Mike Frysinger24151262017-10-03 02:50:01 -0400145 self.device.RunCommand(['start', 'ui'])
David James88e6f032013-03-02 08:13:20 -0800146 raise DeployFailure('Need rootfs verification to be disabled. '
147 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700148
149 logging.info('Removing rootfs verification from %s', self.options.to)
150 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
151 # Use --force to bypass the checks.
Mike Frysinger24151262017-10-03 02:50:01 -0400152 cmd = ['/usr/share/vboot/bin/make_dev_ssd.sh', '--force',
153 '--remove_rootfs_verification', '--partitions']
Ryan Cui3045c5d2012-07-13 18:00:33 -0700154 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Mike Frysinger24151262017-10-03 02:50:01 -0400155 self.device.RunCommand(cmd + [str(partition)], error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156
Justin TerAvestfac210e2017-04-13 11:39:00 -0600157 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
David James88e6f032013-03-02 08:13:20 -0800159 # Now that the machine has been rebooted, we need to kill Chrome again.
160 self._KillProcsIfNeeded()
161
162 # Make sure the rootfs is writable now.
163 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700164
165 def _CheckUiJobStarted(self):
166 # status output is in the format:
167 # <job_name> <status> ['process' <pid>].
168 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800169 try:
Mike Frysinger24151262017-10-03 02:50:01 -0400170 result = self.device.RunCommand(['status', 'ui'], capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800171 except cros_build_lib.RunCommandError as e:
172 if 'Unknown job' in e.result.error:
173 return False
174 else:
175 raise e
176
Ryan Cui3045c5d2012-07-13 18:00:33 -0700177 return result.output.split()[1].split('/')[0] == 'start'
178
179 def _KillProcsIfNeeded(self):
180 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800181 logging.info('Shutting down Chrome...')
Mike Frysinger24151262017-10-03 02:50:01 -0400182 self.device.RunCommand(['stop', 'ui'])
Ryan Cui3045c5d2012-07-13 18:00:33 -0700183
184 # Developers sometimes run session_manager manually, in which case we'll
185 # need to help shut the chrome processes down.
186 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700187 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188 while self._ChromeFileInUse():
189 logging.warning('The chrome binary on the device is in use.')
190 logging.warning('Killing chrome and session_manager processes...\n')
191
Mike Frysinger24151262017-10-03 02:50:01 -0400192 self.device.RunCommand(['pkill', 'chrome|session_manager'],
Mike Frysingere65f3752014-12-08 00:46:39 -0500193 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700194 # Wait for processes to actually terminate
195 time.sleep(POST_KILL_WAIT)
196 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800197 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800198 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700199 'running chrome processes and try again.'
200 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800201 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700202
David James88e6f032013-03-02 08:13:20 -0800203 def _MountRootfsAsWritable(self, error_code_ok=True):
204 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
Robert Flack1dc7ea82014-11-26 13:50:24 -0500206 If the command fails, and error_code_ok is True, and the target dir is not
207 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Mike Frysinger02e1e072013-11-10 22:11:34 -0500209 Args:
David James88e6f032013-03-02 08:13:20 -0800210 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
211 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800212 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500213 result = self.device.RunCommand(MOUNT_RW_COMMAND,
214 error_code_ok=error_code_ok,
215 capture_output=True)
216 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400217 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500218 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700219
Ryan Cui7193a7e2013-04-26 14:15:19 -0700220 def _GetDeviceInfo(self):
221 steps = [
222 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
223 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
224 ]
225 return_values = parallel.RunParallelSteps(steps, return_values=True)
226 return DeviceInfo(*return_values)
227
228 def _CheckDeviceFreeSpace(self, device_info):
229 """See if target device has enough space for Chrome.
230
Mike Frysinger02e1e072013-11-10 22:11:34 -0500231 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700232 device_info: A DeviceInfo named tuple.
233 """
234 effective_free = device_info.target_dir_size + device_info.target_fs_free
235 staging_size = self._GetStagingDirSize()
236 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700237 raise DeployFailure(
238 'Not enough free space on the device. Required: %s MiB, '
239 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700240 if device_info.target_fs_free < (100 * 1024):
241 logging.warning('The device has less than 100MB free. deploy_chrome may '
242 'hang during the transfer.')
243
Satoru Takabayashif2893002017-06-20 14:52:48 +0900244 def _ShouldUseCompression(self):
245 """Checks if compression should be used for rsync."""
246 if self.options.compress == 'always':
247 return True
248 elif self.options.compress == 'never':
249 return False
250 elif self.options.compress == 'auto':
251 return not self.device.HasGigabitEthernet()
252
Ryan Cui3045c5d2012-07-13 18:00:33 -0700253 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800254 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700255 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
256 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900257 if not self.device.HasRsync():
258 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700259 'rsync is not found on the device.\n'
260 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500261 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
262 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700263 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900264 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700265 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500266 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800267
268 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800269 if p.mode:
270 # Set mode if necessary.
Mike Frysinger24151262017-10-03 02:50:01 -0400271 self.device.RunCommand(
272 ['chmod', '%o' % p.mode,
273 '%s/%s' % (self.options.target_dir,
274 p.src if not p.dest else p.dest)])
Steve Funge984a532013-11-25 17:09:25 -0800275
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700276 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
277 # pick up major changes (bus type, logging, etc.), but all we care about is
278 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
279 # be authorized to take ownership of its service names.
Mike Frysinger24151262017-10-03 02:50:01 -0400280 self.device.RunCommand(['killall', '-HUP', 'dbus-daemon'],
281 error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600282
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800283 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800284 logging.info('Starting UI...')
Mike Frysinger24151262017-10-03 02:50:01 -0400285 self.device.RunCommand(['start', 'ui'])
Ryan Cui3045c5d2012-07-13 18:00:33 -0700286
David James88e6f032013-03-02 08:13:20 -0800287 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700288 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800289 logging.info('Testing connection to the device...')
Mike Frysinger24151262017-10-03 02:50:01 -0400290 self.device.RunCommand(['true'])
David James88e6f032013-03-02 08:13:20 -0800291 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700292 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800293 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700294
Steve Funge984a532013-11-25 17:09:25 -0800295 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700296 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700297 def BinaryExists(filename):
298 """Checks if the passed-in file is present in the build directory."""
299 return os.path.exists(os.path.join(self.options.build_dir, filename))
300
Daniel Erat9813f0e2014-11-12 11:00:28 -0700301 # Handle non-Chrome deployments.
302 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700303 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700304 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
305
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700306 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700307 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700308 self.options.startui = False
309
David James88e6f032013-03-02 08:13:20 -0800310 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800311 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
312 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800313
Thiago Goncales12793312013-05-23 11:26:17 -0700314 def _MountTarget(self):
315 logging.info('Mounting Chrome...')
316
317 # Create directory if does not exist
Mike Frysinger24151262017-10-03 02:50:01 -0400318 self.device.RunCommand(['mkdir', '-p', '--mode', '0775',
319 self.options.mount_dir])
Pawel Osciakfb200f52014-12-21 13:42:05 +0900320 # Umount the existing mount on mount_dir if present first
Mike Frysingere7dc3d52017-10-11 17:59:30 -0400321 ret = self.device.RunCommand(['mountpoint', '-q', self.options.mount_dir],
322 error_code_ok=True)
323 if ret.returncode == 0:
324 self.device.RunCommand(['umount', self.options.mount_dir])
Mike Frysinger24151262017-10-03 02:50:01 -0400325 self.device.RunCommand(['mount', '--rbind', self.options.target_dir,
326 self.options.mount_dir])
Thiago Goncales12793312013-05-23 11:26:17 -0700327 # Chrome needs partition to have exec and suid flags set
Mike Frysinger24151262017-10-03 02:50:01 -0400328 self.device.RunCommand(['mount', '-o', 'remount,exec,suid',
329 self.options.mount_dir])
Robert Flack1dc7ea82014-11-26 13:50:24 -0500330
331 def Cleanup(self):
332 """Clean up RemoteDevice."""
333 if not self.options.staging_only:
334 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700335
David James88e6f032013-03-02 08:13:20 -0800336 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800337 self._CheckDeployType()
338
David James88e6f032013-03-02 08:13:20 -0800339 # If requested, just do the staging step.
340 if self.options.staging_only:
341 self._PrepareStagingDir()
342 return 0
343
344 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800345 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800346 steps = [self._GetDeviceInfo, self._CheckConnection,
347 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
348 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700349 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
350 return_values=True)
351 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800352
Robert Flack1dc7ea82014-11-26 13:50:24 -0500353 # If we're trying to deploy to a dir which is not writable and we failed
354 # to mark the rootfs as writable, try disabling rootfs verification.
355 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800356 self._DisableRootfsVerification()
357
Thiago Goncales12793312013-05-23 11:26:17 -0700358 if self.options.mount_dir is not None:
359 self._MountTarget()
360
David James88e6f032013-03-02 08:13:20 -0800361 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700362 self._Deploy()
363
364
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700365def ValidateStagingFlags(value):
366 """Convert formatted string to dictionary."""
367 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700368
369
Steven Bennetts368c3e52016-09-23 13:05:21 -0700370def ValidateGnArgs(value):
371 """Convert GN_ARGS-formatted string to dictionary."""
372 return gn_helpers.FromGNArgs(value)
373
374
Ryan Cuie535b172012-10-19 18:25:03 -0700375def _CreateParser():
376 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400377 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700378
Ryan Cuia56a71e2012-10-18 18:40:35 -0700379 # TODO(rcui): Have this use the UI-V2 format of having source and target
380 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400381 parser.add_argument('--force', action='store_true', default=False,
382 help='Skip all prompts (i.e., for disabling of rootfs '
383 'verification). This may result in the target '
384 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800385 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400386 parser.add_argument('--board', default=sdk_board_env,
387 help="The board the Chrome build is targeted for. When "
388 "in a 'cros chrome-sdk' shell, defaults to the SDK "
389 "board.")
390 parser.add_argument('--build-dir', type='path',
391 help='The directory with Chrome build artifacts to '
392 'deploy from. Typically of format '
393 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700394 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400395 parser.add_argument('--target-dir', type='path',
396 default=None,
397 help='Target directory on device to deploy Chrome into.')
398 parser.add_argument('-g', '--gs-path', type='gs_path',
399 help='GS path that contains the chrome to deploy.')
400 parser.add_argument('--nostartui', action='store_false', dest='startui',
401 default=True,
402 help="Don't restart the ui daemon after deployment.")
403 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
404 default=True,
405 help="Don't strip binaries during deployment. Warning: "
406 'the resulting binaries will be very large!')
407 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
408 help='Port of the target device to connect to.')
409 parser.add_argument('-t', '--to',
410 help='The IP address of the CrOS device to deploy to.')
411 parser.add_argument('-v', '--verbose', action='store_true', default=False,
412 help='Show more debug output.')
413 parser.add_argument('--mount-dir', type='path', default=None,
414 help='Deploy Chrome in target directory and bind it '
415 'to the directory specified by this flag.'
416 'Any existing mount on this directory will be '
417 'umounted first.')
418 parser.add_argument('--mount', action='store_true', default=False,
419 help='Deploy Chrome to default target directory and bind '
420 'it to the default mount directory.'
421 'Any existing mount on this directory will be '
422 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700423
Mike Frysingerc3061a62015-06-04 04:16:18 -0400424 group = parser.add_argument_group('Advanced Options')
425 group.add_argument('-l', '--local-pkg-path', type='path',
426 help='Path to local chrome prebuilt package to deploy.')
427 group.add_argument('--sloppy', action='store_true', default=False,
428 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700429 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400430 help=('Extra flags to control staging. Valid flags are - '
431 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700432 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400433 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700434 help='Deprecated. Default behavior is "strict". Use '
435 '--sloppy to omit warnings for missing optional '
436 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400437 group.add_argument('--strip-flags', default=None,
438 help="Flags to call the 'strip' binutil tool with. "
439 "Overrides the default arguments.")
440 group.add_argument('--ping', action='store_true', default=False,
441 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700442 group.add_argument('--process-timeout', type=int,
443 default=KILL_PROC_MAX_WAIT,
444 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700445
Mike Frysingerc3061a62015-06-04 04:16:18 -0400446 group = parser.add_argument_group(
447 'Metadata Overrides (Advanced)',
448 description='Provide all of these overrides in order to remove '
449 'dependencies on metadata.json existence.')
450 group.add_argument('--target-tc', action='store', default=None,
451 help='Override target toolchain name, e.g. '
452 'x86_64-cros-linux-gnu')
453 group.add_argument('--toolchain-url', action='store', default=None,
454 help='Override toolchain url format pattern, e.g. '
455 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700456
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700457 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
458 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
459 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400460 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700461
462 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
463 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
464 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
465 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
466 help=argparse.SUPPRESS)
467
Ryan Cuia56a71e2012-10-18 18:40:35 -0700468 # Path of an empty directory to stage chrome artifacts to. Defaults to a
469 # temporary directory that is removed when the script finishes. If the path
470 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400471 parser.add_argument('--staging-dir', type='path', default=None,
472 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700473 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400474 parser.add_argument('--staging-only', action='store_true', default=False,
475 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700476 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
477 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
478 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400479 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900480 parser.add_argument('--compress', action='store', default='auto',
481 choices=('always', 'never', 'auto'),
482 help='Choose the data compression behavior. Default '
483 'is set to "auto", that disables compression if '
484 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700485 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700486
Ryan Cuie535b172012-10-19 18:25:03 -0700487
488def _ParseCommandLine(argv):
489 """Parse args, and run environment-independent checks."""
490 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400491 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700492
Ryan Cuia56a71e2012-10-18 18:40:35 -0700493 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
494 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800495 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700496 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
497 parser.error('Cannot specify both --build_dir and '
498 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800499 if options.build_dir and not options.board:
500 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700501 if options.gs_path and options.local_pkg_path:
502 parser.error('Cannot specify both --gs-path and --local-pkg-path')
503 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700504 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700505 if options.staging_flags and not options.build_dir:
506 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700507
Steven Bennetts46a84c32016-08-12 15:20:46 -0700508 if options.strict:
509 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700510 if options.gyp_defines:
511 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700512
513 if options.mount or options.mount_dir:
514 if not options.target_dir:
515 options.target_dir = _CHROME_DIR_MOUNT
516 else:
517 if not options.target_dir:
518 options.target_dir = _CHROME_DIR
519
520 if options.mount and not options.mount_dir:
521 options.mount_dir = _CHROME_DIR
522
Mike Frysingerc3061a62015-06-04 04:16:18 -0400523 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700524
525
Mike Frysingerc3061a62015-06-04 04:16:18 -0400526def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800527 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700528
529 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400530 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700531 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700532 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
533 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
534
Steven Bennetts368c3e52016-09-23 13:05:21 -0700535 if not options.gn_args:
536 gn_env = os.getenv('GN_ARGS')
537 if gn_env is not None:
538 options.gn_args = gn_helpers.FromGNArgs(gn_env)
539 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700540
Steven Bennetts60600462016-05-12 10:40:20 -0700541 if not options.staging_flags:
542 use_env = os.getenv('USE')
543 if use_env is not None:
544 options.staging_flags = ' '.join(set(use_env.split()).intersection(
545 chrome_util.STAGING_FLAGS))
546 logging.info('Staging flags taken from USE in environment: %s',
547 options.staging_flags)
548
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549
Ryan Cui504db722013-01-22 11:48:01 -0800550def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551 """Get the chrome prebuilt tarball from GS.
552
Mike Frysinger02e1e072013-11-10 22:11:34 -0500553 Returns:
554 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700555 """
David James9374aac2013-10-08 16:00:17 -0700556 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500557 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800558 files = [found for found in files if
559 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
560 if not files:
561 raise Exception('No chrome package found at %s' % gs_path)
562 elif len(files) > 1:
563 # - Users should provide us with a direct link to either a stripped or
564 # unstripped chrome package.
565 # - In the case of being provided with an archive directory, where both
566 # stripped and unstripped chrome available, use the stripped chrome
567 # package.
568 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
569 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
570 files = [f for f in files if not 'unstripped' in f]
571 assert len(files) == 1
572 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800573
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800574 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800575 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800576 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
577 chrome_path = os.path.join(tempdir, filename)
578 assert os.path.exists(chrome_path)
579 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700580
581
Ryan Cuif890a3e2013-03-07 18:57:06 -0800582@contextlib.contextmanager
583def _StripBinContext(options):
584 if not options.dostrip:
585 yield None
586 elif options.strip_bin:
587 yield options.strip_bin
588 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800589 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800590 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700591 with sdk.Prepare(components=components, target_tc=options.target_tc,
592 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800593 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
594 constants.CHROME_ENV_FILE)
595 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
596 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
597 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800598 yield strip_bin
599
600
Steve Funge984a532013-11-25 17:09:25 -0800601def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
602 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800603 """Place the necessary files in the staging directory.
604
605 The staging directory is the directory used to rsync the build artifacts over
606 to the device. Only the necessary Chrome build artifacts are put into the
607 staging directory.
608 """
Ryan Cui5866be02013-03-18 14:12:00 -0700609 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400610 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800611 if options.build_dir:
612 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700613 strip_flags = (None if options.strip_flags is None else
614 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800615 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700616 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700617 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700618 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800619 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700620 else:
621 pkg_path = options.local_pkg_path
622 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800623 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
624 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700625
626 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800627 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700628 # Extract only the ./opt/google/chrome contents, directly into the staging
629 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800630 if pkg_path[-4:] == '.zip':
631 cros_build_lib.DebugRunCommand(
632 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
633 staging_dir])
634 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
635 shutil.move(filename, staging_dir)
636 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
637 else:
638 cros_build_lib.DebugRunCommand(
639 ['tar', '--strip-components', '4', '--extract',
640 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
641 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700642
Ryan Cui71aa8de2013-04-19 16:12:55 -0700643
Ryan Cui3045c5d2012-07-13 18:00:33 -0700644def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400645 options = _ParseCommandLine(argv)
646 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700647
648 # Set cros_build_lib debug level to hide RunCommand spew.
649 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700650 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700651 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800652 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700653
Aviv Keshet01a82e92017-03-02 17:39:59 -0800654 with osutils.TempDir(set_global=True) as tempdir:
655 staging_dir = options.staging_dir
656 if not staging_dir:
657 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700658
Aviv Keshet01a82e92017-03-02 17:39:59 -0800659 deploy = DeployChrome(options, tempdir, staging_dir)
660 try:
661 deploy.Perform()
662 except failures_lib.StepFailure as ex:
663 raise SystemExit(str(ex).strip())
664 deploy.Cleanup()