blob: 92198798827d057aa4a2ce28a001d9f469eb965c [file] [log] [blame]
Mike Frysingere58c0e22017-10-04 15:43:30 -04001# -*- coding: utf-8 -*-
Ryan Cui3045c5d2012-07-13 18:00:33 -07002# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Steve Funge984a532013-11-25 17:09:25 -08006"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07007
8The script supports deploying Chrome from these sources:
9
101. A local build output directory, such as chromium/src/out/[Debug|Release].
112. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
123. A Chrome tarball existing locally.
13
14The script copies the necessary contents of the source location (tarball or
15build directory) and rsyncs the contents of the staging directory onto your
16device's rootfs.
17"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070018
Mike Frysinger383367e2014-09-16 15:06:17 -040019from __future__ import print_function
20
Mike Frysingerc3061a62015-06-04 04:16:18 -040021import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080023import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070024import functools
Steve Funge984a532013-11-25 17:09:25 -080025import glob
David James88e6f032013-03-02 08:13:20 -080026import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070027import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070030import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070031
Aviv Keshetb7519e12016-10-04 00:50:00 -070032from chromite.lib import constants
33from chromite.lib import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070034from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070035from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070036from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070037from chromite.lib import cros_build_lib
38from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080039from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070040from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080041from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import remote_access as remote
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
Mike Frysinger24151262017-10-03 02:50:01 -040053# Complicated commands that unittests want to mock out.
54# All others should be inlined.
55MOUNT_RW_COMMAND = ['mount', '-o', 'remount,rw', '/']
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
Mike Frysingere65f3752014-12-08 00:46:39 -050063
Ryan Cui3045c5d2012-07-13 18:00:33 -070064def _UrlBaseName(url):
65 """Return the last component of the URL."""
66 return url.rstrip('/').rpartition('/')[-1]
67
68
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070069class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080070 """Raised whenever the deploy fails."""
71
72
Ryan Cui7193a7e2013-04-26 14:15:19 -070073DeviceInfo = collections.namedtuple(
74 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
75
76
Ryan Cui3045c5d2012-07-13 18:00:33 -070077class DeployChrome(object):
78 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050079
Ryan Cuia56a71e2012-10-18 18:40:35 -070080 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070081 """Initialize the class.
82
Mike Frysinger02e1e072013-11-10 22:11:34 -050083 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040084 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070085 tempdir: Scratch space for the class. Caller has responsibility to clean
86 it up.
Steve Funge984a532013-11-25 17:09:25 -080087 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070088 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070089 self.tempdir = tempdir
90 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070091 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050092 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090093 self.device = remote.RemoteDevice(options.to, port=options.port,
94 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -050095 self._target_dir_is_still_readonly = multiprocessing.Event()
Steven Bennetts5a7c72d2016-10-17 20:04:46 +000096
Sadrul Habib Chowdhury977aef72016-11-21 16:30:37 -050097 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -080098 self.chrome_dir = _CHROME_DIR
99
Ryan Cui7193a7e2013-04-26 14:15:19 -0700100 def _GetRemoteMountFree(self, remote_dir):
Mike Frysinger24151262017-10-03 02:50:01 -0400101 result = self.device.RunCommand(['df', '-k', remote_dir])
Ryan Cui7193a7e2013-04-26 14:15:19 -0700102 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800103 value = line.split()[3]
104 multipliers = {
105 'G': 1024 * 1024 * 1024,
106 'M': 1024 * 1024,
107 'K': 1024,
108 }
109 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110
111 def _GetRemoteDirSize(self, remote_dir):
Mike Frysinger24151262017-10-03 02:50:01 -0400112 result = self.device.RunCommand(['du', '-ks', remote_dir],
Robert Flack1dc7ea82014-11-26 13:50:24 -0500113 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700114 return int(result.output.split()[0])
115
116 def _GetStagingDirSize(self):
117 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800118 redirect_stdout=True,
119 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700120 return int(result.output.split()[0])
121
Ryan Cui3045c5d2012-07-13 18:00:33 -0700122 def _ChromeFileInUse(self):
Mike Frysinger24151262017-10-03 02:50:01 -0400123 result = self.device.RunCommand(
124 ['lsof', os.path.join(self.options.target_dir, 'chrome')],
125 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700126 return result.returncode == 0
127
Justin TerAvestfac210e2017-04-13 11:39:00 -0600128 def _Reboot(self):
129 # A reboot in developer mode takes a while (and has delays), so the user
130 # will have time to read and act on the USB boot instructions below.
131 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
132 self.device.Reboot()
133
Ryan Cui3045c5d2012-07-13 18:00:33 -0700134 def _DisableRootfsVerification(self):
135 if not self.options.force:
136 logging.error('Detected that the device has rootfs verification enabled.')
137 logging.info('This script can automatically remove the rootfs '
138 'verification, which requires that it reboot the device.')
139 logging.info('Make sure the device is in developer mode!')
140 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700141 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800142 # Since we stopped Chrome earlier, it's good form to start it up again.
143 if self.options.startui:
144 logging.info('Starting Chrome...')
Mike Frysinger24151262017-10-03 02:50:01 -0400145 self.device.RunCommand(['start', 'ui'])
David James88e6f032013-03-02 08:13:20 -0800146 raise DeployFailure('Need rootfs verification to be disabled. '
147 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700148
149 logging.info('Removing rootfs verification from %s', self.options.to)
150 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
151 # Use --force to bypass the checks.
Mike Frysinger24151262017-10-03 02:50:01 -0400152 cmd = ['/usr/share/vboot/bin/make_dev_ssd.sh', '--force',
153 '--remove_rootfs_verification', '--partitions']
Ryan Cui3045c5d2012-07-13 18:00:33 -0700154 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Mike Frysinger24151262017-10-03 02:50:01 -0400155 self.device.RunCommand(cmd + [str(partition)], error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156
Justin TerAvestfac210e2017-04-13 11:39:00 -0600157 self._Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
David James88e6f032013-03-02 08:13:20 -0800159 # Now that the machine has been rebooted, we need to kill Chrome again.
160 self._KillProcsIfNeeded()
161
162 # Make sure the rootfs is writable now.
163 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700164
165 def _CheckUiJobStarted(self):
166 # status output is in the format:
167 # <job_name> <status> ['process' <pid>].
168 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800169 try:
Mike Frysinger24151262017-10-03 02:50:01 -0400170 result = self.device.RunCommand(['status', 'ui'], capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800171 except cros_build_lib.RunCommandError as e:
172 if 'Unknown job' in e.result.error:
173 return False
174 else:
175 raise e
176
Ryan Cui3045c5d2012-07-13 18:00:33 -0700177 return result.output.split()[1].split('/')[0] == 'start'
178
179 def _KillProcsIfNeeded(self):
180 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800181 logging.info('Shutting down Chrome...')
Mike Frysinger24151262017-10-03 02:50:01 -0400182 self.device.RunCommand(['stop', 'ui'])
Ryan Cui3045c5d2012-07-13 18:00:33 -0700183
184 # Developers sometimes run session_manager manually, in which case we'll
185 # need to help shut the chrome processes down.
186 try:
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700187 with timeout_util.Timeout(self.options.process_timeout):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188 while self._ChromeFileInUse():
189 logging.warning('The chrome binary on the device is in use.')
190 logging.warning('Killing chrome and session_manager processes...\n')
191
Mike Frysinger24151262017-10-03 02:50:01 -0400192 self.device.RunCommand(['pkill', 'chrome|session_manager'],
Mike Frysingere65f3752014-12-08 00:46:39 -0500193 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700194 # Wait for processes to actually terminate
195 time.sleep(POST_KILL_WAIT)
196 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800197 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800198 msg = ('Could not kill processes after %s seconds. Please exit any '
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700199 'running chrome processes and try again.'
200 % self.options.process_timeout)
David James88e6f032013-03-02 08:13:20 -0800201 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700202
David James88e6f032013-03-02 08:13:20 -0800203 def _MountRootfsAsWritable(self, error_code_ok=True):
204 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
Robert Flack1dc7ea82014-11-26 13:50:24 -0500206 If the command fails, and error_code_ok is True, and the target dir is not
207 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Mike Frysinger02e1e072013-11-10 22:11:34 -0500209 Args:
David James88e6f032013-03-02 08:13:20 -0800210 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
211 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800212 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500213 result = self.device.RunCommand(MOUNT_RW_COMMAND,
214 error_code_ok=error_code_ok,
215 capture_output=True)
216 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400217 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500218 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700219
Ryan Cui7193a7e2013-04-26 14:15:19 -0700220 def _GetDeviceInfo(self):
221 steps = [
222 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
223 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
224 ]
225 return_values = parallel.RunParallelSteps(steps, return_values=True)
226 return DeviceInfo(*return_values)
227
228 def _CheckDeviceFreeSpace(self, device_info):
229 """See if target device has enough space for Chrome.
230
Mike Frysinger02e1e072013-11-10 22:11:34 -0500231 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700232 device_info: A DeviceInfo named tuple.
233 """
234 effective_free = device_info.target_dir_size + device_info.target_fs_free
235 staging_size = self._GetStagingDirSize()
236 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700237 raise DeployFailure(
238 'Not enough free space on the device. Required: %s MiB, '
239 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700240 if device_info.target_fs_free < (100 * 1024):
241 logging.warning('The device has less than 100MB free. deploy_chrome may '
242 'hang during the transfer.')
243
Satoru Takabayashif2893002017-06-20 14:52:48 +0900244 def _ShouldUseCompression(self):
245 """Checks if compression should be used for rsync."""
246 if self.options.compress == 'always':
247 return True
248 elif self.options.compress == 'never':
249 return False
250 elif self.options.compress == 'auto':
251 return not self.device.HasGigabitEthernet()
252
Ryan Cui3045c5d2012-07-13 18:00:33 -0700253 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800254 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ilja H. Friedel0ab63e12017-03-28 13:29:48 -0700255 # CopyToDevice will fall back to scp if rsync is corrupted on stateful.
256 # This does not work for deploy.
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900257 if not self.device.HasRsync():
258 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700259 'rsync is not found on the device.\n'
260 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500261 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
262 self.options.target_dir,
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700263 mode='rsync', inplace=True,
Satoru Takabayashif2893002017-06-20 14:52:48 +0900264 compress=self._ShouldUseCompression(),
Steven Bennettsd57a72a2017-03-20 12:54:00 -0700265 debug_level=logging.INFO,
Robert Flack1dc7ea82014-11-26 13:50:24 -0500266 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800267
268 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800269 if p.mode:
270 # Set mode if necessary.
Mike Frysinger24151262017-10-03 02:50:01 -0400271 self.device.RunCommand(
272 ['chmod', '%o' % p.mode,
273 '%s/%s' % (self.options.target_dir,
274 p.src if not p.dest else p.dest)])
Steve Funge984a532013-11-25 17:09:25 -0800275
Daniel Erat0fce5bf2017-09-28 17:29:23 -0700276 # Send SIGHUP to dbus-daemon to tell it to reload its configs. This won't
277 # pick up major changes (bus type, logging, etc.), but all we care about is
278 # getting the latest policy from /opt/google/chrome/dbus so that Chrome will
279 # be authorized to take ownership of its service names.
Mike Frysinger24151262017-10-03 02:50:01 -0400280 self.device.RunCommand(['killall', '-HUP', 'dbus-daemon'],
281 error_code_ok=True)
Justin TerAvestfac210e2017-04-13 11:39:00 -0600282
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800283 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800284 logging.info('Starting UI...')
Mike Frysinger24151262017-10-03 02:50:01 -0400285 self.device.RunCommand(['start', 'ui'])
Ryan Cui3045c5d2012-07-13 18:00:33 -0700286
David James88e6f032013-03-02 08:13:20 -0800287 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700288 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800289 logging.info('Testing connection to the device...')
Mike Frysinger24151262017-10-03 02:50:01 -0400290 self.device.RunCommand(['true'])
David James88e6f032013-03-02 08:13:20 -0800291 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700292 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800293 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700294
Steve Funge984a532013-11-25 17:09:25 -0800295 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700296 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700297 def BinaryExists(filename):
298 """Checks if the passed-in file is present in the build directory."""
299 return os.path.exists(os.path.join(self.options.build_dir, filename))
300
Daniel Erat9813f0e2014-11-12 11:00:28 -0700301 # Handle non-Chrome deployments.
302 if not BinaryExists('chrome'):
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700303 if BinaryExists('app_shell'):
Daniel Erat9813f0e2014-11-12 11:00:28 -0700304 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
305
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700306 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Eratf53bd3a2016-12-02 11:28:36 -0700307 # app_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700308 self.options.startui = False
309
David James88e6f032013-03-02 08:13:20 -0800310 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800311 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
312 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800313
Thiago Goncales12793312013-05-23 11:26:17 -0700314 def _MountTarget(self):
315 logging.info('Mounting Chrome...')
316
317 # Create directory if does not exist
Mike Frysinger24151262017-10-03 02:50:01 -0400318 self.device.RunCommand(['mkdir', '-p', '--mode', '0775',
319 self.options.mount_dir])
Pawel Osciakfb200f52014-12-21 13:42:05 +0900320 # Umount the existing mount on mount_dir if present first
David Haddockc7cecc72017-10-24 02:38:25 +0000321 cmd = 'if mountpoint -q %(dir)s; then umount %(dir)s; fi'
322 self.device.RunCommand(cmd % {'dir': self.options.mount_dir}, shell=True)
Mike Frysinger24151262017-10-03 02:50:01 -0400323 self.device.RunCommand(['mount', '--rbind', self.options.target_dir,
324 self.options.mount_dir])
Thiago Goncales12793312013-05-23 11:26:17 -0700325 # Chrome needs partition to have exec and suid flags set
Mike Frysinger24151262017-10-03 02:50:01 -0400326 self.device.RunCommand(['mount', '-o', 'remount,exec,suid',
327 self.options.mount_dir])
Robert Flack1dc7ea82014-11-26 13:50:24 -0500328
329 def Cleanup(self):
330 """Clean up RemoteDevice."""
331 if not self.options.staging_only:
332 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700333
David James88e6f032013-03-02 08:13:20 -0800334 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800335 self._CheckDeployType()
336
David James88e6f032013-03-02 08:13:20 -0800337 # If requested, just do the staging step.
338 if self.options.staging_only:
339 self._PrepareStagingDir()
340 return 0
341
342 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800343 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800344 steps = [self._GetDeviceInfo, self._CheckConnection,
345 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
346 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700347 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
348 return_values=True)
349 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800350
Robert Flack1dc7ea82014-11-26 13:50:24 -0500351 # If we're trying to deploy to a dir which is not writable and we failed
352 # to mark the rootfs as writable, try disabling rootfs verification.
353 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800354 self._DisableRootfsVerification()
355
Thiago Goncales12793312013-05-23 11:26:17 -0700356 if self.options.mount_dir is not None:
357 self._MountTarget()
358
David James88e6f032013-03-02 08:13:20 -0800359 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700360 self._Deploy()
361
362
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700363def ValidateStagingFlags(value):
364 """Convert formatted string to dictionary."""
365 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700366
367
Steven Bennetts368c3e52016-09-23 13:05:21 -0700368def ValidateGnArgs(value):
369 """Convert GN_ARGS-formatted string to dictionary."""
370 return gn_helpers.FromGNArgs(value)
371
372
Ryan Cuie535b172012-10-19 18:25:03 -0700373def _CreateParser():
374 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400375 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700376
Ryan Cuia56a71e2012-10-18 18:40:35 -0700377 # TODO(rcui): Have this use the UI-V2 format of having source and target
378 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400379 parser.add_argument('--force', action='store_true', default=False,
380 help='Skip all prompts (i.e., for disabling of rootfs '
381 'verification). This may result in the target '
382 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800383 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400384 parser.add_argument('--board', default=sdk_board_env,
385 help="The board the Chrome build is targeted for. When "
386 "in a 'cros chrome-sdk' shell, defaults to the SDK "
387 "board.")
388 parser.add_argument('--build-dir', type='path',
389 help='The directory with Chrome build artifacts to '
390 'deploy from. Typically of format '
391 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700392 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400393 parser.add_argument('--target-dir', type='path',
394 default=None,
395 help='Target directory on device to deploy Chrome into.')
396 parser.add_argument('-g', '--gs-path', type='gs_path',
397 help='GS path that contains the chrome to deploy.')
398 parser.add_argument('--nostartui', action='store_false', dest='startui',
399 default=True,
400 help="Don't restart the ui daemon after deployment.")
401 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
402 default=True,
403 help="Don't strip binaries during deployment. Warning: "
404 'the resulting binaries will be very large!')
405 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
406 help='Port of the target device to connect to.')
407 parser.add_argument('-t', '--to',
408 help='The IP address of the CrOS device to deploy to.')
409 parser.add_argument('-v', '--verbose', action='store_true', default=False,
410 help='Show more debug output.')
411 parser.add_argument('--mount-dir', type='path', default=None,
412 help='Deploy Chrome in target directory and bind it '
413 'to the directory specified by this flag.'
414 'Any existing mount on this directory will be '
415 'umounted first.')
416 parser.add_argument('--mount', action='store_true', default=False,
417 help='Deploy Chrome to default target directory and bind '
418 'it to the default mount directory.'
419 'Any existing mount on this directory will be '
420 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700421
Mike Frysingerc3061a62015-06-04 04:16:18 -0400422 group = parser.add_argument_group('Advanced Options')
423 group.add_argument('-l', '--local-pkg-path', type='path',
424 help='Path to local chrome prebuilt package to deploy.')
425 group.add_argument('--sloppy', action='store_true', default=False,
426 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700427 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400428 help=('Extra flags to control staging. Valid flags are - '
429 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700430 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400431 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700432 help='Deprecated. Default behavior is "strict". Use '
433 '--sloppy to omit warnings for missing optional '
434 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400435 group.add_argument('--strip-flags', default=None,
436 help="Flags to call the 'strip' binutil tool with. "
437 "Overrides the default arguments.")
438 group.add_argument('--ping', action='store_true', default=False,
439 help='Ping the device before connection attempt.')
Achuith Bhandarkar2f8352f2017-06-02 12:47:18 -0700440 group.add_argument('--process-timeout', type=int,
441 default=KILL_PROC_MAX_WAIT,
442 help='Timeout for process shutdown.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700443
Mike Frysingerc3061a62015-06-04 04:16:18 -0400444 group = parser.add_argument_group(
445 'Metadata Overrides (Advanced)',
446 description='Provide all of these overrides in order to remove '
447 'dependencies on metadata.json existence.')
448 group.add_argument('--target-tc', action='store', default=None,
449 help='Override target toolchain name, e.g. '
450 'x86_64-cros-linux-gnu')
451 group.add_argument('--toolchain-url', action='store', default=None,
452 help='Override toolchain url format pattern, e.g. '
453 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700454
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700455 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
456 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
457 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400458 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700459
460 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
461 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
462 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
463 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
464 help=argparse.SUPPRESS)
465
Ryan Cuia56a71e2012-10-18 18:40:35 -0700466 # Path of an empty directory to stage chrome artifacts to. Defaults to a
467 # temporary directory that is removed when the script finishes. If the path
468 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400469 parser.add_argument('--staging-dir', type='path', default=None,
470 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700471 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400472 parser.add_argument('--staging-only', action='store_true', default=False,
473 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700474 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
475 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
476 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400477 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Satoru Takabayashif2893002017-06-20 14:52:48 +0900478 parser.add_argument('--compress', action='store', default='auto',
479 choices=('always', 'never', 'auto'),
480 help='Choose the data compression behavior. Default '
481 'is set to "auto", that disables compression if '
482 'the target device has a gigabit ethernet port.')
Ryan Cuie535b172012-10-19 18:25:03 -0700483 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700484
Ryan Cuie535b172012-10-19 18:25:03 -0700485
486def _ParseCommandLine(argv):
487 """Parse args, and run environment-independent checks."""
488 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400489 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700490
Ryan Cuia56a71e2012-10-18 18:40:35 -0700491 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
492 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800493 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700494 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
495 parser.error('Cannot specify both --build_dir and '
496 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800497 if options.build_dir and not options.board:
498 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700499 if options.gs_path and options.local_pkg_path:
500 parser.error('Cannot specify both --gs-path and --local-pkg-path')
501 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700502 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700503 if options.staging_flags and not options.build_dir:
504 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700505
Steven Bennetts46a84c32016-08-12 15:20:46 -0700506 if options.strict:
507 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700508 if options.gyp_defines:
509 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700510
511 if options.mount or options.mount_dir:
512 if not options.target_dir:
513 options.target_dir = _CHROME_DIR_MOUNT
514 else:
515 if not options.target_dir:
516 options.target_dir = _CHROME_DIR
517
518 if options.mount and not options.mount_dir:
519 options.mount_dir = _CHROME_DIR
520
Mike Frysingerc3061a62015-06-04 04:16:18 -0400521 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700522
523
Mike Frysingerc3061a62015-06-04 04:16:18 -0400524def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800525 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700526
527 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400528 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700529 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700530 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
531 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
532
Steven Bennetts368c3e52016-09-23 13:05:21 -0700533 if not options.gn_args:
534 gn_env = os.getenv('GN_ARGS')
535 if gn_env is not None:
536 options.gn_args = gn_helpers.FromGNArgs(gn_env)
537 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700538
Steven Bennetts60600462016-05-12 10:40:20 -0700539 if not options.staging_flags:
540 use_env = os.getenv('USE')
541 if use_env is not None:
542 options.staging_flags = ' '.join(set(use_env.split()).intersection(
543 chrome_util.STAGING_FLAGS))
544 logging.info('Staging flags taken from USE in environment: %s',
545 options.staging_flags)
546
Ryan Cuia56a71e2012-10-18 18:40:35 -0700547
Ryan Cui504db722013-01-22 11:48:01 -0800548def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549 """Get the chrome prebuilt tarball from GS.
550
Mike Frysinger02e1e072013-11-10 22:11:34 -0500551 Returns:
552 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700553 """
David James9374aac2013-10-08 16:00:17 -0700554 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500555 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800556 files = [found for found in files if
557 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
558 if not files:
559 raise Exception('No chrome package found at %s' % gs_path)
560 elif len(files) > 1:
561 # - Users should provide us with a direct link to either a stripped or
562 # unstripped chrome package.
563 # - In the case of being provided with an archive directory, where both
564 # stripped and unstripped chrome available, use the stripped chrome
565 # package.
566 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
567 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
568 files = [f for f in files if not 'unstripped' in f]
569 assert len(files) == 1
570 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800571
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800572 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800573 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800574 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
575 chrome_path = os.path.join(tempdir, filename)
576 assert os.path.exists(chrome_path)
577 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700578
579
Ryan Cuif890a3e2013-03-07 18:57:06 -0800580@contextlib.contextmanager
581def _StripBinContext(options):
582 if not options.dostrip:
583 yield None
584 elif options.strip_bin:
585 yield options.strip_bin
586 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800587 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800588 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700589 with sdk.Prepare(components=components, target_tc=options.target_tc,
590 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800591 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
592 constants.CHROME_ENV_FILE)
593 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
594 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
595 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800596 yield strip_bin
597
598
Steve Funge984a532013-11-25 17:09:25 -0800599def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
600 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800601 """Place the necessary files in the staging directory.
602
603 The staging directory is the directory used to rsync the build artifacts over
604 to the device. Only the necessary Chrome build artifacts are put into the
605 staging directory.
606 """
Ryan Cui5866be02013-03-18 14:12:00 -0700607 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400608 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800609 if options.build_dir:
610 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700611 strip_flags = (None if options.strip_flags is None else
612 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800613 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700614 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700615 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700616 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800617 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700618 else:
619 pkg_path = options.local_pkg_path
620 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800621 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
622 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700623
624 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800625 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700626 # Extract only the ./opt/google/chrome contents, directly into the staging
627 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800628 if pkg_path[-4:] == '.zip':
629 cros_build_lib.DebugRunCommand(
630 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
631 staging_dir])
632 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
633 shutil.move(filename, staging_dir)
634 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
635 else:
636 cros_build_lib.DebugRunCommand(
637 ['tar', '--strip-components', '4', '--extract',
638 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
639 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700640
Ryan Cui71aa8de2013-04-19 16:12:55 -0700641
Ryan Cui3045c5d2012-07-13 18:00:33 -0700642def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400643 options = _ParseCommandLine(argv)
644 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700645
646 # Set cros_build_lib debug level to hide RunCommand spew.
647 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700648 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700649 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800650 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700651
Aviv Keshet01a82e92017-03-02 17:39:59 -0800652 with osutils.TempDir(set_global=True) as tempdir:
653 staging_dir = options.staging_dir
654 if not staging_dir:
655 staging_dir = os.path.join(tempdir, 'chrome')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700656
Aviv Keshet01a82e92017-03-02 17:39:59 -0800657 deploy = DeployChrome(options, tempdir, staging_dir)
658 try:
659 deploy.Perform()
660 except failures_lib.StepFailure as ex:
661 raise SystemExit(str(ex).strip())
662 deploy.Cleanup()