blob: a0877c85c6e1727fdf8154c40bbd9b1eaf147223 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Don Garrett88b8d782014-05-13 17:30:55 -070031from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070032from chromite.cbuildbot import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070042from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
Ryan Cuie535b172012-10-19 18:25:03 -070053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080054LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070055
Steve Funge984a532013-11-25 17:09:25 -080056_ANDROID_DIR = '/system/chrome'
57_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
58
David James2cb34002013-03-01 18:42:40 -080059_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070060_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080061
Pawel Osciakc1bb2742014-12-29 16:32:33 +090062_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
63 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070064_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
65_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070066
Steve Funge984a532013-11-25 17:09:25 -080067DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080068
Mike Frysingere65f3752014-12-08 00:46:39 -050069
Ryan Cui3045c5d2012-07-13 18:00:33 -070070def _UrlBaseName(url):
71 """Return the last component of the URL."""
72 return url.rstrip('/').rpartition('/')[-1]
73
74
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070075class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080076 """Raised whenever the deploy fails."""
77
78
Ryan Cui7193a7e2013-04-26 14:15:19 -070079DeviceInfo = collections.namedtuple(
80 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
81
82
Ryan Cui3045c5d2012-07-13 18:00:33 -070083class DeployChrome(object):
84 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050085
Ryan Cuia56a71e2012-10-18 18:40:35 -070086 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070087 """Initialize the class.
88
Mike Frysinger02e1e072013-11-10 22:11:34 -050089 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040090 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070091 tempdir: Scratch space for the class. Caller has responsibility to clean
92 it up.
Steve Funge984a532013-11-25 17:09:25 -080093 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070094 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070095 self.tempdir = tempdir
96 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070097 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050098 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090099 self.device = remote.RemoteDevice(options.to, port=options.port,
100 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500101 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700102
kylechardd2371f2016-04-11 11:09:31 -0400103 if self.options.mash:
104 self.copy_paths = chrome_util.GetCopyPaths('mash')
105 else:
106 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800107 self.chrome_dir = _CHROME_DIR
108
Ryan Cui7193a7e2013-04-26 14:15:19 -0700109 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500110 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700111 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800112 value = line.split()[3]
113 multipliers = {
114 'G': 1024 * 1024 * 1024,
115 'M': 1024 * 1024,
116 'K': 1024,
117 }
118 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119
120 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 result = self.device.RunCommand('du -ks %s' % remote_dir,
122 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700123 return int(result.output.split()[0])
124
125 def _GetStagingDirSize(self):
126 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800127 redirect_stdout=True,
128 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700129 return int(result.output.split()[0])
130
Ryan Cui3045c5d2012-07-13 18:00:33 -0700131 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500132 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
133 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700134 return result.returncode == 0
135
136 def _DisableRootfsVerification(self):
137 if not self.options.force:
138 logging.error('Detected that the device has rootfs verification enabled.')
139 logging.info('This script can automatically remove the rootfs '
140 'verification, which requires that it reboot the device.')
141 logging.info('Make sure the device is in developer mode!')
142 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700143 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800144 # Since we stopped Chrome earlier, it's good form to start it up again.
145 if self.options.startui:
146 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500147 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800148 raise DeployFailure('Need rootfs verification to be disabled. '
149 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700150
151 logging.info('Removing rootfs verification from %s', self.options.to)
152 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
153 # Use --force to bypass the checks.
154 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
155 '--remove_rootfs_verification --force')
156 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500157 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 # A reboot in developer mode takes a while (and has delays), so the user
160 # will have time to read and act on the USB boot instructions below.
161 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500162 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700163
David James88e6f032013-03-02 08:13:20 -0800164 # Now that the machine has been rebooted, we need to kill Chrome again.
165 self._KillProcsIfNeeded()
166
167 # Make sure the rootfs is writable now.
168 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700169
170 def _CheckUiJobStarted(self):
171 # status output is in the format:
172 # <job_name> <status> ['process' <pid>].
173 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500175 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800176 except cros_build_lib.RunCommandError as e:
177 if 'Unknown job' in e.result.error:
178 return False
179 else:
180 raise e
181
Ryan Cui3045c5d2012-07-13 18:00:33 -0700182 return result.output.split()[1].split('/')[0] == 'start'
183
184 def _KillProcsIfNeeded(self):
185 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800186 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500187 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188
189 # Developers sometimes run session_manager manually, in which case we'll
190 # need to help shut the chrome processes down.
191 try:
David James3432acd2013-11-27 10:02:18 -0800192 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700193 while self._ChromeFileInUse():
194 logging.warning('The chrome binary on the device is in use.')
195 logging.warning('Killing chrome and session_manager processes...\n')
196
Robert Flack1dc7ea82014-11-26 13:50:24 -0500197 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500198 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700199 # Wait for processes to actually terminate
200 time.sleep(POST_KILL_WAIT)
201 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800202 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800203 msg = ('Could not kill processes after %s seconds. Please exit any '
204 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400221 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Ryan Cui3045c5d2012-07-13 18:00:33 -0700248 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800249 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700250 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800251 dest_path = _CHROME_DIR
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900252 if not self.device.HasRsync():
253 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700254 'rsync is not found on the device.\n'
255 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500256 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
257 self.options.target_dir,
258 inplace=True, debug_level=logging.INFO,
259 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800260
261 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800262 if p.mode:
263 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500264 self.device.RunCommand('chmod %o %s/%s' % (
265 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800266
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800267 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800268 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500269 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270
David James88e6f032013-03-02 08:13:20 -0800271 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800273 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500274 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800275 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700276 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800277 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700278
Steve Funge984a532013-11-25 17:09:25 -0800279 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700280 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700281 def BinaryExists(filename):
282 """Checks if the passed-in file is present in the build directory."""
283 return os.path.exists(os.path.join(self.options.build_dir, filename))
284
Daniel Erat9813f0e2014-11-12 11:00:28 -0700285 # Handle non-Chrome deployments.
286 if not BinaryExists('chrome'):
287 if BinaryExists('envoy_shell'):
288 self.copy_paths = chrome_util.GetCopyPaths('envoy')
289 elif BinaryExists('app_shell'):
290 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
291
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700292 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700293 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700294 self.options.startui = False
295
David James88e6f032013-03-02 08:13:20 -0800296 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800297 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
298 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800299
Thiago Goncales12793312013-05-23 11:26:17 -0700300 def _MountTarget(self):
301 logging.info('Mounting Chrome...')
302
303 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500304 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500305 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900306 # Umount the existing mount on mount_dir if present first
307 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
308 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500309 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
310 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700311 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500312 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
313
314 def Cleanup(self):
315 """Clean up RemoteDevice."""
316 if not self.options.staging_only:
317 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700318
David James88e6f032013-03-02 08:13:20 -0800319 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800320 self._CheckDeployType()
321
David James88e6f032013-03-02 08:13:20 -0800322 # If requested, just do the staging step.
323 if self.options.staging_only:
324 self._PrepareStagingDir()
325 return 0
326
327 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800328 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800329 steps = [self._GetDeviceInfo, self._CheckConnection,
330 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
331 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700332 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
333 return_values=True)
334 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800335
Robert Flack1dc7ea82014-11-26 13:50:24 -0500336 # If we're trying to deploy to a dir which is not writable and we failed
337 # to mark the rootfs as writable, try disabling rootfs verification.
338 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800339 self._DisableRootfsVerification()
340
Thiago Goncales12793312013-05-23 11:26:17 -0700341 if self.options.mount_dir is not None:
342 self._MountTarget()
343
David James88e6f032013-03-02 08:13:20 -0800344 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700345 self._Deploy()
346
347
Mike Frysingerc3061a62015-06-04 04:16:18 -0400348def ValidateGypDefines(value):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700349 """Convert GYP_DEFINES-formatted string to dictionary."""
350 return chrome_util.ProcessGypDefines(value)
351
352
Steven Bennetts368c3e52016-09-23 13:05:21 -0700353def ValidateGnArgs(value):
354 """Convert GN_ARGS-formatted string to dictionary."""
355 return gn_helpers.FromGNArgs(value)
356
357
Ryan Cuie535b172012-10-19 18:25:03 -0700358def _CreateParser():
359 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400360 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700361
Ryan Cuia56a71e2012-10-18 18:40:35 -0700362 # TODO(rcui): Have this use the UI-V2 format of having source and target
363 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400364 parser.add_argument('--force', action='store_true', default=False,
365 help='Skip all prompts (i.e., for disabling of rootfs '
366 'verification). This may result in the target '
367 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800368 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400369 parser.add_argument('--board', default=sdk_board_env,
370 help="The board the Chrome build is targeted for. When "
371 "in a 'cros chrome-sdk' shell, defaults to the SDK "
372 "board.")
373 parser.add_argument('--build-dir', type='path',
374 help='The directory with Chrome build artifacts to '
375 'deploy from. Typically of format '
376 '<chrome_root>/out/Debug. When this option is used, '
377 'the GYP_DEFINES environment variable must be set.')
378 parser.add_argument('--target-dir', type='path',
379 default=None,
380 help='Target directory on device to deploy Chrome into.')
381 parser.add_argument('-g', '--gs-path', type='gs_path',
382 help='GS path that contains the chrome to deploy.')
383 parser.add_argument('--nostartui', action='store_false', dest='startui',
384 default=True,
385 help="Don't restart the ui daemon after deployment.")
386 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
387 default=True,
388 help="Don't strip binaries during deployment. Warning: "
389 'the resulting binaries will be very large!')
390 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
391 help='Port of the target device to connect to.')
392 parser.add_argument('-t', '--to',
393 help='The IP address of the CrOS device to deploy to.')
394 parser.add_argument('-v', '--verbose', action='store_true', default=False,
395 help='Show more debug output.')
396 parser.add_argument('--mount-dir', type='path', default=None,
397 help='Deploy Chrome in target directory and bind it '
398 'to the directory specified by this flag.'
399 'Any existing mount on this directory will be '
400 'umounted first.')
401 parser.add_argument('--mount', action='store_true', default=False,
402 help='Deploy Chrome to default target directory and bind '
403 'it to the default mount directory.'
404 'Any existing mount on this directory will be '
405 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700406
Mike Frysingerc3061a62015-06-04 04:16:18 -0400407 group = parser.add_argument_group('Advanced Options')
408 group.add_argument('-l', '--local-pkg-path', type='path',
409 help='Path to local chrome prebuilt package to deploy.')
410 group.add_argument('--sloppy', action='store_true', default=False,
411 help='Ignore when mandatory artifacts are missing.')
412 group.add_argument('--staging-flags', default=None, type=ValidateGypDefines,
413 help=('Extra flags to control staging. Valid flags are - '
414 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700415 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400416 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700417 help='Deprecated. Default behavior is "strict". Use '
418 '--sloppy to omit warnings for missing optional '
419 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400420 group.add_argument('--strip-flags', default=None,
421 help="Flags to call the 'strip' binutil tool with. "
422 "Overrides the default arguments.")
423 group.add_argument('--ping', action='store_true', default=False,
424 help='Ping the device before connection attempt.')
kylechardd2371f2016-04-11 11:09:31 -0400425 group.add_argument('--mash', action='store_true', default=False,
426 help='Copy additional files for mus+ash. Will not fit in '
427 'the default target-dir.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700428
Mike Frysingerc3061a62015-06-04 04:16:18 -0400429 group = parser.add_argument_group(
430 'Metadata Overrides (Advanced)',
431 description='Provide all of these overrides in order to remove '
432 'dependencies on metadata.json existence.')
433 group.add_argument('--target-tc', action='store', default=None,
434 help='Override target toolchain name, e.g. '
435 'x86_64-cros-linux-gnu')
436 group.add_argument('--toolchain-url', action='store', default=None,
437 help='Override toolchain url format pattern, e.g. '
438 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700439
Ryan Cuif890a3e2013-03-07 18:57:06 -0800440 # GYP_DEFINES that Chrome was built with. Influences which files are staged
441 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
Steven Bennetts368c3e52016-09-23 13:05:21 -0700442 # enviroment variable. WILL BE DEPRECATED.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400443 parser.add_argument('--gyp-defines', default=None, type=ValidateGypDefines,
444 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700445
446 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
447 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
448 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
449 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
450 help=argparse.SUPPRESS)
451
Ryan Cuia56a71e2012-10-18 18:40:35 -0700452 # Path of an empty directory to stage chrome artifacts to. Defaults to a
453 # temporary directory that is removed when the script finishes. If the path
454 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400455 parser.add_argument('--staging-dir', type='path', default=None,
456 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700457 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400458 parser.add_argument('--staging-only', action='store_true', default=False,
459 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700460 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
461 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
462 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400463 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Ryan Cuie535b172012-10-19 18:25:03 -0700464 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700465
Ryan Cuie535b172012-10-19 18:25:03 -0700466
467def _ParseCommandLine(argv):
468 """Parse args, and run environment-independent checks."""
469 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400470 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700471
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
473 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800474 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700475 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
476 parser.error('Cannot specify both --build_dir and '
477 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800478 if options.build_dir and not options.board:
479 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700480 if options.gs_path and options.local_pkg_path:
481 parser.error('Cannot specify both --gs-path and --local-pkg-path')
482 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700483 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700484 if options.staging_flags and not options.build_dir:
485 parser.error('--staging-flags require --build-dir to be set.')
486 if options.strict:
487 logging.warning('--strict is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700488
489 if options.mount or options.mount_dir:
490 if not options.target_dir:
491 options.target_dir = _CHROME_DIR_MOUNT
492 else:
493 if not options.target_dir:
494 options.target_dir = _CHROME_DIR
495
496 if options.mount and not options.mount_dir:
497 options.mount_dir = _CHROME_DIR
498
Mike Frysingerc3061a62015-06-04 04:16:18 -0400499 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700500
501
Mike Frysingerc3061a62015-06-04 04:16:18 -0400502def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800503 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700504
505 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400506 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700507 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700508 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
509 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
510
Ryan Cuib623e7b2013-03-14 12:54:11 -0700511 if not options.gyp_defines:
Steven Bennetts60600462016-05-12 10:40:20 -0700512 gyp_env = os.getenv('GYP_DEFINES')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700513 if gyp_env is not None:
514 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700515 logging.info('GYP_DEFINES taken from environment: %s',
516 options.gyp_defines)
517
518 if not options.gn_args:
519 gn_env = os.getenv('GN_ARGS')
520 if gn_env is not None:
521 options.gn_args = gn_helpers.FromGNArgs(gn_env)
522 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700523
Steven Bennetts60600462016-05-12 10:40:20 -0700524 if not options.staging_flags:
525 use_env = os.getenv('USE')
526 if use_env is not None:
527 options.staging_flags = ' '.join(set(use_env.split()).intersection(
528 chrome_util.STAGING_FLAGS))
529 logging.info('Staging flags taken from USE in environment: %s',
530 options.staging_flags)
531
Ryan Cuia56a71e2012-10-18 18:40:35 -0700532
Ryan Cui504db722013-01-22 11:48:01 -0800533def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700534 """Get the chrome prebuilt tarball from GS.
535
Mike Frysinger02e1e072013-11-10 22:11:34 -0500536 Returns:
537 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700538 """
David James9374aac2013-10-08 16:00:17 -0700539 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500540 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800541 files = [found for found in files if
542 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
543 if not files:
544 raise Exception('No chrome package found at %s' % gs_path)
545 elif len(files) > 1:
546 # - Users should provide us with a direct link to either a stripped or
547 # unstripped chrome package.
548 # - In the case of being provided with an archive directory, where both
549 # stripped and unstripped chrome available, use the stripped chrome
550 # package.
551 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
552 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
553 files = [f for f in files if not 'unstripped' in f]
554 assert len(files) == 1
555 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800556
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800557 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800558 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800559 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
560 chrome_path = os.path.join(tempdir, filename)
561 assert os.path.exists(chrome_path)
562 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700563
564
Ryan Cuif890a3e2013-03-07 18:57:06 -0800565@contextlib.contextmanager
566def _StripBinContext(options):
567 if not options.dostrip:
568 yield None
569 elif options.strip_bin:
570 yield options.strip_bin
571 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800572 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800573 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700574 with sdk.Prepare(components=components, target_tc=options.target_tc,
575 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800576 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
577 constants.CHROME_ENV_FILE)
578 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
579 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
580 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800581 yield strip_bin
582
583
Steve Funge984a532013-11-25 17:09:25 -0800584def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
585 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800586 """Place the necessary files in the staging directory.
587
588 The staging directory is the directory used to rsync the build artifacts over
589 to the device. Only the necessary Chrome build artifacts are put into the
590 staging directory.
591 """
Ryan Cui5866be02013-03-18 14:12:00 -0700592 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400593 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800594 if options.build_dir:
595 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700596 strip_flags = (None if options.strip_flags is None else
597 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800598 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700599 staging_dir, options.build_dir, strip_bin,
David Jamesa6e08892013-03-01 13:34:11 -0800600 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700601 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800602 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700603 else:
604 pkg_path = options.local_pkg_path
605 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800606 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
607 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700608
609 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800610 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700611 # Extract only the ./opt/google/chrome contents, directly into the staging
612 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800613 if pkg_path[-4:] == '.zip':
614 cros_build_lib.DebugRunCommand(
615 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
616 staging_dir])
617 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
618 shutil.move(filename, staging_dir)
619 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
620 else:
621 cros_build_lib.DebugRunCommand(
622 ['tar', '--strip-components', '4', '--extract',
623 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
624 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700625
Ryan Cui71aa8de2013-04-19 16:12:55 -0700626
Ryan Cui3045c5d2012-07-13 18:00:33 -0700627def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400628 options = _ParseCommandLine(argv)
629 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700630
631 # Set cros_build_lib debug level to hide RunCommand spew.
632 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700633 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700634 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800635 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700636
Ryan Cui71aa8de2013-04-19 16:12:55 -0700637 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700638 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
639 if cmd_stats:
640 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700641
Ryan Cui71aa8de2013-04-19 16:12:55 -0700642 with osutils.TempDir(set_global=True) as tempdir:
643 staging_dir = options.staging_dir
644 if not staging_dir:
645 staging_dir = os.path.join(tempdir, 'chrome')
646
647 deploy = DeployChrome(options, tempdir, staging_dir)
648 try:
649 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700650 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700651 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500652 deploy.Cleanup()