blob: 8a9e110c0d8cf448d67214bc72871476351d638a [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Don Garrett88b8d782014-05-13 17:30:55 -070031from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070032from chromite.cbuildbot import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070042from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Ryan Cui3045c5d2012-07-13 18:00:33 -070046KERNEL_A_PARTITION = 2
47KERNEL_B_PARTITION = 4
48
49KILL_PROC_MAX_WAIT = 10
50POST_KILL_WAIT = 2
51
Ryan Cuie535b172012-10-19 18:25:03 -070052MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080053LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070054
Steve Funge984a532013-11-25 17:09:25 -080055_ANDROID_DIR = '/system/chrome'
56_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
57
David James2cb34002013-03-01 18:42:40 -080058_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070059_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080060
Pawel Osciakc1bb2742014-12-29 16:32:33 +090061_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
62 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070063_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
64_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070065
Steve Funge984a532013-11-25 17:09:25 -080066DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080067
Mike Frysingere65f3752014-12-08 00:46:39 -050068
Ryan Cui3045c5d2012-07-13 18:00:33 -070069def _UrlBaseName(url):
70 """Return the last component of the URL."""
71 return url.rstrip('/').rpartition('/')[-1]
72
73
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070074class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080075 """Raised whenever the deploy fails."""
76
77
Ryan Cui7193a7e2013-04-26 14:15:19 -070078DeviceInfo = collections.namedtuple(
79 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
80
81
Ryan Cui3045c5d2012-07-13 18:00:33 -070082class DeployChrome(object):
83 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050084
Ryan Cuia56a71e2012-10-18 18:40:35 -070085 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070086 """Initialize the class.
87
Mike Frysinger02e1e072013-11-10 22:11:34 -050088 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040089 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070090 tempdir: Scratch space for the class. Caller has responsibility to clean
91 it up.
Steve Funge984a532013-11-25 17:09:25 -080092 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070093 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070094 self.tempdir = tempdir
95 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070096 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050097 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090098 self.device = remote.RemoteDevice(options.to, port=options.port,
99 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500100 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700101
kylechardd2371f2016-04-11 11:09:31 -0400102 if self.options.mash:
103 self.copy_paths = chrome_util.GetCopyPaths('mash')
104 else:
105 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800106 self.chrome_dir = _CHROME_DIR
107
Ryan Cui7193a7e2013-04-26 14:15:19 -0700108 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500109 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800111 value = line.split()[3]
112 multipliers = {
113 'G': 1024 * 1024 * 1024,
114 'M': 1024 * 1024,
115 'K': 1024,
116 }
117 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700118
119 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500120 result = self.device.RunCommand('du -ks %s' % remote_dir,
121 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122 return int(result.output.split()[0])
123
124 def _GetStagingDirSize(self):
125 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800126 redirect_stdout=True,
127 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700128 return int(result.output.split()[0])
129
Ryan Cui3045c5d2012-07-13 18:00:33 -0700130 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500131 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
132 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700133 return result.returncode == 0
134
135 def _DisableRootfsVerification(self):
136 if not self.options.force:
137 logging.error('Detected that the device has rootfs verification enabled.')
138 logging.info('This script can automatically remove the rootfs '
139 'verification, which requires that it reboot the device.')
140 logging.info('Make sure the device is in developer mode!')
141 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700142 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800143 # Since we stopped Chrome earlier, it's good form to start it up again.
144 if self.options.startui:
145 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500146 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800147 raise DeployFailure('Need rootfs verification to be disabled. '
148 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700149
150 logging.info('Removing rootfs verification from %s', self.options.to)
151 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
152 # Use --force to bypass the checks.
153 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
154 '--remove_rootfs_verification --force')
155 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500156 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700157
158 # A reboot in developer mode takes a while (and has delays), so the user
159 # will have time to read and act on the USB boot instructions below.
160 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500161 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700162
David James88e6f032013-03-02 08:13:20 -0800163 # Now that the machine has been rebooted, we need to kill Chrome again.
164 self._KillProcsIfNeeded()
165
166 # Make sure the rootfs is writable now.
167 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700168
169 def _CheckUiJobStarted(self):
170 # status output is in the format:
171 # <job_name> <status> ['process' <pid>].
172 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800173 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500174 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800175 except cros_build_lib.RunCommandError as e:
176 if 'Unknown job' in e.result.error:
177 return False
178 else:
179 raise e
180
Ryan Cui3045c5d2012-07-13 18:00:33 -0700181 return result.output.split()[1].split('/')[0] == 'start'
182
183 def _KillProcsIfNeeded(self):
184 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800185 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500186 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700187
188 # Developers sometimes run session_manager manually, in which case we'll
189 # need to help shut the chrome processes down.
190 try:
David James3432acd2013-11-27 10:02:18 -0800191 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700192 while self._ChromeFileInUse():
193 logging.warning('The chrome binary on the device is in use.')
194 logging.warning('Killing chrome and session_manager processes...\n')
195
Robert Flack1dc7ea82014-11-26 13:50:24 -0500196 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500197 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 # Wait for processes to actually terminate
199 time.sleep(POST_KILL_WAIT)
200 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800201 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800202 msg = ('Could not kill processes after %s seconds. Please exit any '
203 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
204 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700205
David James88e6f032013-03-02 08:13:20 -0800206 def _MountRootfsAsWritable(self, error_code_ok=True):
207 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700208
Robert Flack1dc7ea82014-11-26 13:50:24 -0500209 If the command fails, and error_code_ok is True, and the target dir is not
210 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700211
Mike Frysinger02e1e072013-11-10 22:11:34 -0500212 Args:
David James88e6f032013-03-02 08:13:20 -0800213 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
214 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800215 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500216 result = self.device.RunCommand(MOUNT_RW_COMMAND,
217 error_code_ok=error_code_ok,
218 capture_output=True)
219 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400220 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500221 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
Ryan Cui7193a7e2013-04-26 14:15:19 -0700223 def _GetDeviceInfo(self):
224 steps = [
225 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
226 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
227 ]
228 return_values = parallel.RunParallelSteps(steps, return_values=True)
229 return DeviceInfo(*return_values)
230
231 def _CheckDeviceFreeSpace(self, device_info):
232 """See if target device has enough space for Chrome.
233
Mike Frysinger02e1e072013-11-10 22:11:34 -0500234 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700235 device_info: A DeviceInfo named tuple.
236 """
237 effective_free = device_info.target_dir_size + device_info.target_fs_free
238 staging_size = self._GetStagingDirSize()
239 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700240 raise DeployFailure(
241 'Not enough free space on the device. Required: %s MiB, '
242 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700243 if device_info.target_fs_free < (100 * 1024):
244 logging.warning('The device has less than 100MB free. deploy_chrome may '
245 'hang during the transfer.')
246
Ryan Cui3045c5d2012-07-13 18:00:33 -0700247 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800248 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700249 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800250 dest_path = _CHROME_DIR
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900251 if not self.device.HasRsync():
252 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700253 'rsync is not found on the device.\n'
254 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500255 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
256 self.options.target_dir,
257 inplace=True, debug_level=logging.INFO,
258 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800259
260 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800261 if p.mode:
262 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500263 self.device.RunCommand('chmod %o %s/%s' % (
264 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800265
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800266 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800267 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500268 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700269
David James88e6f032013-03-02 08:13:20 -0800270 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700271 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800272 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500273 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800274 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700275 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800276 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700277
Steve Funge984a532013-11-25 17:09:25 -0800278 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700279 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700280 def BinaryExists(filename):
281 """Checks if the passed-in file is present in the build directory."""
282 return os.path.exists(os.path.join(self.options.build_dir, filename))
283
Daniel Erat9813f0e2014-11-12 11:00:28 -0700284 # Handle non-Chrome deployments.
285 if not BinaryExists('chrome'):
286 if BinaryExists('envoy_shell'):
287 self.copy_paths = chrome_util.GetCopyPaths('envoy')
288 elif BinaryExists('app_shell'):
289 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
290
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700291 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700292 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700293 self.options.startui = False
294
David James88e6f032013-03-02 08:13:20 -0800295 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800296 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
297 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800298
Thiago Goncales12793312013-05-23 11:26:17 -0700299 def _MountTarget(self):
300 logging.info('Mounting Chrome...')
301
302 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500303 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500304 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900305 # Umount the existing mount on mount_dir if present first
306 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
307 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500308 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
309 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700310 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500311 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
312
313 def Cleanup(self):
314 """Clean up RemoteDevice."""
315 if not self.options.staging_only:
316 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700317
David James88e6f032013-03-02 08:13:20 -0800318 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800319 self._CheckDeployType()
320
David James88e6f032013-03-02 08:13:20 -0800321 # If requested, just do the staging step.
322 if self.options.staging_only:
323 self._PrepareStagingDir()
324 return 0
325
326 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800327 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800328 steps = [self._GetDeviceInfo, self._CheckConnection,
329 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
330 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700331 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
332 return_values=True)
333 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800334
Robert Flack1dc7ea82014-11-26 13:50:24 -0500335 # If we're trying to deploy to a dir which is not writable and we failed
336 # to mark the rootfs as writable, try disabling rootfs verification.
337 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800338 self._DisableRootfsVerification()
339
Thiago Goncales12793312013-05-23 11:26:17 -0700340 if self.options.mount_dir is not None:
341 self._MountTarget()
342
David James88e6f032013-03-02 08:13:20 -0800343 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700344 self._Deploy()
345
346
Mike Frysingerc3061a62015-06-04 04:16:18 -0400347def ValidateGypDefines(value):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700348 """Convert GYP_DEFINES-formatted string to dictionary."""
349 return chrome_util.ProcessGypDefines(value)
350
351
Ryan Cuie535b172012-10-19 18:25:03 -0700352def _CreateParser():
353 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400354 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700355
Ryan Cuia56a71e2012-10-18 18:40:35 -0700356 # TODO(rcui): Have this use the UI-V2 format of having source and target
357 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400358 parser.add_argument('--force', action='store_true', default=False,
359 help='Skip all prompts (i.e., for disabling of rootfs '
360 'verification). This may result in the target '
361 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800362 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400363 parser.add_argument('--board', default=sdk_board_env,
364 help="The board the Chrome build is targeted for. When "
365 "in a 'cros chrome-sdk' shell, defaults to the SDK "
366 "board.")
367 parser.add_argument('--build-dir', type='path',
368 help='The directory with Chrome build artifacts to '
369 'deploy from. Typically of format '
370 '<chrome_root>/out/Debug. When this option is used, '
371 'the GYP_DEFINES environment variable must be set.')
372 parser.add_argument('--target-dir', type='path',
373 default=None,
374 help='Target directory on device to deploy Chrome into.')
375 parser.add_argument('-g', '--gs-path', type='gs_path',
376 help='GS path that contains the chrome to deploy.')
377 parser.add_argument('--nostartui', action='store_false', dest='startui',
378 default=True,
379 help="Don't restart the ui daemon after deployment.")
380 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
381 default=True,
382 help="Don't strip binaries during deployment. Warning: "
383 'the resulting binaries will be very large!')
384 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
385 help='Port of the target device to connect to.')
386 parser.add_argument('-t', '--to',
387 help='The IP address of the CrOS device to deploy to.')
388 parser.add_argument('-v', '--verbose', action='store_true', default=False,
389 help='Show more debug output.')
390 parser.add_argument('--mount-dir', type='path', default=None,
391 help='Deploy Chrome in target directory and bind it '
392 'to the directory specified by this flag.'
393 'Any existing mount on this directory will be '
394 'umounted first.')
395 parser.add_argument('--mount', action='store_true', default=False,
396 help='Deploy Chrome to default target directory and bind '
397 'it to the default mount directory.'
398 'Any existing mount on this directory will be '
399 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700400
Mike Frysingerc3061a62015-06-04 04:16:18 -0400401 group = parser.add_argument_group('Advanced Options')
402 group.add_argument('-l', '--local-pkg-path', type='path',
403 help='Path to local chrome prebuilt package to deploy.')
404 group.add_argument('--sloppy', action='store_true', default=False,
405 help='Ignore when mandatory artifacts are missing.')
406 group.add_argument('--staging-flags', default=None, type=ValidateGypDefines,
407 help=('Extra flags to control staging. Valid flags are - '
408 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
409 group.add_argument('--strict', action='store_true', default=False,
410 help='Stage artifacts based on the GYP_DEFINES '
411 'environment variable and --staging-flags, if set. '
412 'Enforce that all optional artifacts are deployed.')
413 group.add_argument('--strip-flags', default=None,
414 help="Flags to call the 'strip' binutil tool with. "
415 "Overrides the default arguments.")
416 group.add_argument('--ping', action='store_true', default=False,
417 help='Ping the device before connection attempt.')
kylechardd2371f2016-04-11 11:09:31 -0400418 group.add_argument('--mash', action='store_true', default=False,
419 help='Copy additional files for mus+ash. Will not fit in '
420 'the default target-dir.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700421
Mike Frysingerc3061a62015-06-04 04:16:18 -0400422 group = parser.add_argument_group(
423 'Metadata Overrides (Advanced)',
424 description='Provide all of these overrides in order to remove '
425 'dependencies on metadata.json existence.')
426 group.add_argument('--target-tc', action='store', default=None,
427 help='Override target toolchain name, e.g. '
428 'x86_64-cros-linux-gnu')
429 group.add_argument('--toolchain-url', action='store', default=None,
430 help='Override toolchain url format pattern, e.g. '
431 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700432
Ryan Cuif890a3e2013-03-07 18:57:06 -0800433 # GYP_DEFINES that Chrome was built with. Influences which files are staged
434 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
435 # enviroment variable.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400436 parser.add_argument('--gyp-defines', default=None, type=ValidateGypDefines,
437 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700438 # Path of an empty directory to stage chrome artifacts to. Defaults to a
439 # temporary directory that is removed when the script finishes. If the path
440 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400441 parser.add_argument('--staging-dir', type='path', default=None,
442 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700443 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400444 parser.add_argument('--staging-only', action='store_true', default=False,
445 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700446 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
447 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
448 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400449 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Ryan Cuie535b172012-10-19 18:25:03 -0700450 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700451
Ryan Cuie535b172012-10-19 18:25:03 -0700452
453def _ParseCommandLine(argv):
454 """Parse args, and run environment-independent checks."""
455 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400456 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700457
Ryan Cuia56a71e2012-10-18 18:40:35 -0700458 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
459 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800460 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700461 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
462 parser.error('Cannot specify both --build_dir and '
463 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800464 if options.build_dir and not options.board:
465 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700466 if options.gs_path and options.local_pkg_path:
467 parser.error('Cannot specify both --gs-path and --local-pkg-path')
468 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700469 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800470 if (options.strict or options.staging_flags) and not options.build_dir:
471 parser.error('--strict and --staging-flags require --build-dir to be '
472 'set.')
473 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800474 parser.error('--staging-flags requires --strict to be set.')
475 if options.sloppy and options.strict:
476 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700477
478 if options.mount or options.mount_dir:
479 if not options.target_dir:
480 options.target_dir = _CHROME_DIR_MOUNT
481 else:
482 if not options.target_dir:
483 options.target_dir = _CHROME_DIR
484
485 if options.mount and not options.mount_dir:
486 options.mount_dir = _CHROME_DIR
487
Mike Frysingerc3061a62015-06-04 04:16:18 -0400488 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700489
490
Mike Frysingerc3061a62015-06-04 04:16:18 -0400491def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800492 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700493
494 Args:
Steve Funge984a532013-11-25 17:09:25 -0800495 options: The options object returned by optparse.
496 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700497 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700498 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
499 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
500
Ryan Cuib623e7b2013-03-14 12:54:11 -0700501 if not options.gyp_defines:
Steven Bennetts60600462016-05-12 10:40:20 -0700502 gyp_env = os.getenv('GYP_DEFINES')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700503 if gyp_env is not None:
504 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700505 logging.debug('GYP_DEFINES taken from environment: %s',
Mike Frysingere65f3752014-12-08 00:46:39 -0500506 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700507
508 if options.strict and not options.gyp_defines:
509 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Mike Frysingere65f3752014-12-08 00:46:39 -0500510 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700511
Steven Bennetts60600462016-05-12 10:40:20 -0700512 if not options.staging_flags:
513 use_env = os.getenv('USE')
514 if use_env is not None:
515 options.staging_flags = ' '.join(set(use_env.split()).intersection(
516 chrome_util.STAGING_FLAGS))
517 logging.info('Staging flags taken from USE in environment: %s',
518 options.staging_flags)
519
Ryan Cuia56a71e2012-10-18 18:40:35 -0700520
Ryan Cui504db722013-01-22 11:48:01 -0800521def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700522 """Get the chrome prebuilt tarball from GS.
523
Mike Frysinger02e1e072013-11-10 22:11:34 -0500524 Returns:
525 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700526 """
David James9374aac2013-10-08 16:00:17 -0700527 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500528 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800529 files = [found for found in files if
530 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
531 if not files:
532 raise Exception('No chrome package found at %s' % gs_path)
533 elif len(files) > 1:
534 # - Users should provide us with a direct link to either a stripped or
535 # unstripped chrome package.
536 # - In the case of being provided with an archive directory, where both
537 # stripped and unstripped chrome available, use the stripped chrome
538 # package.
539 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
540 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
541 files = [f for f in files if not 'unstripped' in f]
542 assert len(files) == 1
543 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800544
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800545 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800546 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800547 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
548 chrome_path = os.path.join(tempdir, filename)
549 assert os.path.exists(chrome_path)
550 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551
552
Ryan Cuif890a3e2013-03-07 18:57:06 -0800553@contextlib.contextmanager
554def _StripBinContext(options):
555 if not options.dostrip:
556 yield None
557 elif options.strip_bin:
558 yield options.strip_bin
559 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800560 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800561 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700562 with sdk.Prepare(components=components, target_tc=options.target_tc,
563 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800564 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
565 constants.CHROME_ENV_FILE)
566 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
567 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
568 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800569 yield strip_bin
570
571
Steve Funge984a532013-11-25 17:09:25 -0800572def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
573 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800574 """Place the necessary files in the staging directory.
575
576 The staging directory is the directory used to rsync the build artifacts over
577 to the device. Only the necessary Chrome build artifacts are put into the
578 staging directory.
579 """
Ryan Cui5866be02013-03-18 14:12:00 -0700580 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400581 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800582 if options.build_dir:
583 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700584 strip_flags = (None if options.strip_flags is None else
585 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800586 chrome_util.StageChromeFromBuildDir(
587 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800588 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700589 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800590 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700591 else:
592 pkg_path = options.local_pkg_path
593 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800594 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
595 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700596
597 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800598 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700599 # Extract only the ./opt/google/chrome contents, directly into the staging
600 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800601 if pkg_path[-4:] == '.zip':
602 cros_build_lib.DebugRunCommand(
603 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
604 staging_dir])
605 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
606 shutil.move(filename, staging_dir)
607 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
608 else:
609 cros_build_lib.DebugRunCommand(
610 ['tar', '--strip-components', '4', '--extract',
611 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
612 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700613
Ryan Cui71aa8de2013-04-19 16:12:55 -0700614
Ryan Cui3045c5d2012-07-13 18:00:33 -0700615def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400616 options = _ParseCommandLine(argv)
617 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700618
619 # Set cros_build_lib debug level to hide RunCommand spew.
620 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700621 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700622 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800623 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700624
Ryan Cui71aa8de2013-04-19 16:12:55 -0700625 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700626 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
627 if cmd_stats:
628 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700629
Ryan Cui71aa8de2013-04-19 16:12:55 -0700630 with osutils.TempDir(set_global=True) as tempdir:
631 staging_dir = options.staging_dir
632 if not staging_dir:
633 staging_dir = os.path.join(tempdir, 'chrome')
634
635 deploy = DeployChrome(options, tempdir, staging_dir)
636 try:
637 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700638 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700639 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500640 deploy.Cleanup()