blob: 5209e2befe05baf87eb55be71a3acabe3e4fdf76 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
David James88e6f032013-03-02 08:13:20 -080024import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070025import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070026import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Don Garrett88b8d782014-05-13 17:30:55 -070031from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070032from chromite.cbuildbot import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070042from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070044
45
Mike Frysingere65f3752014-12-08 00:46:39 -050046_USAGE = 'deploy_chrome [--]\n\n %s' % __doc__
Ryan Cuia56a71e2012-10-18 18:40:35 -070047
Ryan Cui3045c5d2012-07-13 18:00:33 -070048KERNEL_A_PARTITION = 2
49KERNEL_B_PARTITION = 4
50
51KILL_PROC_MAX_WAIT = 10
52POST_KILL_WAIT = 2
53
Ryan Cuie535b172012-10-19 18:25:03 -070054MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080055LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070056
Steve Funge984a532013-11-25 17:09:25 -080057_ANDROID_DIR = '/system/chrome'
58_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
59
David James2cb34002013-03-01 18:42:40 -080060_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070061_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080062
Pawel Osciakc1bb2742014-12-29 16:32:33 +090063_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
64 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070065_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
66_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070067
Steve Funge984a532013-11-25 17:09:25 -080068DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080069
Mike Frysingere65f3752014-12-08 00:46:39 -050070
Ryan Cui3045c5d2012-07-13 18:00:33 -070071def _UrlBaseName(url):
72 """Return the last component of the URL."""
73 return url.rstrip('/').rpartition('/')[-1]
74
75
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070076class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080077 """Raised whenever the deploy fails."""
78
79
Ryan Cui7193a7e2013-04-26 14:15:19 -070080DeviceInfo = collections.namedtuple(
81 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
82
83
Ryan Cui3045c5d2012-07-13 18:00:33 -070084class DeployChrome(object):
85 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050086
Ryan Cuia56a71e2012-10-18 18:40:35 -070087 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070088 """Initialize the class.
89
Mike Frysinger02e1e072013-11-10 22:11:34 -050090 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070091 options: Optparse result structure.
92 tempdir: Scratch space for the class. Caller has responsibility to clean
93 it up.
Steve Funge984a532013-11-25 17:09:25 -080094 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070095 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070096 self.tempdir = tempdir
97 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070098 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050099 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +0900100 self.device = remote.RemoteDevice(options.to, port=options.port,
101 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500102 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700103
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700104 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800105 self.chrome_dir = _CHROME_DIR
106
Ryan Cui7193a7e2013-04-26 14:15:19 -0700107 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500108 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700109 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800110 value = line.split()[3]
111 multipliers = {
112 'G': 1024 * 1024 * 1024,
113 'M': 1024 * 1024,
114 'K': 1024,
115 }
116 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700117
118 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500119 result = self.device.RunCommand('du -ks %s' % remote_dir,
120 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700121 return int(result.output.split()[0])
122
123 def _GetStagingDirSize(self):
124 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800125 redirect_stdout=True,
126 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700127 return int(result.output.split()[0])
128
Ryan Cui3045c5d2012-07-13 18:00:33 -0700129 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500130 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
131 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700132 return result.returncode == 0
133
134 def _DisableRootfsVerification(self):
135 if not self.options.force:
136 logging.error('Detected that the device has rootfs verification enabled.')
137 logging.info('This script can automatically remove the rootfs '
138 'verification, which requires that it reboot the device.')
139 logging.info('Make sure the device is in developer mode!')
140 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700141 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800142 # Since we stopped Chrome earlier, it's good form to start it up again.
143 if self.options.startui:
144 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500145 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800146 raise DeployFailure('Need rootfs verification to be disabled. '
147 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700148
149 logging.info('Removing rootfs verification from %s', self.options.to)
150 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
151 # Use --force to bypass the checks.
152 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
153 '--remove_rootfs_verification --force')
154 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500155 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156
157 # A reboot in developer mode takes a while (and has delays), so the user
158 # will have time to read and act on the USB boot instructions below.
159 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500160 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700161
David James88e6f032013-03-02 08:13:20 -0800162 # Now that the machine has been rebooted, we need to kill Chrome again.
163 self._KillProcsIfNeeded()
164
165 # Make sure the rootfs is writable now.
166 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700167
168 def _CheckUiJobStarted(self):
169 # status output is in the format:
170 # <job_name> <status> ['process' <pid>].
171 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800172 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500173 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 except cros_build_lib.RunCommandError as e:
175 if 'Unknown job' in e.result.error:
176 return False
177 else:
178 raise e
179
Ryan Cui3045c5d2012-07-13 18:00:33 -0700180 return result.output.split()[1].split('/')[0] == 'start'
181
182 def _KillProcsIfNeeded(self):
183 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800184 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500185 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700186
187 # Developers sometimes run session_manager manually, in which case we'll
188 # need to help shut the chrome processes down.
189 try:
David James3432acd2013-11-27 10:02:18 -0800190 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700191 while self._ChromeFileInUse():
192 logging.warning('The chrome binary on the device is in use.')
193 logging.warning('Killing chrome and session_manager processes...\n')
194
Robert Flack1dc7ea82014-11-26 13:50:24 -0500195 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500196 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700197 # Wait for processes to actually terminate
198 time.sleep(POST_KILL_WAIT)
199 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800200 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800201 msg = ('Could not kill processes after %s seconds. Please exit any '
202 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
203 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700204
David James88e6f032013-03-02 08:13:20 -0800205 def _MountRootfsAsWritable(self, error_code_ok=True):
206 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700207
Robert Flack1dc7ea82014-11-26 13:50:24 -0500208 If the command fails, and error_code_ok is True, and the target dir is not
209 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700210
Mike Frysinger02e1e072013-11-10 22:11:34 -0500211 Args:
David James88e6f032013-03-02 08:13:20 -0800212 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
213 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800214 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500215 result = self.device.RunCommand(MOUNT_RW_COMMAND,
216 error_code_ok=error_code_ok,
217 capture_output=True)
218 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400219 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500220 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700221
Ryan Cui7193a7e2013-04-26 14:15:19 -0700222 def _GetDeviceInfo(self):
223 steps = [
224 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
225 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
226 ]
227 return_values = parallel.RunParallelSteps(steps, return_values=True)
228 return DeviceInfo(*return_values)
229
230 def _CheckDeviceFreeSpace(self, device_info):
231 """See if target device has enough space for Chrome.
232
Mike Frysinger02e1e072013-11-10 22:11:34 -0500233 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700234 device_info: A DeviceInfo named tuple.
235 """
236 effective_free = device_info.target_dir_size + device_info.target_fs_free
237 staging_size = self._GetStagingDirSize()
238 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700239 raise DeployFailure(
240 'Not enough free space on the device. Required: %s MiB, '
241 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700242 if device_info.target_fs_free < (100 * 1024):
243 logging.warning('The device has less than 100MB free. deploy_chrome may '
244 'hang during the transfer.')
245
Ryan Cui3045c5d2012-07-13 18:00:33 -0700246 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800247 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700248 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800249 dest_path = _CHROME_DIR
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900250 if not self.device.HasRsync():
251 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700252 'rsync is not found on the device.\n'
253 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500254 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
255 self.options.target_dir,
256 inplace=True, debug_level=logging.INFO,
257 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800258
259 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800260 if p.mode:
261 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500262 self.device.RunCommand('chmod %o %s/%s' % (
263 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800264
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800265 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800266 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500267 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700268
David James88e6f032013-03-02 08:13:20 -0800269 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800271 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500272 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800273 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700274 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800275 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700276
Steve Funge984a532013-11-25 17:09:25 -0800277 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700278 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700279 def BinaryExists(filename):
280 """Checks if the passed-in file is present in the build directory."""
281 return os.path.exists(os.path.join(self.options.build_dir, filename))
282
Daniel Erat9813f0e2014-11-12 11:00:28 -0700283 # Handle non-Chrome deployments.
284 if not BinaryExists('chrome'):
285 if BinaryExists('envoy_shell'):
286 self.copy_paths = chrome_util.GetCopyPaths('envoy')
287 elif BinaryExists('app_shell'):
288 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
289
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700290 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700291 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700292 self.options.startui = False
293
David James88e6f032013-03-02 08:13:20 -0800294 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800295 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
296 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800297
Thiago Goncales12793312013-05-23 11:26:17 -0700298 def _MountTarget(self):
299 logging.info('Mounting Chrome...')
300
301 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500302 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500303 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900304 # Umount the existing mount on mount_dir if present first
305 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
306 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500307 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
308 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700309 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500310 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
311
312 def Cleanup(self):
313 """Clean up RemoteDevice."""
314 if not self.options.staging_only:
315 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700316
David James88e6f032013-03-02 08:13:20 -0800317 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800318 self._CheckDeployType()
319
David James88e6f032013-03-02 08:13:20 -0800320 # If requested, just do the staging step.
321 if self.options.staging_only:
322 self._PrepareStagingDir()
323 return 0
324
325 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800326 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800327 steps = [self._GetDeviceInfo, self._CheckConnection,
328 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
329 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700330 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
331 return_values=True)
332 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800333
Robert Flack1dc7ea82014-11-26 13:50:24 -0500334 # If we're trying to deploy to a dir which is not writable and we failed
335 # to mark the rootfs as writable, try disabling rootfs verification.
336 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800337 self._DisableRootfsVerification()
338
Thiago Goncales12793312013-05-23 11:26:17 -0700339 if self.options.mount_dir is not None:
340 self._MountTarget()
341
David James88e6f032013-03-02 08:13:20 -0800342 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700343 self._Deploy()
344
345
Ryan Cuia56a71e2012-10-18 18:40:35 -0700346def ValidateGypDefines(_option, _opt, value):
347 """Convert GYP_DEFINES-formatted string to dictionary."""
348 return chrome_util.ProcessGypDefines(value)
349
350
351class CustomOption(commandline.Option):
352 """Subclass Option class to implement path evaluation."""
353 TYPES = commandline.Option.TYPES + ('gyp_defines',)
354 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
355 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
356
357
Ryan Cuie535b172012-10-19 18:25:03 -0700358def _CreateParser():
359 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800360 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
361 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700362
Ryan Cuia56a71e2012-10-18 18:40:35 -0700363 # TODO(rcui): Have this use the UI-V2 format of having source and target
364 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700365 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800366 help='Skip all prompts (i.e., for disabling of rootfs '
367 'verification). This may result in the target '
368 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800369 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
370 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800371 help="The board the Chrome build is targeted for. When in "
372 "a 'cros chrome-sdk' shell, defaults to the SDK "
373 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700374 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800375 help='The directory with Chrome build artifacts to deploy '
376 'from. Typically of format <chrome_root>/out/Debug. '
377 'When this option is used, the GYP_DEFINES '
378 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800379 parser.add_option('--target-dir', type='path',
380 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700381 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700382 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800383 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800384 parser.add_option('--nostartui', action='store_false', dest='startui',
385 default=True,
386 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800387 parser.add_option('--nostrip', action='store_false', dest='dostrip',
388 default=True,
389 help="Don't strip binaries during deployment. Warning: "
390 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700391 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800392 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700393 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800394 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700395 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800396 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700397 parser.add_option('--mount-dir', type='path', default=None,
Mike Frysingercd304692014-10-02 16:33:41 -0400398 help='Deploy Chrome in target directory and bind it '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900399 'to the directory specified by this flag.'
400 'Any existing mount on this directory will be '
401 'umounted first.')
Thiago Goncales12793312013-05-23 11:26:17 -0700402 parser.add_option('--mount', action='store_true', default=False,
Mike Frysingercd304692014-10-02 16:33:41 -0400403 help='Deploy Chrome to default target directory and bind '
Pawel Osciakfb200f52014-12-21 13:42:05 +0900404 'it to the default mount directory.'
405 'Any existing mount on this directory will be '
406 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700407
408 group = optparse.OptionGroup(parser, 'Advanced Options')
409 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800410 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800411 group.add_option('--sloppy', action='store_true', default=False,
412 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700413 group.add_option('--staging-flags', default=None, type='gyp_defines',
Mike Frysingere65f3752014-12-08 00:46:39 -0500414 help=('Extra flags to control staging. Valid flags are - %s'
415 % ', '.join(chrome_util.STAGING_FLAGS)))
Ryan Cuief91e702013-02-04 12:06:36 -0800416 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800417 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800418 'variable and --staging-flags, if set. Enforce that '
419 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700420 group.add_option('--strip-flags', default=None,
421 help="Flags to call the 'strip' binutil tool with. "
422 "Overrides the default arguments.")
Pawel Osciak141b2262014-12-21 10:27:05 +0900423 group.add_option('--ping', action='store_true', default=False,
424 help='Ping the device before connection attempt.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700425 parser.add_option_group(group)
426
Aviv Keshet1c986f32014-04-24 13:20:49 -0700427 group = optparse.OptionGroup(parser, 'Metadata Overrides (Advanced)',
428 description='Provide all of these overrides '
429 'in order to remove dependencies on '
430 'metadata.json existence.')
431 group.add_option('--target-tc', action='store', default=None,
432 help='Override target toolchain name, e.g. '
433 'x86_64-cros-linux-gnu')
434 group.add_option('--toolchain-url', action='store', default=None,
435 help='Override toolchain url format pattern, e.g. '
436 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
437 parser.add_option_group(group)
438
Ryan Cuif890a3e2013-03-07 18:57:06 -0800439 # GYP_DEFINES that Chrome was built with. Influences which files are staged
440 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
441 # enviroment variable.
442 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
443 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700444 # Path of an empty directory to stage chrome artifacts to. Defaults to a
445 # temporary directory that is removed when the script finishes. If the path
446 # is specified, then it will not be removed.
447 parser.add_option('--staging-dir', type='path', default=None,
448 help=optparse.SUPPRESS_HELP)
449 # Only prepare the staging directory, and skip deploying to the device.
450 parser.add_option('--staging-only', action='store_true', default=False,
451 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700452 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
453 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
454 # fetching the SDK toolchain.
455 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700456 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700457
Ryan Cuie535b172012-10-19 18:25:03 -0700458
459def _ParseCommandLine(argv):
460 """Parse args, and run environment-independent checks."""
461 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700462 (options, args) = parser.parse_args(argv)
463
Ryan Cuia56a71e2012-10-18 18:40:35 -0700464 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
465 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800466 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700467 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
468 parser.error('Cannot specify both --build_dir and '
469 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800470 if options.build_dir and not options.board:
471 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700472 if options.gs_path and options.local_pkg_path:
473 parser.error('Cannot specify both --gs-path and --local-pkg-path')
474 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700475 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800476 if (options.strict or options.staging_flags) and not options.build_dir:
477 parser.error('--strict and --staging-flags require --build-dir to be '
478 'set.')
479 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800480 parser.error('--staging-flags requires --strict to be set.')
481 if options.sloppy and options.strict:
482 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700483
484 if options.mount or options.mount_dir:
485 if not options.target_dir:
486 options.target_dir = _CHROME_DIR_MOUNT
487 else:
488 if not options.target_dir:
489 options.target_dir = _CHROME_DIR
490
491 if options.mount and not options.mount_dir:
492 options.mount_dir = _CHROME_DIR
493
Ryan Cui3045c5d2012-07-13 18:00:33 -0700494 return options, args
495
496
Ryan Cuie535b172012-10-19 18:25:03 -0700497def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800498 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700499
500 Args:
Steve Funge984a532013-11-25 17:09:25 -0800501 options: The options object returned by optparse.
502 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700503 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700504 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
505 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
506
Ryan Cuib623e7b2013-03-14 12:54:11 -0700507 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700508 gyp_env = os.getenv('GYP_DEFINES', None)
509 if gyp_env is not None:
510 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700511 logging.debug('GYP_DEFINES taken from environment: %s',
Mike Frysingere65f3752014-12-08 00:46:39 -0500512 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700513
514 if options.strict and not options.gyp_defines:
515 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Mike Frysingere65f3752014-12-08 00:46:39 -0500516 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700517
518
Ryan Cui504db722013-01-22 11:48:01 -0800519def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700520 """Get the chrome prebuilt tarball from GS.
521
Mike Frysinger02e1e072013-11-10 22:11:34 -0500522 Returns:
523 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700524 """
David James9374aac2013-10-08 16:00:17 -0700525 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500526 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800527 files = [found for found in files if
528 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
529 if not files:
530 raise Exception('No chrome package found at %s' % gs_path)
531 elif len(files) > 1:
532 # - Users should provide us with a direct link to either a stripped or
533 # unstripped chrome package.
534 # - In the case of being provided with an archive directory, where both
535 # stripped and unstripped chrome available, use the stripped chrome
536 # package.
537 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
538 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
539 files = [f for f in files if not 'unstripped' in f]
540 assert len(files) == 1
541 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800542
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800543 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800544 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800545 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
546 chrome_path = os.path.join(tempdir, filename)
547 assert os.path.exists(chrome_path)
548 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700549
550
Ryan Cuif890a3e2013-03-07 18:57:06 -0800551@contextlib.contextmanager
552def _StripBinContext(options):
553 if not options.dostrip:
554 yield None
555 elif options.strip_bin:
556 yield options.strip_bin
557 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800558 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800559 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700560 with sdk.Prepare(components=components, target_tc=options.target_tc,
561 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800562 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
563 constants.CHROME_ENV_FILE)
564 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
565 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
566 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800567 yield strip_bin
568
569
Steve Funge984a532013-11-25 17:09:25 -0800570def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
571 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800572 """Place the necessary files in the staging directory.
573
574 The staging directory is the directory used to rsync the build artifacts over
575 to the device. Only the necessary Chrome build artifacts are put into the
576 staging directory.
577 """
Ryan Cui5866be02013-03-18 14:12:00 -0700578 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400579 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800580 if options.build_dir:
581 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700582 strip_flags = (None if options.strip_flags is None else
583 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800584 chrome_util.StageChromeFromBuildDir(
585 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800586 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700587 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800588 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700589 else:
590 pkg_path = options.local_pkg_path
591 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800592 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
593 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700594
595 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800596 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700597 # Extract only the ./opt/google/chrome contents, directly into the staging
598 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800599 if pkg_path[-4:] == '.zip':
600 cros_build_lib.DebugRunCommand(
601 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
602 staging_dir])
603 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
604 shutil.move(filename, staging_dir)
605 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
606 else:
607 cros_build_lib.DebugRunCommand(
608 ['tar', '--strip-components', '4', '--extract',
609 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
610 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700611
Ryan Cui71aa8de2013-04-19 16:12:55 -0700612
Ryan Cui3045c5d2012-07-13 18:00:33 -0700613def main(argv):
614 options, args = _ParseCommandLine(argv)
615 _PostParseCheck(options, args)
616
617 # Set cros_build_lib debug level to hide RunCommand spew.
618 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700619 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700620 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800621 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700622
Ryan Cui71aa8de2013-04-19 16:12:55 -0700623 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700624 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
625 if cmd_stats:
626 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700627
Ryan Cui71aa8de2013-04-19 16:12:55 -0700628 with osutils.TempDir(set_global=True) as tempdir:
629 staging_dir = options.staging_dir
630 if not staging_dir:
631 staging_dir = os.path.join(tempdir, 'chrome')
632
633 deploy = DeployChrome(options, tempdir, staging_dir)
634 try:
635 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700636 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700637 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500638 deploy.Cleanup()