blob: 98dd17b27b197274e78be8b25e08e0ce382cf88e [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
Steve Funge984a532013-11-25 17:09:25 -08007"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07008
9The script supports deploying Chrome from these sources:
10
111. A local build output directory, such as chromium/src/out/[Debug|Release].
122. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
133. A Chrome tarball existing locally.
14
15The script copies the necessary contents of the source location (tarball or
16build directory) and rsyncs the contents of the staging directory onto your
17device's rootfs.
18"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070019
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070024import logging
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070027import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
30import tarfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070031import time
Steve Funge984a532013-11-25 17:09:25 -080032import zipfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070033
Ryan Cuia56a71e2012-10-18 18:40:35 -070034
David James629febb2012-11-25 13:07:34 -080035from chromite.buildbot import constants
David James88e6f032013-03-02 08:13:20 -080036from chromite.buildbot import cbuildbot_results as results_lib
Ryan Cui686ec052013-02-12 16:39:41 -080037from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070038from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070040from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080041from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080043from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070044from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070045from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080046from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070047from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070048
49
Ryan Cuia56a71e2012-10-18 18:40:35 -070050_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
51
Ryan Cui3045c5d2012-07-13 18:00:33 -070052KERNEL_A_PARTITION = 2
53KERNEL_B_PARTITION = 4
54
55KILL_PROC_MAX_WAIT = 10
56POST_KILL_WAIT = 2
57
Ryan Cuie535b172012-10-19 18:25:03 -070058MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080059LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070060
Steve Funge984a532013-11-25 17:09:25 -080061MOUNT_RW_COMMAND_ANDROID = 'mount -o remount,rw /system'
62
63_ANDROID_DIR = '/system/chrome'
64_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
65
David James2cb34002013-03-01 18:42:40 -080066_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070067_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080068
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
73DF_COMMAND_ANDROID = 'df %s'
74
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
David James88e6f032013-03-02 08:13:20 -080080class DeployFailure(results_lib.StepFailure):
81 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700102 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800103 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Steve Funge984a532013-11-25 17:09:25 -0800105 # Used to track whether deploying content_shell or chrome to a device.
106 self.content_shell = False
107 self.copy_paths = chrome_util.GetCopyPaths(False)
108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800111 result = self.host.RemoteSh((DF_COMMAND if not self.content_shell
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800112 else DF_COMMAND_ANDROID) % remote_dir,
113 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700114 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800115 value = line.split()[3]
116 multipliers = {
117 'G': 1024 * 1024 * 1024,
118 'M': 1024 * 1024,
119 'K': 1024,
120 }
121 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122
123 def _GetRemoteDirSize(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800124 if self.content_shell:
125 # Content Shell devices currently do not contain the du binary.
126 logging.warning('Remote host does not contain du; cannot get remote '
127 'directory size to properly calculate available free '
128 'space.')
129 return 0
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800130 result = self.host.RemoteSh('du -ks %s' % remote_dir, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700131 return int(result.output.split()[0])
132
133 def _GetStagingDirSize(self):
134 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800135 redirect_stdout=True,
136 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700137 return int(result.output.split()[0])
138
Ryan Cui3045c5d2012-07-13 18:00:33 -0700139 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800140 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800141 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700142 return result.returncode == 0
143
144 def _DisableRootfsVerification(self):
145 if not self.options.force:
146 logging.error('Detected that the device has rootfs verification enabled.')
147 logging.info('This script can automatically remove the rootfs '
148 'verification, which requires that it reboot the device.')
149 logging.info('Make sure the device is in developer mode!')
150 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700151 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800152 # Since we stopped Chrome earlier, it's good form to start it up again.
153 if self.options.startui:
154 logging.info('Starting Chrome...')
155 self.host.RemoteSh('start ui')
156 raise DeployFailure('Need rootfs verification to be disabled. '
157 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 logging.info('Removing rootfs verification from %s', self.options.to)
160 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
161 # Use --force to bypass the checks.
162 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
163 '--remove_rootfs_verification --force')
164 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
165 self.host.RemoteSh(cmd % partition, error_code_ok=True)
166
167 # A reboot in developer mode takes a while (and has delays), so the user
168 # will have time to read and act on the USB boot instructions below.
169 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
170 self.host.RemoteReboot()
171
David James88e6f032013-03-02 08:13:20 -0800172 # Now that the machine has been rebooted, we need to kill Chrome again.
173 self._KillProcsIfNeeded()
174
175 # Make sure the rootfs is writable now.
176 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700177
178 def _CheckUiJobStarted(self):
179 # status output is in the format:
180 # <job_name> <status> ['process' <pid>].
181 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800182 try:
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800183 result = self.host.RemoteSh('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800184 except cros_build_lib.RunCommandError as e:
185 if 'Unknown job' in e.result.error:
186 return False
187 else:
188 raise e
189
Ryan Cui3045c5d2012-07-13 18:00:33 -0700190 return result.output.split()[1].split('/')[0] == 'start'
191
192 def _KillProcsIfNeeded(self):
Steve Funge984a532013-11-25 17:09:25 -0800193 if self.content_shell:
194 logging.info('Shutting down content_shell...')
195 self.host.RemoteSh('stop content_shell')
196 return
197
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800199 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700200 self.host.RemoteSh('stop ui')
201
202 # Developers sometimes run session_manager manually, in which case we'll
203 # need to help shut the chrome processes down.
204 try:
David James3432acd2013-11-27 10:02:18 -0800205 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206 while self._ChromeFileInUse():
207 logging.warning('The chrome binary on the device is in use.')
208 logging.warning('Killing chrome and session_manager processes...\n')
209
210 self.host.RemoteSh("pkill 'chrome|session_manager'",
211 error_code_ok=True)
212 # Wait for processes to actually terminate
213 time.sleep(POST_KILL_WAIT)
214 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800215 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800216 msg = ('Could not kill processes after %s seconds. Please exit any '
217 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
218 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700219
David James88e6f032013-03-02 08:13:20 -0800220 def _MountRootfsAsWritable(self, error_code_ok=True):
221 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
David James88e6f032013-03-02 08:13:20 -0800223 If the command fails, and error_code_ok is True, then this function sets
224 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700225
Mike Frysinger02e1e072013-11-10 22:11:34 -0500226 Args:
David James88e6f032013-03-02 08:13:20 -0800227 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
228 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800229 # TODO: Should migrate to use the remount functions in remote_access.
Steve Funge984a532013-11-25 17:09:25 -0800230 result = self.host.RemoteSh(MOUNT_RW_COMMAND if not self.content_shell
231 else MOUNT_RW_COMMAND_ANDROID,
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800232 error_code_ok=error_code_ok,
233 capture_output=True)
David James88e6f032013-03-02 08:13:20 -0800234 if result.returncode:
235 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700236
Ryan Cui7193a7e2013-04-26 14:15:19 -0700237 def _GetDeviceInfo(self):
238 steps = [
239 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
240 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
241 ]
242 return_values = parallel.RunParallelSteps(steps, return_values=True)
243 return DeviceInfo(*return_values)
244
245 def _CheckDeviceFreeSpace(self, device_info):
246 """See if target device has enough space for Chrome.
247
Mike Frysinger02e1e072013-11-10 22:11:34 -0500248 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700249 device_info: A DeviceInfo named tuple.
250 """
251 effective_free = device_info.target_dir_size + device_info.target_fs_free
252 staging_size = self._GetStagingDirSize()
Steve Funge984a532013-11-25 17:09:25 -0800253 # For content shell deployments, which do not contain the du binary,
254 # do not raise DeployFailure since can't get exact free space available
255 # for staging files.
Ryan Cui7193a7e2013-04-26 14:15:19 -0700256 if effective_free < staging_size:
Steve Funge984a532013-11-25 17:09:25 -0800257 if self.content_shell:
258 logging.warning('Not enough free space on the device. If overwriting '
259 'files, deployment may still succeed. Required: %s '
260 'MiB, actual: %s MiB.', staging_size / 1024,
261 effective_free / 1024)
262 else:
263 raise DeployFailure(
264 'Not enough free space on the device. Required: %s MiB, '
265 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700266 if device_info.target_fs_free < (100 * 1024):
267 logging.warning('The device has less than 100MB free. deploy_chrome may '
268 'hang during the transfer.')
269
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800271 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800273 dest_path = _CHROME_DIR
274 if self.content_shell:
275 try:
276 self.host.Scp('%s/*' % os.path.abspath(self.staging_dir),
277 '%s/' % self.options.target_dir,
Steve Funge984a532013-11-25 17:09:25 -0800278 debug_level=logging.INFO,
279 verbose=self.options.verbose)
280 except cros_build_lib.RunCommandError as ex:
281 if ex.result.returncode != 1:
282 logging.error('Scp failure [%s]', ex.result.returncode)
283 raise DeployFailure(ex)
284 else:
285 # TODO(stevefung): Update Dropbear SSHD on device.
286 # http://crbug.com/329656
287 logging.info('Potential conflict with DropBear SSHD return status')
288
289 dest_path = _ANDROID_DIR
290 else:
291 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
292 self.options.target_dir,
293 inplace=True, debug_level=logging.INFO,
294 verbose=self.options.verbose)
295
296 for p in self.copy_paths:
297 if p.owner:
298 self.host.RemoteSh('chown %s %s/%s' % (p.owner, dest_path,
299 p.src if not p.dest else p.dest))
300 if p.mode:
301 # Set mode if necessary.
302 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
303 p.src if not p.dest else p.dest))
304
305
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800306 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800307 logging.info('Starting UI...')
308 if self.content_shell:
309 self.host.RemoteSh('start content_shell')
310 else:
311 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700312
David James88e6f032013-03-02 08:13:20 -0800313 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700314 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800315 logging.info('Testing connection to the device...')
Steve Funge984a532013-11-25 17:09:25 -0800316 if self.content_shell:
317 # true command over ssh returns error code 255, so as workaround
318 # use `sleep 0` as no-op.
319 self.host.RemoteSh('sleep 0')
320 else:
321 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800322 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700323 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800324 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700325
Steve Funge984a532013-11-25 17:09:25 -0800326 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700327 if self.options.build_dir:
328 if os.path.exists(os.path.join(self.options.build_dir, 'system.unand')):
329 # Unand Content shell deployment.
330 self.content_shell = True
331 self.options.build_dir = os.path.join(self.options.build_dir,
332 'system.unand/chrome/')
333 self.options.dostrip = False
334 self.options.target_dir = _ANDROID_DIR
335 self.copy_paths = chrome_util.GetCopyPaths(True)
336 elif os.path.exists(os.path.join(self.options.build_dir,
337 'content_shell')) and not os.path.exists(
338 os.path.join(self.options.build_dir, 'chrome')):
339 # Content shell deployment
340 self.content_shell = True
341 self.copy_paths = chrome_util.GetCopyPaths(True)
Steve Funge984a532013-11-25 17:09:25 -0800342 elif self.options.local_pkg_path or self.options.gs_path:
343 # Package deployment.
344 pkg_path = self.options.local_pkg_path
345 if self.options.gs_path:
346 pkg_path = _FetchChromePackage(self.options.cache_dir, self.tempdir,
347 self.options.gs_path)
348
349 assert pkg_path
350 logging.info('Checking %s for content_shell...', pkg_path)
351 if pkg_path[-4:] == '.zip':
352 zip_pkg = zipfile.ZipFile(pkg_path)
353 if any('eureka_shell' in name for name in zip_pkg.namelist()):
354 self.content_shell = True
355 zip_pkg.close()
356 else:
357 tar = tarfile.open(pkg_path)
358 if any('eureka_shell' in member.name for member in tar.getmembers()):
359 self.content_shell = True
360 tar.close()
361
362 if self.content_shell:
363 self.options.dostrip = False
364 self.options.target_dir = _ANDROID_DIR
365 self.copy_paths = chrome_util.GetCopyPaths(True)
366 self.chrome_dir = _ANDROID_DIR
367
David James88e6f032013-03-02 08:13:20 -0800368 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800369 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
370 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800371
Thiago Goncales12793312013-05-23 11:26:17 -0700372 def _MountTarget(self):
373 logging.info('Mounting Chrome...')
374
375 # Create directory if does not exist
376 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
377 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
378 self.options.mount_dir))
379 # Chrome needs partition to have exec and suid flags set
380 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
381
David James88e6f032013-03-02 08:13:20 -0800382 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800383 self._CheckDeployType()
384
David James88e6f032013-03-02 08:13:20 -0800385 # If requested, just do the staging step.
386 if self.options.staging_only:
387 self._PrepareStagingDir()
388 return 0
389
390 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800391 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800392 steps = [self._GetDeviceInfo, self._CheckConnection,
393 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
394 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700395 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
396 return_values=True)
397 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800398
399 # If we failed to mark the rootfs as writable, try disabling rootfs
400 # verification.
401 if self._rootfs_is_still_readonly.is_set():
402 self._DisableRootfsVerification()
403
Thiago Goncales12793312013-05-23 11:26:17 -0700404 if self.options.mount_dir is not None:
405 self._MountTarget()
406
David James88e6f032013-03-02 08:13:20 -0800407 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700408 self._Deploy()
409
410
Ryan Cuia56a71e2012-10-18 18:40:35 -0700411def ValidateGypDefines(_option, _opt, value):
412 """Convert GYP_DEFINES-formatted string to dictionary."""
413 return chrome_util.ProcessGypDefines(value)
414
415
416class CustomOption(commandline.Option):
417 """Subclass Option class to implement path evaluation."""
418 TYPES = commandline.Option.TYPES + ('gyp_defines',)
419 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
420 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
421
422
Ryan Cuie535b172012-10-19 18:25:03 -0700423def _CreateParser():
424 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800425 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
426 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700427
Ryan Cuia56a71e2012-10-18 18:40:35 -0700428 # TODO(rcui): Have this use the UI-V2 format of having source and target
429 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700430 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800431 help='Skip all prompts (i.e., for disabling of rootfs '
432 'verification). This may result in the target '
433 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800434 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
435 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800436 help="The board the Chrome build is targeted for. When in "
437 "a 'cros chrome-sdk' shell, defaults to the SDK "
438 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700439 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800440 help='The directory with Chrome build artifacts to deploy '
441 'from. Typically of format <chrome_root>/out/Debug. '
442 'When this option is used, the GYP_DEFINES '
443 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800444 parser.add_option('--target-dir', type='path',
445 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700446 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700447 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800448 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800449 parser.add_option('--nostartui', action='store_false', dest='startui',
450 default=True,
451 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800452 parser.add_option('--nostrip', action='store_false', dest='dostrip',
453 default=True,
454 help="Don't strip binaries during deployment. Warning: "
455 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700456 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800457 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700458 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800459 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700460 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800461 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700462 parser.add_option('--mount-dir', type='path', default=None,
463 help='Deploy Chrome in target directory and bind it'
464 'to directory specified by this flag.')
465 parser.add_option('--mount', action='store_true', default=False,
466 help='Deploy Chrome to default target directory and bind it'
467 'to default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700468
469 group = optparse.OptionGroup(parser, 'Advanced Options')
470 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800471 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800472 group.add_option('--sloppy', action='store_true', default=False,
473 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700474 group.add_option('--staging-flags', default=None, type='gyp_defines',
475 help='Extra flags to control staging. Valid flags are - %s'
476 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800477 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800478 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800479 'variable and --staging-flags, if set. Enforce that '
480 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700481 group.add_option('--strip-flags', default=None,
482 help="Flags to call the 'strip' binutil tool with. "
483 "Overrides the default arguments.")
Ryan Cuief91e702013-02-04 12:06:36 -0800484
Ryan Cuia56a71e2012-10-18 18:40:35 -0700485 parser.add_option_group(group)
486
Ryan Cuif890a3e2013-03-07 18:57:06 -0800487 # GYP_DEFINES that Chrome was built with. Influences which files are staged
488 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
489 # enviroment variable.
490 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
491 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700492 # Path of an empty directory to stage chrome artifacts to. Defaults to a
493 # temporary directory that is removed when the script finishes. If the path
494 # is specified, then it will not be removed.
495 parser.add_option('--staging-dir', type='path', default=None,
496 help=optparse.SUPPRESS_HELP)
497 # Only prepare the staging directory, and skip deploying to the device.
498 parser.add_option('--staging-only', action='store_true', default=False,
499 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700500 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
501 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
502 # fetching the SDK toolchain.
503 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700504 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700505
Ryan Cuie535b172012-10-19 18:25:03 -0700506
507def _ParseCommandLine(argv):
508 """Parse args, and run environment-independent checks."""
509 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700510 (options, args) = parser.parse_args(argv)
511
Ryan Cuia56a71e2012-10-18 18:40:35 -0700512 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
513 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800514 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700515 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
516 parser.error('Cannot specify both --build_dir and '
517 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800518 if options.build_dir and not options.board:
519 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700520 if options.gs_path and options.local_pkg_path:
521 parser.error('Cannot specify both --gs-path and --local-pkg-path')
522 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700523 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800524 if (options.strict or options.staging_flags) and not options.build_dir:
525 parser.error('--strict and --staging-flags require --build-dir to be '
526 'set.')
527 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800528 parser.error('--staging-flags requires --strict to be set.')
529 if options.sloppy and options.strict:
530 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700531
532 if options.mount or options.mount_dir:
533 if not options.target_dir:
534 options.target_dir = _CHROME_DIR_MOUNT
535 else:
536 if not options.target_dir:
537 options.target_dir = _CHROME_DIR
538
539 if options.mount and not options.mount_dir:
540 options.mount_dir = _CHROME_DIR
541
Ryan Cui3045c5d2012-07-13 18:00:33 -0700542 return options, args
543
544
Ryan Cuie535b172012-10-19 18:25:03 -0700545def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800546 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700547
548 Args:
Steve Funge984a532013-11-25 17:09:25 -0800549 options: The options object returned by optparse.
550 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700551 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700552 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
553 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
554
Ryan Cuib623e7b2013-03-14 12:54:11 -0700555 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700556 gyp_env = os.getenv('GYP_DEFINES', None)
557 if gyp_env is not None:
558 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700559 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700560 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700561
562 if options.strict and not options.gyp_defines:
563 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800564 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700565
Ryan Cui3c183c22013-04-29 18:04:11 -0700566 if options.build_dir:
567 chrome_path = os.path.join(options.build_dir, 'chrome')
568 if os.path.isfile(chrome_path):
569 deps = lddtree.ParseELF(chrome_path)
570 if 'libbase.so' in deps['libs']:
571 cros_build_lib.Warning(
572 'Detected a component build of Chrome. component build is '
573 'not working properly for Chrome OS. See crbug.com/196317. '
574 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700575
Ryan Cuia56a71e2012-10-18 18:40:35 -0700576
Ryan Cui504db722013-01-22 11:48:01 -0800577def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700578 """Get the chrome prebuilt tarball from GS.
579
Mike Frysinger02e1e072013-11-10 22:11:34 -0500580 Returns:
581 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700582 """
David James9374aac2013-10-08 16:00:17 -0700583 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500584 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800585 files = [found for found in files if
586 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
587 if not files:
588 raise Exception('No chrome package found at %s' % gs_path)
589 elif len(files) > 1:
590 # - Users should provide us with a direct link to either a stripped or
591 # unstripped chrome package.
592 # - In the case of being provided with an archive directory, where both
593 # stripped and unstripped chrome available, use the stripped chrome
594 # package.
595 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
596 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
597 files = [f for f in files if not 'unstripped' in f]
598 assert len(files) == 1
599 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800600
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800601 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800602 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800603 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
604 chrome_path = os.path.join(tempdir, filename)
605 assert os.path.exists(chrome_path)
606 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700607
608
Ryan Cuif890a3e2013-03-07 18:57:06 -0800609@contextlib.contextmanager
610def _StripBinContext(options):
611 if not options.dostrip:
612 yield None
613 elif options.strip_bin:
614 yield options.strip_bin
615 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800616 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800617 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
618 with sdk.Prepare(components=components) as ctx:
619 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
620 constants.CHROME_ENV_FILE)
621 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
622 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
623 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800624 yield strip_bin
625
626
Steve Funge984a532013-11-25 17:09:25 -0800627def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
628 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800629 """Place the necessary files in the staging directory.
630
631 The staging directory is the directory used to rsync the build artifacts over
632 to the device. Only the necessary Chrome build artifacts are put into the
633 staging directory.
634 """
Ryan Cui5866be02013-03-18 14:12:00 -0700635 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400636 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800637 if options.build_dir:
638 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700639 strip_flags = (None if options.strip_flags is None else
640 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800641 chrome_util.StageChromeFromBuildDir(
642 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800643 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700644 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800645 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700646 else:
647 pkg_path = options.local_pkg_path
648 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800649 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
650 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700651
652 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800653 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700654 # Extract only the ./opt/google/chrome contents, directly into the staging
655 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800656 if pkg_path[-4:] == '.zip':
657 cros_build_lib.DebugRunCommand(
658 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
659 staging_dir])
660 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
661 shutil.move(filename, staging_dir)
662 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
663 else:
664 cros_build_lib.DebugRunCommand(
665 ['tar', '--strip-components', '4', '--extract',
666 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
667 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700668
Ryan Cui71aa8de2013-04-19 16:12:55 -0700669
Ryan Cui3045c5d2012-07-13 18:00:33 -0700670def main(argv):
671 options, args = _ParseCommandLine(argv)
672 _PostParseCheck(options, args)
673
674 # Set cros_build_lib debug level to hide RunCommand spew.
675 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700676 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700677 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800678 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700679
Ryan Cui71aa8de2013-04-19 16:12:55 -0700680 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700681 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
682 if cmd_stats:
683 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700684
Ryan Cui71aa8de2013-04-19 16:12:55 -0700685 with osutils.TempDir(set_global=True) as tempdir:
686 staging_dir = options.staging_dir
687 if not staging_dir:
688 staging_dir = os.path.join(tempdir, 'chrome')
689
690 deploy = DeployChrome(options, tempdir, staging_dir)
691 try:
692 deploy.Perform()
693 except results_lib.StepFailure as ex:
694 raise SystemExit(str(ex).strip())