blob: 617a5f6ab023d75b663c01b316d2011946c8df19 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
Steve Funge984a532013-11-25 17:09:25 -08007"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07008
9The script supports deploying Chrome from these sources:
10
111. A local build output directory, such as chromium/src/out/[Debug|Release].
122. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
133. A Chrome tarball existing locally.
14
15The script copies the necessary contents of the source location (tarball or
16build directory) and rsyncs the contents of the staging directory onto your
17device's rootfs.
18"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070019
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070024import logging
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070027import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
30import tarfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070031import time
Steve Funge984a532013-11-25 17:09:25 -080032import zipfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070033
Ryan Cuia56a71e2012-10-18 18:40:35 -070034
David James629febb2012-11-25 13:07:34 -080035from chromite.buildbot import constants
David James88e6f032013-03-02 08:13:20 -080036from chromite.buildbot import cbuildbot_results as results_lib
Ryan Cui686ec052013-02-12 16:39:41 -080037from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070038from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070040from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080041from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080043from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070044from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070045from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080046from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070047from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070048
49
Ryan Cuia56a71e2012-10-18 18:40:35 -070050_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
51
Ryan Cui3045c5d2012-07-13 18:00:33 -070052KERNEL_A_PARTITION = 2
53KERNEL_B_PARTITION = 4
54
55KILL_PROC_MAX_WAIT = 10
56POST_KILL_WAIT = 2
57
Ryan Cuie535b172012-10-19 18:25:03 -070058MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080059LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070060
Steve Funge984a532013-11-25 17:09:25 -080061MOUNT_RW_COMMAND_ANDROID = 'mount -o remount,rw /system'
62
63_ANDROID_DIR = '/system/chrome'
64_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
65
David James2cb34002013-03-01 18:42:40 -080066_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070067_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080068
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
73DF_COMMAND_ANDROID = 'df %s'
74
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
David James88e6f032013-03-02 08:13:20 -080080class DeployFailure(results_lib.StepFailure):
81 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700102 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800103 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Steve Funge984a532013-11-25 17:09:25 -0800105 # Used to track whether deploying content_shell or chrome to a device.
106 self.content_shell = False
107 self.copy_paths = chrome_util.GetCopyPaths(False)
108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800111 result = self.host.RemoteSh((DF_COMMAND if not self.content_shell
112 else DF_COMMAND_ANDROID) % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700113 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800114 value = line.split()[3]
115 multipliers = {
116 'G': 1024 * 1024 * 1024,
117 'M': 1024 * 1024,
118 'K': 1024,
119 }
120 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700121
122 def _GetRemoteDirSize(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800123 if self.content_shell:
124 # Content Shell devices currently do not contain the du binary.
125 logging.warning('Remote host does not contain du; cannot get remote '
126 'directory size to properly calculate available free '
127 'space.')
128 return 0
Ryan Cui7193a7e2013-04-26 14:15:19 -0700129 result = self.host.RemoteSh('du -ks %s' % remote_dir)
130 return int(result.output.split()[0])
131
132 def _GetStagingDirSize(self):
133 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
134 redirect_stdout=True)
135 return int(result.output.split()[0])
136
Ryan Cui3045c5d2012-07-13 18:00:33 -0700137 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800138 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
139 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700140 return result.returncode == 0
141
142 def _DisableRootfsVerification(self):
143 if not self.options.force:
144 logging.error('Detected that the device has rootfs verification enabled.')
145 logging.info('This script can automatically remove the rootfs '
146 'verification, which requires that it reboot the device.')
147 logging.info('Make sure the device is in developer mode!')
148 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700149 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800150 # Since we stopped Chrome earlier, it's good form to start it up again.
151 if self.options.startui:
152 logging.info('Starting Chrome...')
153 self.host.RemoteSh('start ui')
154 raise DeployFailure('Need rootfs verification to be disabled. '
155 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156
157 logging.info('Removing rootfs verification from %s', self.options.to)
158 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
159 # Use --force to bypass the checks.
160 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
161 '--remove_rootfs_verification --force')
162 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
163 self.host.RemoteSh(cmd % partition, error_code_ok=True)
164
165 # A reboot in developer mode takes a while (and has delays), so the user
166 # will have time to read and act on the USB boot instructions below.
167 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
168 self.host.RemoteReboot()
169
David James88e6f032013-03-02 08:13:20 -0800170 # Now that the machine has been rebooted, we need to kill Chrome again.
171 self._KillProcsIfNeeded()
172
173 # Make sure the rootfs is writable now.
174 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700175
176 def _CheckUiJobStarted(self):
177 # status output is in the format:
178 # <job_name> <status> ['process' <pid>].
179 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800180 try:
181 result = self.host.RemoteSh('status ui')
182 except cros_build_lib.RunCommandError as e:
183 if 'Unknown job' in e.result.error:
184 return False
185 else:
186 raise e
187
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188 return result.output.split()[1].split('/')[0] == 'start'
189
190 def _KillProcsIfNeeded(self):
Steve Funge984a532013-11-25 17:09:25 -0800191 if self.content_shell:
192 logging.info('Shutting down content_shell...')
193 self.host.RemoteSh('stop content_shell')
194 return
195
Ryan Cui3045c5d2012-07-13 18:00:33 -0700196 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800197 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 self.host.RemoteSh('stop ui')
199
200 # Developers sometimes run session_manager manually, in which case we'll
201 # need to help shut the chrome processes down.
202 try:
David James3432acd2013-11-27 10:02:18 -0800203 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700204 while self._ChromeFileInUse():
205 logging.warning('The chrome binary on the device is in use.')
206 logging.warning('Killing chrome and session_manager processes...\n')
207
208 self.host.RemoteSh("pkill 'chrome|session_manager'",
209 error_code_ok=True)
210 # Wait for processes to actually terminate
211 time.sleep(POST_KILL_WAIT)
212 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800213 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800214 msg = ('Could not kill processes after %s seconds. Please exit any '
215 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
216 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700217
David James88e6f032013-03-02 08:13:20 -0800218 def _MountRootfsAsWritable(self, error_code_ok=True):
219 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700220
David James88e6f032013-03-02 08:13:20 -0800221 If the command fails, and error_code_ok is True, then this function sets
222 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Mike Frysinger02e1e072013-11-10 22:11:34 -0500224 Args:
David James88e6f032013-03-02 08:13:20 -0800225 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
226 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800227 # TODO: Should migrate to use the remount functions in remote_access.
Steve Funge984a532013-11-25 17:09:25 -0800228 result = self.host.RemoteSh(MOUNT_RW_COMMAND if not self.content_shell
229 else MOUNT_RW_COMMAND_ANDROID,
230 error_code_ok=error_code_ok)
David James88e6f032013-03-02 08:13:20 -0800231 if result.returncode:
232 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700233
Ryan Cui7193a7e2013-04-26 14:15:19 -0700234 def _GetDeviceInfo(self):
235 steps = [
236 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
237 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
238 ]
239 return_values = parallel.RunParallelSteps(steps, return_values=True)
240 return DeviceInfo(*return_values)
241
242 def _CheckDeviceFreeSpace(self, device_info):
243 """See if target device has enough space for Chrome.
244
Mike Frysinger02e1e072013-11-10 22:11:34 -0500245 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700246 device_info: A DeviceInfo named tuple.
247 """
248 effective_free = device_info.target_dir_size + device_info.target_fs_free
249 staging_size = self._GetStagingDirSize()
Steve Funge984a532013-11-25 17:09:25 -0800250 # For content shell deployments, which do not contain the du binary,
251 # do not raise DeployFailure since can't get exact free space available
252 # for staging files.
Ryan Cui7193a7e2013-04-26 14:15:19 -0700253 if effective_free < staging_size:
Steve Funge984a532013-11-25 17:09:25 -0800254 if self.content_shell:
255 logging.warning('Not enough free space on the device. If overwriting '
256 'files, deployment may still succeed. Required: %s '
257 'MiB, actual: %s MiB.', staging_size / 1024,
258 effective_free / 1024)
259 else:
260 raise DeployFailure(
261 'Not enough free space on the device. Required: %s MiB, '
262 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700263 if device_info.target_fs_free < (100 * 1024):
264 logging.warning('The device has less than 100MB free. deploy_chrome may '
265 'hang during the transfer.')
266
Ryan Cui3045c5d2012-07-13 18:00:33 -0700267 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800268 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700269 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800270 dest_path = _CHROME_DIR
271 if self.content_shell:
272 try:
273 self.host.Scp('%s/*' % os.path.abspath(self.staging_dir),
274 '%s/' % self.options.target_dir,
275 recursive=True,
276 debug_level=logging.INFO,
277 verbose=self.options.verbose)
278 except cros_build_lib.RunCommandError as ex:
279 if ex.result.returncode != 1:
280 logging.error('Scp failure [%s]', ex.result.returncode)
281 raise DeployFailure(ex)
282 else:
283 # TODO(stevefung): Update Dropbear SSHD on device.
284 # http://crbug.com/329656
285 logging.info('Potential conflict with DropBear SSHD return status')
286
287 dest_path = _ANDROID_DIR
288 else:
289 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
290 self.options.target_dir,
291 inplace=True, debug_level=logging.INFO,
292 verbose=self.options.verbose)
293
294 for p in self.copy_paths:
295 if p.owner:
296 self.host.RemoteSh('chown %s %s/%s' % (p.owner, dest_path,
297 p.src if not p.dest else p.dest))
298 if p.mode:
299 # Set mode if necessary.
300 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
301 p.src if not p.dest else p.dest))
302
303
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800304 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800305 logging.info('Starting UI...')
306 if self.content_shell:
307 self.host.RemoteSh('start content_shell')
308 else:
309 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700310
David James88e6f032013-03-02 08:13:20 -0800311 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700312 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800313 logging.info('Testing connection to the device...')
Steve Funge984a532013-11-25 17:09:25 -0800314 if self.content_shell:
315 # true command over ssh returns error code 255, so as workaround
316 # use `sleep 0` as no-op.
317 self.host.RemoteSh('sleep 0')
318 else:
319 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800320 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700321 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800322 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700323
Steve Funge984a532013-11-25 17:09:25 -0800324 def _CheckDeployType(self):
325 if self.options.build_dir and os.path.exists(
326 os.path.join(self.options.build_dir, 'system.unand')):
327 # Content shell deployment.
328 self.content_shell = True
329 self.options.build_dir = os.path.join(self.options.build_dir,
330 'system.unand/chrome/')
331 self.options.dostrip = False
332 self.options.target_dir = _ANDROID_DIR
333 self.copy_paths = chrome_util.GetCopyPaths(True)
334 elif self.options.local_pkg_path or self.options.gs_path:
335 # Package deployment.
336 pkg_path = self.options.local_pkg_path
337 if self.options.gs_path:
338 pkg_path = _FetchChromePackage(self.options.cache_dir, self.tempdir,
339 self.options.gs_path)
340
341 assert pkg_path
342 logging.info('Checking %s for content_shell...', pkg_path)
343 if pkg_path[-4:] == '.zip':
344 zip_pkg = zipfile.ZipFile(pkg_path)
345 if any('eureka_shell' in name for name in zip_pkg.namelist()):
346 self.content_shell = True
347 zip_pkg.close()
348 else:
349 tar = tarfile.open(pkg_path)
350 if any('eureka_shell' in member.name for member in tar.getmembers()):
351 self.content_shell = True
352 tar.close()
353
354 if self.content_shell:
355 self.options.dostrip = False
356 self.options.target_dir = _ANDROID_DIR
357 self.copy_paths = chrome_util.GetCopyPaths(True)
358 self.chrome_dir = _ANDROID_DIR
359
David James88e6f032013-03-02 08:13:20 -0800360 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800361 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
362 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800363
Thiago Goncales12793312013-05-23 11:26:17 -0700364 def _MountTarget(self):
365 logging.info('Mounting Chrome...')
366
367 # Create directory if does not exist
368 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
369 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
370 self.options.mount_dir))
371 # Chrome needs partition to have exec and suid flags set
372 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
373
David James88e6f032013-03-02 08:13:20 -0800374 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800375 self._CheckDeployType()
376
David James88e6f032013-03-02 08:13:20 -0800377 # If requested, just do the staging step.
378 if self.options.staging_only:
379 self._PrepareStagingDir()
380 return 0
381
382 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800383 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800384 steps = [self._GetDeviceInfo, self._CheckConnection,
385 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
386 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700387 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
388 return_values=True)
389 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800390
391 # If we failed to mark the rootfs as writable, try disabling rootfs
392 # verification.
393 if self._rootfs_is_still_readonly.is_set():
394 self._DisableRootfsVerification()
395
Thiago Goncales12793312013-05-23 11:26:17 -0700396 if self.options.mount_dir is not None:
397 self._MountTarget()
398
David James88e6f032013-03-02 08:13:20 -0800399 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700400 self._Deploy()
401
402
Ryan Cuia56a71e2012-10-18 18:40:35 -0700403def ValidateGypDefines(_option, _opt, value):
404 """Convert GYP_DEFINES-formatted string to dictionary."""
405 return chrome_util.ProcessGypDefines(value)
406
407
408class CustomOption(commandline.Option):
409 """Subclass Option class to implement path evaluation."""
410 TYPES = commandline.Option.TYPES + ('gyp_defines',)
411 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
412 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
413
414
Ryan Cuie535b172012-10-19 18:25:03 -0700415def _CreateParser():
416 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800417 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
418 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700419
Ryan Cuia56a71e2012-10-18 18:40:35 -0700420 # TODO(rcui): Have this use the UI-V2 format of having source and target
421 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700422 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800423 help='Skip all prompts (i.e., for disabling of rootfs '
424 'verification). This may result in the target '
425 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800426 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
427 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800428 help="The board the Chrome build is targeted for. When in "
429 "a 'cros chrome-sdk' shell, defaults to the SDK "
430 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700431 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800432 help='The directory with Chrome build artifacts to deploy '
433 'from. Typically of format <chrome_root>/out/Debug. '
434 'When this option is used, the GYP_DEFINES '
435 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800436 parser.add_option('--target-dir', type='path',
437 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700438 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700439 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800440 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800441 parser.add_option('--nostartui', action='store_false', dest='startui',
442 default=True,
443 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800444 parser.add_option('--nostrip', action='store_false', dest='dostrip',
445 default=True,
446 help="Don't strip binaries during deployment. Warning: "
447 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700448 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800449 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700450 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800451 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700452 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800453 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700454 parser.add_option('--mount-dir', type='path', default=None,
455 help='Deploy Chrome in target directory and bind it'
456 'to directory specified by this flag.')
457 parser.add_option('--mount', action='store_true', default=False,
458 help='Deploy Chrome to default target directory and bind it'
459 'to default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700460
461 group = optparse.OptionGroup(parser, 'Advanced Options')
462 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800463 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800464 group.add_option('--sloppy', action='store_true', default=False,
465 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700466 group.add_option('--staging-flags', default=None, type='gyp_defines',
467 help='Extra flags to control staging. Valid flags are - %s'
468 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800469 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800470 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800471 'variable and --staging-flags, if set. Enforce that '
472 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700473 group.add_option('--strip-flags', default=None,
474 help="Flags to call the 'strip' binutil tool with. "
475 "Overrides the default arguments.")
Ryan Cuief91e702013-02-04 12:06:36 -0800476
Ryan Cuia56a71e2012-10-18 18:40:35 -0700477 parser.add_option_group(group)
478
Ryan Cuif890a3e2013-03-07 18:57:06 -0800479 # GYP_DEFINES that Chrome was built with. Influences which files are staged
480 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
481 # enviroment variable.
482 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
483 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700484 # Path of an empty directory to stage chrome artifacts to. Defaults to a
485 # temporary directory that is removed when the script finishes. If the path
486 # is specified, then it will not be removed.
487 parser.add_option('--staging-dir', type='path', default=None,
488 help=optparse.SUPPRESS_HELP)
489 # Only prepare the staging directory, and skip deploying to the device.
490 parser.add_option('--staging-only', action='store_true', default=False,
491 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700492 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
493 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
494 # fetching the SDK toolchain.
495 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700496 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700497
Ryan Cuie535b172012-10-19 18:25:03 -0700498
499def _ParseCommandLine(argv):
500 """Parse args, and run environment-independent checks."""
501 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700502 (options, args) = parser.parse_args(argv)
503
Ryan Cuia56a71e2012-10-18 18:40:35 -0700504 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
505 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800506 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700507 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
508 parser.error('Cannot specify both --build_dir and '
509 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800510 if options.build_dir and not options.board:
511 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700512 if options.gs_path and options.local_pkg_path:
513 parser.error('Cannot specify both --gs-path and --local-pkg-path')
514 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700515 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800516 if (options.strict or options.staging_flags) and not options.build_dir:
517 parser.error('--strict and --staging-flags require --build-dir to be '
518 'set.')
519 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800520 parser.error('--staging-flags requires --strict to be set.')
521 if options.sloppy and options.strict:
522 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700523
524 if options.mount or options.mount_dir:
525 if not options.target_dir:
526 options.target_dir = _CHROME_DIR_MOUNT
527 else:
528 if not options.target_dir:
529 options.target_dir = _CHROME_DIR
530
531 if options.mount and not options.mount_dir:
532 options.mount_dir = _CHROME_DIR
533
Ryan Cui3045c5d2012-07-13 18:00:33 -0700534 return options, args
535
536
Ryan Cuie535b172012-10-19 18:25:03 -0700537def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800538 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700539
540 Args:
Steve Funge984a532013-11-25 17:09:25 -0800541 options: The options object returned by optparse.
542 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700543 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700544 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
545 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
546
Ryan Cuib623e7b2013-03-14 12:54:11 -0700547 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700548 gyp_env = os.getenv('GYP_DEFINES', None)
549 if gyp_env is not None:
550 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700551 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700552 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700553
554 if options.strict and not options.gyp_defines:
555 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800556 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700557
Ryan Cui3c183c22013-04-29 18:04:11 -0700558 if options.build_dir:
559 chrome_path = os.path.join(options.build_dir, 'chrome')
560 if os.path.isfile(chrome_path):
561 deps = lddtree.ParseELF(chrome_path)
562 if 'libbase.so' in deps['libs']:
563 cros_build_lib.Warning(
564 'Detected a component build of Chrome. component build is '
565 'not working properly for Chrome OS. See crbug.com/196317. '
566 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700567
Ryan Cuia56a71e2012-10-18 18:40:35 -0700568
Ryan Cui504db722013-01-22 11:48:01 -0800569def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700570 """Get the chrome prebuilt tarball from GS.
571
Mike Frysinger02e1e072013-11-10 22:11:34 -0500572 Returns:
573 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700574 """
David James9374aac2013-10-08 16:00:17 -0700575 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800576 files = gs_ctx.LS(gs_path).output.splitlines()
577 files = [found for found in files if
578 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
579 if not files:
580 raise Exception('No chrome package found at %s' % gs_path)
581 elif len(files) > 1:
582 # - Users should provide us with a direct link to either a stripped or
583 # unstripped chrome package.
584 # - In the case of being provided with an archive directory, where both
585 # stripped and unstripped chrome available, use the stripped chrome
586 # package.
587 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
588 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
589 files = [f for f in files if not 'unstripped' in f]
590 assert len(files) == 1
591 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800592
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800593 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800594 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800595 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
596 chrome_path = os.path.join(tempdir, filename)
597 assert os.path.exists(chrome_path)
598 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700599
600
Ryan Cuif890a3e2013-03-07 18:57:06 -0800601@contextlib.contextmanager
602def _StripBinContext(options):
603 if not options.dostrip:
604 yield None
605 elif options.strip_bin:
606 yield options.strip_bin
607 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800608 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800609 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
610 with sdk.Prepare(components=components) as ctx:
611 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
612 constants.CHROME_ENV_FILE)
613 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
614 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
615 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800616 yield strip_bin
617
618
Steve Funge984a532013-11-25 17:09:25 -0800619def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
620 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800621 """Place the necessary files in the staging directory.
622
623 The staging directory is the directory used to rsync the build artifacts over
624 to the device. Only the necessary Chrome build artifacts are put into the
625 staging directory.
626 """
Ryan Cui5866be02013-03-18 14:12:00 -0700627 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400628 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800629 if options.build_dir:
630 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700631 strip_flags = (None if options.strip_flags is None else
632 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800633 chrome_util.StageChromeFromBuildDir(
634 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800635 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700636 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800637 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700638 else:
639 pkg_path = options.local_pkg_path
640 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800641 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
642 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700643
644 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800645 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700646 # Extract only the ./opt/google/chrome contents, directly into the staging
647 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800648 if pkg_path[-4:] == '.zip':
649 cros_build_lib.DebugRunCommand(
650 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
651 staging_dir])
652 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
653 shutil.move(filename, staging_dir)
654 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
655 else:
656 cros_build_lib.DebugRunCommand(
657 ['tar', '--strip-components', '4', '--extract',
658 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
659 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700660
Ryan Cui71aa8de2013-04-19 16:12:55 -0700661
Ryan Cui3045c5d2012-07-13 18:00:33 -0700662def main(argv):
663 options, args = _ParseCommandLine(argv)
664 _PostParseCheck(options, args)
665
666 # Set cros_build_lib debug level to hide RunCommand spew.
667 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700668 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700669 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800670 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700671
Ryan Cui71aa8de2013-04-19 16:12:55 -0700672 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700673 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
674 if cmd_stats:
675 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700676
Ryan Cui71aa8de2013-04-19 16:12:55 -0700677 with osutils.TempDir(set_global=True) as tempdir:
678 staging_dir = options.staging_dir
679 if not staging_dir:
680 staging_dir = os.path.join(tempdir, 'chrome')
681
682 deploy = DeployChrome(options, tempdir, staging_dir)
683 try:
684 deploy.Perform()
685 except results_lib.StepFailure as ex:
686 raise SystemExit(str(ex).strip())