blob: d78a6e75c5be763438307c71930f86be837f9f13 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
Steve Funge984a532013-11-25 17:09:25 -08007"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07008
9The script supports deploying Chrome from these sources:
10
111. A local build output directory, such as chromium/src/out/[Debug|Release].
122. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
133. A Chrome tarball existing locally.
14
15The script copies the necessary contents of the source location (tarball or
16build directory) and rsyncs the contents of the staging directory onto your
17device's rootfs.
18"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070019
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070024import logging
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070027import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
30import tarfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070031import time
Steve Funge984a532013-11-25 17:09:25 -080032import zipfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070033
Ryan Cuia56a71e2012-10-18 18:40:35 -070034
David James629febb2012-11-25 13:07:34 -080035from chromite.buildbot import constants
David James88e6f032013-03-02 08:13:20 -080036from chromite.buildbot import cbuildbot_results as results_lib
Ryan Cui686ec052013-02-12 16:39:41 -080037from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070038from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070040from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080041from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080043from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070044from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070045from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080046from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070047from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070048
49
Ryan Cuia56a71e2012-10-18 18:40:35 -070050_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
51
Ryan Cui3045c5d2012-07-13 18:00:33 -070052KERNEL_A_PARTITION = 2
53KERNEL_B_PARTITION = 4
54
55KILL_PROC_MAX_WAIT = 10
56POST_KILL_WAIT = 2
57
Ryan Cuie535b172012-10-19 18:25:03 -070058MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080059LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070060
Steve Funge984a532013-11-25 17:09:25 -080061MOUNT_RW_COMMAND_ANDROID = 'mount -o remount,rw /system'
62
63_ANDROID_DIR = '/system/chrome'
64_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
65
David James2cb34002013-03-01 18:42:40 -080066_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070067_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080068
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
73DF_COMMAND_ANDROID = 'df %s'
74
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
David James88e6f032013-03-02 08:13:20 -080080class DeployFailure(results_lib.StepFailure):
81 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700102 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800103 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Steve Funge984a532013-11-25 17:09:25 -0800105 # Used to track whether deploying content_shell or chrome to a device.
106 self.content_shell = False
107 self.copy_paths = chrome_util.GetCopyPaths(False)
108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800111 result = self.host.RemoteSh((DF_COMMAND if not self.content_shell
112 else DF_COMMAND_ANDROID) % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700113 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800114 value = line.split()[3]
115 multipliers = {
116 'G': 1024 * 1024 * 1024,
117 'M': 1024 * 1024,
118 'K': 1024,
119 }
120 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700121
122 def _GetRemoteDirSize(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800123 if self.content_shell:
124 # Content Shell devices currently do not contain the du binary.
125 logging.warning('Remote host does not contain du; cannot get remote '
126 'directory size to properly calculate available free '
127 'space.')
128 return 0
Ryan Cui7193a7e2013-04-26 14:15:19 -0700129 result = self.host.RemoteSh('du -ks %s' % remote_dir)
130 return int(result.output.split()[0])
131
132 def _GetStagingDirSize(self):
133 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
134 redirect_stdout=True)
135 return int(result.output.split()[0])
136
Ryan Cui3045c5d2012-07-13 18:00:33 -0700137 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800138 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
139 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700140 return result.returncode == 0
141
142 def _DisableRootfsVerification(self):
143 if not self.options.force:
144 logging.error('Detected that the device has rootfs verification enabled.')
145 logging.info('This script can automatically remove the rootfs '
146 'verification, which requires that it reboot the device.')
147 logging.info('Make sure the device is in developer mode!')
148 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700149 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800150 # Since we stopped Chrome earlier, it's good form to start it up again.
151 if self.options.startui:
152 logging.info('Starting Chrome...')
153 self.host.RemoteSh('start ui')
154 raise DeployFailure('Need rootfs verification to be disabled. '
155 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700156
157 logging.info('Removing rootfs verification from %s', self.options.to)
158 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
159 # Use --force to bypass the checks.
160 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
161 '--remove_rootfs_verification --force')
162 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
163 self.host.RemoteSh(cmd % partition, error_code_ok=True)
164
165 # A reboot in developer mode takes a while (and has delays), so the user
166 # will have time to read and act on the USB boot instructions below.
167 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
168 self.host.RemoteReboot()
169
David James88e6f032013-03-02 08:13:20 -0800170 # Now that the machine has been rebooted, we need to kill Chrome again.
171 self._KillProcsIfNeeded()
172
173 # Make sure the rootfs is writable now.
174 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700175
176 def _CheckUiJobStarted(self):
177 # status output is in the format:
178 # <job_name> <status> ['process' <pid>].
179 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800180 try:
181 result = self.host.RemoteSh('status ui')
182 except cros_build_lib.RunCommandError as e:
183 if 'Unknown job' in e.result.error:
184 return False
185 else:
186 raise e
187
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188 return result.output.split()[1].split('/')[0] == 'start'
189
190 def _KillProcsIfNeeded(self):
Steve Funge984a532013-11-25 17:09:25 -0800191 if self.content_shell:
192 logging.info('Shutting down content_shell...')
193 self.host.RemoteSh('stop content_shell')
194 return
195
Ryan Cui3045c5d2012-07-13 18:00:33 -0700196 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800197 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 self.host.RemoteSh('stop ui')
199
200 # Developers sometimes run session_manager manually, in which case we'll
201 # need to help shut the chrome processes down.
202 try:
David James3432acd2013-11-27 10:02:18 -0800203 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700204 while self._ChromeFileInUse():
205 logging.warning('The chrome binary on the device is in use.')
206 logging.warning('Killing chrome and session_manager processes...\n')
207
208 self.host.RemoteSh("pkill 'chrome|session_manager'",
209 error_code_ok=True)
210 # Wait for processes to actually terminate
211 time.sleep(POST_KILL_WAIT)
212 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800213 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800214 msg = ('Could not kill processes after %s seconds. Please exit any '
215 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
216 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700217
David James88e6f032013-03-02 08:13:20 -0800218 def _MountRootfsAsWritable(self, error_code_ok=True):
219 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700220
David James88e6f032013-03-02 08:13:20 -0800221 If the command fails, and error_code_ok is True, then this function sets
222 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Mike Frysinger02e1e072013-11-10 22:11:34 -0500224 Args:
David James88e6f032013-03-02 08:13:20 -0800225 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
226 """
Steve Funge984a532013-11-25 17:09:25 -0800227 result = self.host.RemoteSh(MOUNT_RW_COMMAND if not self.content_shell
228 else MOUNT_RW_COMMAND_ANDROID,
229 error_code_ok=error_code_ok)
David James88e6f032013-03-02 08:13:20 -0800230 if result.returncode:
231 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700232
Ryan Cui7193a7e2013-04-26 14:15:19 -0700233 def _GetDeviceInfo(self):
234 steps = [
235 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
236 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
237 ]
238 return_values = parallel.RunParallelSteps(steps, return_values=True)
239 return DeviceInfo(*return_values)
240
241 def _CheckDeviceFreeSpace(self, device_info):
242 """See if target device has enough space for Chrome.
243
Mike Frysinger02e1e072013-11-10 22:11:34 -0500244 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700245 device_info: A DeviceInfo named tuple.
246 """
247 effective_free = device_info.target_dir_size + device_info.target_fs_free
248 staging_size = self._GetStagingDirSize()
Steve Funge984a532013-11-25 17:09:25 -0800249 # For content shell deployments, which do not contain the du binary,
250 # do not raise DeployFailure since can't get exact free space available
251 # for staging files.
Ryan Cui7193a7e2013-04-26 14:15:19 -0700252 if effective_free < staging_size:
Steve Funge984a532013-11-25 17:09:25 -0800253 if self.content_shell:
254 logging.warning('Not enough free space on the device. If overwriting '
255 'files, deployment may still succeed. Required: %s '
256 'MiB, actual: %s MiB.', staging_size / 1024,
257 effective_free / 1024)
258 else:
259 raise DeployFailure(
260 'Not enough free space on the device. Required: %s MiB, '
261 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700262 if device_info.target_fs_free < (100 * 1024):
263 logging.warning('The device has less than 100MB free. deploy_chrome may '
264 'hang during the transfer.')
265
Ryan Cui3045c5d2012-07-13 18:00:33 -0700266 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800267 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700268 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800269 dest_path = _CHROME_DIR
270 if self.content_shell:
271 try:
272 self.host.Scp('%s/*' % os.path.abspath(self.staging_dir),
273 '%s/' % self.options.target_dir,
274 recursive=True,
275 debug_level=logging.INFO,
276 verbose=self.options.verbose)
277 except cros_build_lib.RunCommandError as ex:
278 if ex.result.returncode != 1:
279 logging.error('Scp failure [%s]', ex.result.returncode)
280 raise DeployFailure(ex)
281 else:
282 # TODO(stevefung): Update Dropbear SSHD on device.
283 # http://crbug.com/329656
284 logging.info('Potential conflict with DropBear SSHD return status')
285
286 dest_path = _ANDROID_DIR
287 else:
288 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
289 self.options.target_dir,
290 inplace=True, debug_level=logging.INFO,
291 verbose=self.options.verbose)
292
293 for p in self.copy_paths:
294 if p.owner:
295 self.host.RemoteSh('chown %s %s/%s' % (p.owner, dest_path,
296 p.src if not p.dest else p.dest))
297 if p.mode:
298 # Set mode if necessary.
299 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
300 p.src if not p.dest else p.dest))
301
302
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800303 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800304 logging.info('Starting UI...')
305 if self.content_shell:
306 self.host.RemoteSh('start content_shell')
307 else:
308 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700309
David James88e6f032013-03-02 08:13:20 -0800310 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700311 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800312 logging.info('Testing connection to the device...')
Steve Funge984a532013-11-25 17:09:25 -0800313 if self.content_shell:
314 # true command over ssh returns error code 255, so as workaround
315 # use `sleep 0` as no-op.
316 self.host.RemoteSh('sleep 0')
317 else:
318 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800319 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700320 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800321 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700322
Steve Funge984a532013-11-25 17:09:25 -0800323 def _CheckDeployType(self):
324 if self.options.build_dir and os.path.exists(
325 os.path.join(self.options.build_dir, 'system.unand')):
326 # Content shell deployment.
327 self.content_shell = True
328 self.options.build_dir = os.path.join(self.options.build_dir,
329 'system.unand/chrome/')
330 self.options.dostrip = False
331 self.options.target_dir = _ANDROID_DIR
332 self.copy_paths = chrome_util.GetCopyPaths(True)
333 elif self.options.local_pkg_path or self.options.gs_path:
334 # Package deployment.
335 pkg_path = self.options.local_pkg_path
336 if self.options.gs_path:
337 pkg_path = _FetchChromePackage(self.options.cache_dir, self.tempdir,
338 self.options.gs_path)
339
340 assert pkg_path
341 logging.info('Checking %s for content_shell...', pkg_path)
342 if pkg_path[-4:] == '.zip':
343 zip_pkg = zipfile.ZipFile(pkg_path)
344 if any('eureka_shell' in name for name in zip_pkg.namelist()):
345 self.content_shell = True
346 zip_pkg.close()
347 else:
348 tar = tarfile.open(pkg_path)
349 if any('eureka_shell' in member.name for member in tar.getmembers()):
350 self.content_shell = True
351 tar.close()
352
353 if self.content_shell:
354 self.options.dostrip = False
355 self.options.target_dir = _ANDROID_DIR
356 self.copy_paths = chrome_util.GetCopyPaths(True)
357 self.chrome_dir = _ANDROID_DIR
358
David James88e6f032013-03-02 08:13:20 -0800359 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800360 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
361 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800362
Thiago Goncales12793312013-05-23 11:26:17 -0700363 def _MountTarget(self):
364 logging.info('Mounting Chrome...')
365
366 # Create directory if does not exist
367 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
368 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
369 self.options.mount_dir))
370 # Chrome needs partition to have exec and suid flags set
371 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
372
David James88e6f032013-03-02 08:13:20 -0800373 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800374 self._CheckDeployType()
375
David James88e6f032013-03-02 08:13:20 -0800376 # If requested, just do the staging step.
377 if self.options.staging_only:
378 self._PrepareStagingDir()
379 return 0
380
381 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800382 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800383 steps = [self._GetDeviceInfo, self._CheckConnection,
384 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
385 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700386 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
387 return_values=True)
388 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800389
390 # If we failed to mark the rootfs as writable, try disabling rootfs
391 # verification.
392 if self._rootfs_is_still_readonly.is_set():
393 self._DisableRootfsVerification()
394
Thiago Goncales12793312013-05-23 11:26:17 -0700395 if self.options.mount_dir is not None:
396 self._MountTarget()
397
David James88e6f032013-03-02 08:13:20 -0800398 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700399 self._Deploy()
400
401
Ryan Cuia56a71e2012-10-18 18:40:35 -0700402def ValidateGypDefines(_option, _opt, value):
403 """Convert GYP_DEFINES-formatted string to dictionary."""
404 return chrome_util.ProcessGypDefines(value)
405
406
407class CustomOption(commandline.Option):
408 """Subclass Option class to implement path evaluation."""
409 TYPES = commandline.Option.TYPES + ('gyp_defines',)
410 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
411 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
412
413
Ryan Cuie535b172012-10-19 18:25:03 -0700414def _CreateParser():
415 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800416 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
417 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700418
Ryan Cuia56a71e2012-10-18 18:40:35 -0700419 # TODO(rcui): Have this use the UI-V2 format of having source and target
420 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700421 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800422 help='Skip all prompts (i.e., for disabling of rootfs '
423 'verification). This may result in the target '
424 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800425 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
426 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800427 help="The board the Chrome build is targeted for. When in "
428 "a 'cros chrome-sdk' shell, defaults to the SDK "
429 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700430 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800431 help='The directory with Chrome build artifacts to deploy '
432 'from. Typically of format <chrome_root>/out/Debug. '
433 'When this option is used, the GYP_DEFINES '
434 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800435 parser.add_option('--target-dir', type='path',
436 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700437 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700438 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800439 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800440 parser.add_option('--nostartui', action='store_false', dest='startui',
441 default=True,
442 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800443 parser.add_option('--nostrip', action='store_false', dest='dostrip',
444 default=True,
445 help="Don't strip binaries during deployment. Warning: "
446 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700447 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800448 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700449 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800450 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700451 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800452 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700453 parser.add_option('--mount-dir', type='path', default=None,
454 help='Deploy Chrome in target directory and bind it'
455 'to directory specified by this flag.')
456 parser.add_option('--mount', action='store_true', default=False,
457 help='Deploy Chrome to default target directory and bind it'
458 'to default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700459
460 group = optparse.OptionGroup(parser, 'Advanced Options')
461 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800462 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800463 group.add_option('--sloppy', action='store_true', default=False,
464 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700465 group.add_option('--staging-flags', default=None, type='gyp_defines',
466 help='Extra flags to control staging. Valid flags are - %s'
467 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800468 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800469 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800470 'variable and --staging-flags, if set. Enforce that '
471 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700472 group.add_option('--strip-flags', default=None,
473 help="Flags to call the 'strip' binutil tool with. "
474 "Overrides the default arguments.")
Ryan Cuief91e702013-02-04 12:06:36 -0800475
Ryan Cuia56a71e2012-10-18 18:40:35 -0700476 parser.add_option_group(group)
477
Ryan Cuif890a3e2013-03-07 18:57:06 -0800478 # GYP_DEFINES that Chrome was built with. Influences which files are staged
479 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
480 # enviroment variable.
481 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
482 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700483 # Path of an empty directory to stage chrome artifacts to. Defaults to a
484 # temporary directory that is removed when the script finishes. If the path
485 # is specified, then it will not be removed.
486 parser.add_option('--staging-dir', type='path', default=None,
487 help=optparse.SUPPRESS_HELP)
488 # Only prepare the staging directory, and skip deploying to the device.
489 parser.add_option('--staging-only', action='store_true', default=False,
490 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700491 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
492 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
493 # fetching the SDK toolchain.
494 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700495 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700496
Ryan Cuie535b172012-10-19 18:25:03 -0700497
498def _ParseCommandLine(argv):
499 """Parse args, and run environment-independent checks."""
500 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700501 (options, args) = parser.parse_args(argv)
502
Ryan Cuia56a71e2012-10-18 18:40:35 -0700503 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
504 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800505 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700506 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
507 parser.error('Cannot specify both --build_dir and '
508 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800509 if options.build_dir and not options.board:
510 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700511 if options.gs_path and options.local_pkg_path:
512 parser.error('Cannot specify both --gs-path and --local-pkg-path')
513 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700514 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800515 if (options.strict or options.staging_flags) and not options.build_dir:
516 parser.error('--strict and --staging-flags require --build-dir to be '
517 'set.')
518 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800519 parser.error('--staging-flags requires --strict to be set.')
520 if options.sloppy and options.strict:
521 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700522
523 if options.mount or options.mount_dir:
524 if not options.target_dir:
525 options.target_dir = _CHROME_DIR_MOUNT
526 else:
527 if not options.target_dir:
528 options.target_dir = _CHROME_DIR
529
530 if options.mount and not options.mount_dir:
531 options.mount_dir = _CHROME_DIR
532
Ryan Cui3045c5d2012-07-13 18:00:33 -0700533 return options, args
534
535
Ryan Cuie535b172012-10-19 18:25:03 -0700536def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800537 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700538
539 Args:
Steve Funge984a532013-11-25 17:09:25 -0800540 options: The options object returned by optparse.
541 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700542 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700543 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
544 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
545
Ryan Cuib623e7b2013-03-14 12:54:11 -0700546 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700547 gyp_env = os.getenv('GYP_DEFINES', None)
548 if gyp_env is not None:
549 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700550 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700552
553 if options.strict and not options.gyp_defines:
554 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800555 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700556
Ryan Cui3c183c22013-04-29 18:04:11 -0700557 if options.build_dir:
558 chrome_path = os.path.join(options.build_dir, 'chrome')
559 if os.path.isfile(chrome_path):
560 deps = lddtree.ParseELF(chrome_path)
561 if 'libbase.so' in deps['libs']:
562 cros_build_lib.Warning(
563 'Detected a component build of Chrome. component build is '
564 'not working properly for Chrome OS. See crbug.com/196317. '
565 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700566
Ryan Cuia56a71e2012-10-18 18:40:35 -0700567
Ryan Cui504db722013-01-22 11:48:01 -0800568def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700569 """Get the chrome prebuilt tarball from GS.
570
Mike Frysinger02e1e072013-11-10 22:11:34 -0500571 Returns:
572 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700573 """
David James9374aac2013-10-08 16:00:17 -0700574 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800575 files = gs_ctx.LS(gs_path).output.splitlines()
576 files = [found for found in files if
577 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
578 if not files:
579 raise Exception('No chrome package found at %s' % gs_path)
580 elif len(files) > 1:
581 # - Users should provide us with a direct link to either a stripped or
582 # unstripped chrome package.
583 # - In the case of being provided with an archive directory, where both
584 # stripped and unstripped chrome available, use the stripped chrome
585 # package.
586 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
587 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
588 files = [f for f in files if not 'unstripped' in f]
589 assert len(files) == 1
590 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800591
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800592 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800593 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800594 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
595 chrome_path = os.path.join(tempdir, filename)
596 assert os.path.exists(chrome_path)
597 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700598
599
Ryan Cuif890a3e2013-03-07 18:57:06 -0800600@contextlib.contextmanager
601def _StripBinContext(options):
602 if not options.dostrip:
603 yield None
604 elif options.strip_bin:
605 yield options.strip_bin
606 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800607 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800608 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
609 with sdk.Prepare(components=components) as ctx:
610 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
611 constants.CHROME_ENV_FILE)
612 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
613 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
614 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800615 yield strip_bin
616
617
Steve Funge984a532013-11-25 17:09:25 -0800618def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
619 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800620 """Place the necessary files in the staging directory.
621
622 The staging directory is the directory used to rsync the build artifacts over
623 to the device. Only the necessary Chrome build artifacts are put into the
624 staging directory.
625 """
Ryan Cui5866be02013-03-18 14:12:00 -0700626 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400627 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800628 if options.build_dir:
629 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700630 strip_flags = (None if options.strip_flags is None else
631 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800632 chrome_util.StageChromeFromBuildDir(
633 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800634 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700635 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800636 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700637 else:
638 pkg_path = options.local_pkg_path
639 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800640 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
641 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700642
643 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800644 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700645 # Extract only the ./opt/google/chrome contents, directly into the staging
646 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800647 if pkg_path[-4:] == '.zip':
648 cros_build_lib.DebugRunCommand(
649 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
650 staging_dir])
651 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
652 shutil.move(filename, staging_dir)
653 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
654 else:
655 cros_build_lib.DebugRunCommand(
656 ['tar', '--strip-components', '4', '--extract',
657 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
658 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700659
Ryan Cui71aa8de2013-04-19 16:12:55 -0700660
Ryan Cui3045c5d2012-07-13 18:00:33 -0700661def main(argv):
662 options, args = _ParseCommandLine(argv)
663 _PostParseCheck(options, args)
664
665 # Set cros_build_lib debug level to hide RunCommand spew.
666 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700667 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700668 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800669 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700670
Ryan Cui71aa8de2013-04-19 16:12:55 -0700671 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700672 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
673 if cmd_stats:
674 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700675
Ryan Cui71aa8de2013-04-19 16:12:55 -0700676 with osutils.TempDir(set_global=True) as tempdir:
677 staging_dir = options.staging_dir
678 if not staging_dir:
679 staging_dir = os.path.join(tempdir, 'chrome')
680
681 deploy = DeployChrome(options, tempdir, staging_dir)
682 try:
683 deploy.Perform()
684 except results_lib.StepFailure as ex:
685 raise SystemExit(str(ex).strip())