blob: d18aa8138df7bcdc0e258249e3493e9febd5bb2e [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
Steve Funge984a532013-11-25 17:09:25 -08007"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07008
9The script supports deploying Chrome from these sources:
10
111. A local build output directory, such as chromium/src/out/[Debug|Release].
122. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
133. A Chrome tarball existing locally.
14
15The script copies the necessary contents of the source location (tarball or
16build directory) and rsyncs the contents of the staging directory onto your
17device's rootfs.
18"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070019
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070024import logging
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070027import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
30import tarfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070031import time
Steve Funge984a532013-11-25 17:09:25 -080032import zipfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070033
Ryan Cuia56a71e2012-10-18 18:40:35 -070034
David James629febb2012-11-25 13:07:34 -080035from chromite.buildbot import constants
David James88e6f032013-03-02 08:13:20 -080036from chromite.buildbot import cbuildbot_results as results_lib
Ryan Cui686ec052013-02-12 16:39:41 -080037from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070038from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070040from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080041from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080043from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070044from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070045from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080046from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070047from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070048
49
Ryan Cuia56a71e2012-10-18 18:40:35 -070050_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
51
Ryan Cui3045c5d2012-07-13 18:00:33 -070052KERNEL_A_PARTITION = 2
53KERNEL_B_PARTITION = 4
54
55KILL_PROC_MAX_WAIT = 10
56POST_KILL_WAIT = 2
57
Ryan Cuie535b172012-10-19 18:25:03 -070058MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080059LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070060
Steve Funge984a532013-11-25 17:09:25 -080061MOUNT_RW_COMMAND_ANDROID = 'mount -o remount,rw /system'
62
63_ANDROID_DIR = '/system/chrome'
64_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
65
David James2cb34002013-03-01 18:42:40 -080066_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070067_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080068
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
73DF_COMMAND_ANDROID = 'df %s'
74
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
David James88e6f032013-03-02 08:13:20 -080080class DeployFailure(results_lib.StepFailure):
81 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700102 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800103 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Steve Funge984a532013-11-25 17:09:25 -0800105 # Used to track whether deploying content_shell or chrome to a device.
106 self.content_shell = False
107 self.copy_paths = chrome_util.GetCopyPaths(False)
108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800111 result = self.host.RemoteSh((DF_COMMAND if not self.content_shell
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800112 else DF_COMMAND_ANDROID) % remote_dir,
113 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700114 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800115 value = line.split()[3]
116 multipliers = {
117 'G': 1024 * 1024 * 1024,
118 'M': 1024 * 1024,
119 'K': 1024,
120 }
121 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122
123 def _GetRemoteDirSize(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800124 if self.content_shell:
125 # Content Shell devices currently do not contain the du binary.
126 logging.warning('Remote host does not contain du; cannot get remote '
127 'directory size to properly calculate available free '
128 'space.')
129 return 0
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800130 result = self.host.RemoteSh('du -ks %s' % remote_dir, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700131 return int(result.output.split()[0])
132
133 def _GetStagingDirSize(self):
134 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800135 redirect_stdout=True,
136 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700137 return int(result.output.split()[0])
138
Ryan Cui3045c5d2012-07-13 18:00:33 -0700139 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800140 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800141 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700142 return result.returncode == 0
143
144 def _DisableRootfsVerification(self):
145 if not self.options.force:
146 logging.error('Detected that the device has rootfs verification enabled.')
147 logging.info('This script can automatically remove the rootfs '
148 'verification, which requires that it reboot the device.')
149 logging.info('Make sure the device is in developer mode!')
150 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700151 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800152 # Since we stopped Chrome earlier, it's good form to start it up again.
153 if self.options.startui:
154 logging.info('Starting Chrome...')
155 self.host.RemoteSh('start ui')
156 raise DeployFailure('Need rootfs verification to be disabled. '
157 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 logging.info('Removing rootfs verification from %s', self.options.to)
160 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
161 # Use --force to bypass the checks.
162 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
163 '--remove_rootfs_verification --force')
164 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
165 self.host.RemoteSh(cmd % partition, error_code_ok=True)
166
167 # A reboot in developer mode takes a while (and has delays), so the user
168 # will have time to read and act on the USB boot instructions below.
169 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
170 self.host.RemoteReboot()
171
David James88e6f032013-03-02 08:13:20 -0800172 # Now that the machine has been rebooted, we need to kill Chrome again.
173 self._KillProcsIfNeeded()
174
175 # Make sure the rootfs is writable now.
176 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700177
178 def _CheckUiJobStarted(self):
179 # status output is in the format:
180 # <job_name> <status> ['process' <pid>].
181 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800182 try:
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800183 result = self.host.RemoteSh('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800184 except cros_build_lib.RunCommandError as e:
185 if 'Unknown job' in e.result.error:
186 return False
187 else:
188 raise e
189
Ryan Cui3045c5d2012-07-13 18:00:33 -0700190 return result.output.split()[1].split('/')[0] == 'start'
191
192 def _KillProcsIfNeeded(self):
Steve Funge984a532013-11-25 17:09:25 -0800193 if self.content_shell:
194 logging.info('Shutting down content_shell...')
195 self.host.RemoteSh('stop content_shell')
196 return
197
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800199 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700200 self.host.RemoteSh('stop ui')
201
202 # Developers sometimes run session_manager manually, in which case we'll
203 # need to help shut the chrome processes down.
204 try:
David James3432acd2013-11-27 10:02:18 -0800205 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206 while self._ChromeFileInUse():
207 logging.warning('The chrome binary on the device is in use.')
208 logging.warning('Killing chrome and session_manager processes...\n')
209
210 self.host.RemoteSh("pkill 'chrome|session_manager'",
211 error_code_ok=True)
212 # Wait for processes to actually terminate
213 time.sleep(POST_KILL_WAIT)
214 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800215 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800216 msg = ('Could not kill processes after %s seconds. Please exit any '
217 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
218 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700219
David James88e6f032013-03-02 08:13:20 -0800220 def _MountRootfsAsWritable(self, error_code_ok=True):
221 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
David James88e6f032013-03-02 08:13:20 -0800223 If the command fails, and error_code_ok is True, then this function sets
224 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700225
Mike Frysinger02e1e072013-11-10 22:11:34 -0500226 Args:
David James88e6f032013-03-02 08:13:20 -0800227 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
228 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800229 # TODO: Should migrate to use the remount functions in remote_access.
Steve Funge984a532013-11-25 17:09:25 -0800230 result = self.host.RemoteSh(MOUNT_RW_COMMAND if not self.content_shell
231 else MOUNT_RW_COMMAND_ANDROID,
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800232 error_code_ok=error_code_ok,
233 capture_output=True)
David James88e6f032013-03-02 08:13:20 -0800234 if result.returncode:
235 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700236
Ryan Cui7193a7e2013-04-26 14:15:19 -0700237 def _GetDeviceInfo(self):
238 steps = [
239 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
240 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
241 ]
242 return_values = parallel.RunParallelSteps(steps, return_values=True)
243 return DeviceInfo(*return_values)
244
245 def _CheckDeviceFreeSpace(self, device_info):
246 """See if target device has enough space for Chrome.
247
Mike Frysinger02e1e072013-11-10 22:11:34 -0500248 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700249 device_info: A DeviceInfo named tuple.
250 """
251 effective_free = device_info.target_dir_size + device_info.target_fs_free
252 staging_size = self._GetStagingDirSize()
Steve Funge984a532013-11-25 17:09:25 -0800253 # For content shell deployments, which do not contain the du binary,
254 # do not raise DeployFailure since can't get exact free space available
255 # for staging files.
Ryan Cui7193a7e2013-04-26 14:15:19 -0700256 if effective_free < staging_size:
Steve Funge984a532013-11-25 17:09:25 -0800257 if self.content_shell:
258 logging.warning('Not enough free space on the device. If overwriting '
259 'files, deployment may still succeed. Required: %s '
260 'MiB, actual: %s MiB.', staging_size / 1024,
261 effective_free / 1024)
262 else:
263 raise DeployFailure(
264 'Not enough free space on the device. Required: %s MiB, '
265 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700266 if device_info.target_fs_free < (100 * 1024):
267 logging.warning('The device has less than 100MB free. deploy_chrome may '
268 'hang during the transfer.')
269
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800271 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800273 dest_path = _CHROME_DIR
274 if self.content_shell:
275 try:
276 self.host.Scp('%s/*' % os.path.abspath(self.staging_dir),
277 '%s/' % self.options.target_dir,
278 recursive=True,
279 debug_level=logging.INFO,
280 verbose=self.options.verbose)
281 except cros_build_lib.RunCommandError as ex:
282 if ex.result.returncode != 1:
283 logging.error('Scp failure [%s]', ex.result.returncode)
284 raise DeployFailure(ex)
285 else:
286 # TODO(stevefung): Update Dropbear SSHD on device.
287 # http://crbug.com/329656
288 logging.info('Potential conflict with DropBear SSHD return status')
289
290 dest_path = _ANDROID_DIR
291 else:
292 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
293 self.options.target_dir,
294 inplace=True, debug_level=logging.INFO,
295 verbose=self.options.verbose)
296
297 for p in self.copy_paths:
298 if p.owner:
299 self.host.RemoteSh('chown %s %s/%s' % (p.owner, dest_path,
300 p.src if not p.dest else p.dest))
301 if p.mode:
302 # Set mode if necessary.
303 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
304 p.src if not p.dest else p.dest))
305
306
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800307 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800308 logging.info('Starting UI...')
309 if self.content_shell:
310 self.host.RemoteSh('start content_shell')
311 else:
312 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700313
David James88e6f032013-03-02 08:13:20 -0800314 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700315 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800316 logging.info('Testing connection to the device...')
Steve Funge984a532013-11-25 17:09:25 -0800317 if self.content_shell:
318 # true command over ssh returns error code 255, so as workaround
319 # use `sleep 0` as no-op.
320 self.host.RemoteSh('sleep 0')
321 else:
322 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800323 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700324 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800325 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700326
Steve Funge984a532013-11-25 17:09:25 -0800327 def _CheckDeployType(self):
328 if self.options.build_dir and os.path.exists(
329 os.path.join(self.options.build_dir, 'system.unand')):
330 # Content shell deployment.
331 self.content_shell = True
332 self.options.build_dir = os.path.join(self.options.build_dir,
333 'system.unand/chrome/')
334 self.options.dostrip = False
335 self.options.target_dir = _ANDROID_DIR
336 self.copy_paths = chrome_util.GetCopyPaths(True)
337 elif self.options.local_pkg_path or self.options.gs_path:
338 # Package deployment.
339 pkg_path = self.options.local_pkg_path
340 if self.options.gs_path:
341 pkg_path = _FetchChromePackage(self.options.cache_dir, self.tempdir,
342 self.options.gs_path)
343
344 assert pkg_path
345 logging.info('Checking %s for content_shell...', pkg_path)
346 if pkg_path[-4:] == '.zip':
347 zip_pkg = zipfile.ZipFile(pkg_path)
348 if any('eureka_shell' in name for name in zip_pkg.namelist()):
349 self.content_shell = True
350 zip_pkg.close()
351 else:
352 tar = tarfile.open(pkg_path)
353 if any('eureka_shell' in member.name for member in tar.getmembers()):
354 self.content_shell = True
355 tar.close()
356
357 if self.content_shell:
358 self.options.dostrip = False
359 self.options.target_dir = _ANDROID_DIR
360 self.copy_paths = chrome_util.GetCopyPaths(True)
361 self.chrome_dir = _ANDROID_DIR
362
David James88e6f032013-03-02 08:13:20 -0800363 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800364 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
365 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800366
Thiago Goncales12793312013-05-23 11:26:17 -0700367 def _MountTarget(self):
368 logging.info('Mounting Chrome...')
369
370 # Create directory if does not exist
371 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
372 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
373 self.options.mount_dir))
374 # Chrome needs partition to have exec and suid flags set
375 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
376
David James88e6f032013-03-02 08:13:20 -0800377 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800378 self._CheckDeployType()
379
David James88e6f032013-03-02 08:13:20 -0800380 # If requested, just do the staging step.
381 if self.options.staging_only:
382 self._PrepareStagingDir()
383 return 0
384
385 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800386 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800387 steps = [self._GetDeviceInfo, self._CheckConnection,
388 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
389 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700390 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
391 return_values=True)
392 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800393
394 # If we failed to mark the rootfs as writable, try disabling rootfs
395 # verification.
396 if self._rootfs_is_still_readonly.is_set():
397 self._DisableRootfsVerification()
398
Thiago Goncales12793312013-05-23 11:26:17 -0700399 if self.options.mount_dir is not None:
400 self._MountTarget()
401
David James88e6f032013-03-02 08:13:20 -0800402 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700403 self._Deploy()
404
405
Ryan Cuia56a71e2012-10-18 18:40:35 -0700406def ValidateGypDefines(_option, _opt, value):
407 """Convert GYP_DEFINES-formatted string to dictionary."""
408 return chrome_util.ProcessGypDefines(value)
409
410
411class CustomOption(commandline.Option):
412 """Subclass Option class to implement path evaluation."""
413 TYPES = commandline.Option.TYPES + ('gyp_defines',)
414 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
415 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
416
417
Ryan Cuie535b172012-10-19 18:25:03 -0700418def _CreateParser():
419 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800420 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
421 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700422
Ryan Cuia56a71e2012-10-18 18:40:35 -0700423 # TODO(rcui): Have this use the UI-V2 format of having source and target
424 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700425 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800426 help='Skip all prompts (i.e., for disabling of rootfs '
427 'verification). This may result in the target '
428 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800429 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
430 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800431 help="The board the Chrome build is targeted for. When in "
432 "a 'cros chrome-sdk' shell, defaults to the SDK "
433 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700434 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800435 help='The directory with Chrome build artifacts to deploy '
436 'from. Typically of format <chrome_root>/out/Debug. '
437 'When this option is used, the GYP_DEFINES '
438 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800439 parser.add_option('--target-dir', type='path',
440 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700441 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700442 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800443 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800444 parser.add_option('--nostartui', action='store_false', dest='startui',
445 default=True,
446 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800447 parser.add_option('--nostrip', action='store_false', dest='dostrip',
448 default=True,
449 help="Don't strip binaries during deployment. Warning: "
450 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700451 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800452 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700453 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800454 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700455 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800456 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700457 parser.add_option('--mount-dir', type='path', default=None,
458 help='Deploy Chrome in target directory and bind it'
459 'to directory specified by this flag.')
460 parser.add_option('--mount', action='store_true', default=False,
461 help='Deploy Chrome to default target directory and bind it'
462 'to default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700463
464 group = optparse.OptionGroup(parser, 'Advanced Options')
465 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800466 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800467 group.add_option('--sloppy', action='store_true', default=False,
468 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700469 group.add_option('--staging-flags', default=None, type='gyp_defines',
470 help='Extra flags to control staging. Valid flags are - %s'
471 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800472 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800473 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800474 'variable and --staging-flags, if set. Enforce that '
475 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700476 group.add_option('--strip-flags', default=None,
477 help="Flags to call the 'strip' binutil tool with. "
478 "Overrides the default arguments.")
Ryan Cuief91e702013-02-04 12:06:36 -0800479
Ryan Cuia56a71e2012-10-18 18:40:35 -0700480 parser.add_option_group(group)
481
Ryan Cuif890a3e2013-03-07 18:57:06 -0800482 # GYP_DEFINES that Chrome was built with. Influences which files are staged
483 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
484 # enviroment variable.
485 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
486 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700487 # Path of an empty directory to stage chrome artifacts to. Defaults to a
488 # temporary directory that is removed when the script finishes. If the path
489 # is specified, then it will not be removed.
490 parser.add_option('--staging-dir', type='path', default=None,
491 help=optparse.SUPPRESS_HELP)
492 # Only prepare the staging directory, and skip deploying to the device.
493 parser.add_option('--staging-only', action='store_true', default=False,
494 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700495 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
496 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
497 # fetching the SDK toolchain.
498 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700499 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700500
Ryan Cuie535b172012-10-19 18:25:03 -0700501
502def _ParseCommandLine(argv):
503 """Parse args, and run environment-independent checks."""
504 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700505 (options, args) = parser.parse_args(argv)
506
Ryan Cuia56a71e2012-10-18 18:40:35 -0700507 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
508 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800509 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700510 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
511 parser.error('Cannot specify both --build_dir and '
512 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800513 if options.build_dir and not options.board:
514 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700515 if options.gs_path and options.local_pkg_path:
516 parser.error('Cannot specify both --gs-path and --local-pkg-path')
517 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700518 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800519 if (options.strict or options.staging_flags) and not options.build_dir:
520 parser.error('--strict and --staging-flags require --build-dir to be '
521 'set.')
522 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800523 parser.error('--staging-flags requires --strict to be set.')
524 if options.sloppy and options.strict:
525 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700526
527 if options.mount or options.mount_dir:
528 if not options.target_dir:
529 options.target_dir = _CHROME_DIR_MOUNT
530 else:
531 if not options.target_dir:
532 options.target_dir = _CHROME_DIR
533
534 if options.mount and not options.mount_dir:
535 options.mount_dir = _CHROME_DIR
536
Ryan Cui3045c5d2012-07-13 18:00:33 -0700537 return options, args
538
539
Ryan Cuie535b172012-10-19 18:25:03 -0700540def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800541 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700542
543 Args:
Steve Funge984a532013-11-25 17:09:25 -0800544 options: The options object returned by optparse.
545 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700546 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700547 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
548 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
549
Ryan Cuib623e7b2013-03-14 12:54:11 -0700550 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700551 gyp_env = os.getenv('GYP_DEFINES', None)
552 if gyp_env is not None:
553 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700554 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700555 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700556
557 if options.strict and not options.gyp_defines:
558 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800559 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700560
Ryan Cui3c183c22013-04-29 18:04:11 -0700561 if options.build_dir:
562 chrome_path = os.path.join(options.build_dir, 'chrome')
563 if os.path.isfile(chrome_path):
564 deps = lddtree.ParseELF(chrome_path)
565 if 'libbase.so' in deps['libs']:
566 cros_build_lib.Warning(
567 'Detected a component build of Chrome. component build is '
568 'not working properly for Chrome OS. See crbug.com/196317. '
569 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700570
Ryan Cuia56a71e2012-10-18 18:40:35 -0700571
Ryan Cui504db722013-01-22 11:48:01 -0800572def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700573 """Get the chrome prebuilt tarball from GS.
574
Mike Frysinger02e1e072013-11-10 22:11:34 -0500575 Returns:
576 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700577 """
David James9374aac2013-10-08 16:00:17 -0700578 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500579 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800580 files = [found for found in files if
581 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
582 if not files:
583 raise Exception('No chrome package found at %s' % gs_path)
584 elif len(files) > 1:
585 # - Users should provide us with a direct link to either a stripped or
586 # unstripped chrome package.
587 # - In the case of being provided with an archive directory, where both
588 # stripped and unstripped chrome available, use the stripped chrome
589 # package.
590 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
591 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
592 files = [f for f in files if not 'unstripped' in f]
593 assert len(files) == 1
594 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800595
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800596 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800597 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800598 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
599 chrome_path = os.path.join(tempdir, filename)
600 assert os.path.exists(chrome_path)
601 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700602
603
Ryan Cuif890a3e2013-03-07 18:57:06 -0800604@contextlib.contextmanager
605def _StripBinContext(options):
606 if not options.dostrip:
607 yield None
608 elif options.strip_bin:
609 yield options.strip_bin
610 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800611 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800612 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
613 with sdk.Prepare(components=components) as ctx:
614 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
615 constants.CHROME_ENV_FILE)
616 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
617 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
618 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800619 yield strip_bin
620
621
Steve Funge984a532013-11-25 17:09:25 -0800622def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
623 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800624 """Place the necessary files in the staging directory.
625
626 The staging directory is the directory used to rsync the build artifacts over
627 to the device. Only the necessary Chrome build artifacts are put into the
628 staging directory.
629 """
Ryan Cui5866be02013-03-18 14:12:00 -0700630 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400631 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800632 if options.build_dir:
633 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700634 strip_flags = (None if options.strip_flags is None else
635 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800636 chrome_util.StageChromeFromBuildDir(
637 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800638 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700639 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800640 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700641 else:
642 pkg_path = options.local_pkg_path
643 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800644 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
645 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700646
647 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800648 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700649 # Extract only the ./opt/google/chrome contents, directly into the staging
650 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800651 if pkg_path[-4:] == '.zip':
652 cros_build_lib.DebugRunCommand(
653 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
654 staging_dir])
655 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
656 shutil.move(filename, staging_dir)
657 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
658 else:
659 cros_build_lib.DebugRunCommand(
660 ['tar', '--strip-components', '4', '--extract',
661 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
662 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700663
Ryan Cui71aa8de2013-04-19 16:12:55 -0700664
Ryan Cui3045c5d2012-07-13 18:00:33 -0700665def main(argv):
666 options, args = _ParseCommandLine(argv)
667 _PostParseCheck(options, args)
668
669 # Set cros_build_lib debug level to hide RunCommand spew.
670 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700671 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700672 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800673 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700674
Ryan Cui71aa8de2013-04-19 16:12:55 -0700675 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700676 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
677 if cmd_stats:
678 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700679
Ryan Cui71aa8de2013-04-19 16:12:55 -0700680 with osutils.TempDir(set_global=True) as tempdir:
681 staging_dir = options.staging_dir
682 if not staging_dir:
683 staging_dir = os.path.join(tempdir, 'chrome')
684
685 deploy = DeployChrome(options, tempdir, staging_dir)
686 try:
687 deploy.Perform()
688 except results_lib.StepFailure as ex:
689 raise SystemExit(str(ex).strip())