blob: 64a7541f2152cff409a83532aaefb56aa0b30046 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001#!/usr/bin/python
2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
Steve Funge984a532013-11-25 17:09:25 -08007"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07008
9The script supports deploying Chrome from these sources:
10
111. A local build output directory, such as chromium/src/out/[Debug|Release].
122. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
133. A Chrome tarball existing locally.
14
15The script copies the necessary contents of the source location (tarball or
16build directory) and rsyncs the contents of the staging directory onto your
17device's rootfs.
18"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070019
Ryan Cui7193a7e2013-04-26 14:15:19 -070020import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080021import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070022import functools
Steve Funge984a532013-11-25 17:09:25 -080023import glob
Ryan Cui3045c5d2012-07-13 18:00:33 -070024import logging
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cuia56a71e2012-10-18 18:40:35 -070027import optparse
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070028import shlex
Steve Funge984a532013-11-25 17:09:25 -080029import shutil
30import tarfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070031import time
Steve Funge984a532013-11-25 17:09:25 -080032import zipfile
Ryan Cui3045c5d2012-07-13 18:00:33 -070033
Ryan Cuia56a71e2012-10-18 18:40:35 -070034
David James629febb2012-11-25 13:07:34 -080035from chromite.buildbot import constants
David James88e6f032013-03-02 08:13:20 -080036from chromite.buildbot import cbuildbot_results as results_lib
Ryan Cui686ec052013-02-12 16:39:41 -080037from chromite.cros.commands import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070038from chromite.lib import chrome_util
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import cros_build_lib
Ryan Cuie535b172012-10-19 18:25:03 -070040from chromite.lib import commandline
Ryan Cui777ff422012-12-07 13:12:54 -080041from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070042from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080043from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070044from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070045from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080046from chromite.lib import timeout_util
Ryan Cui3c183c22013-04-29 18:04:11 -070047from chromite.scripts import lddtree
Ryan Cui3045c5d2012-07-13 18:00:33 -070048
49
Ryan Cuia56a71e2012-10-18 18:40:35 -070050_USAGE = "deploy_chrome [--]\n\n %s" % __doc__
51
Ryan Cui3045c5d2012-07-13 18:00:33 -070052KERNEL_A_PARTITION = 2
53KERNEL_B_PARTITION = 4
54
55KILL_PROC_MAX_WAIT = 10
56POST_KILL_WAIT = 2
57
Ryan Cuie535b172012-10-19 18:25:03 -070058MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080059LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070060
Steve Funge984a532013-11-25 17:09:25 -080061MOUNT_RW_COMMAND_ANDROID = 'mount -o remount,rw /system'
62
63_ANDROID_DIR = '/system/chrome'
64_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
65
David James2cb34002013-03-01 18:42:40 -080066_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070067_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080068
Thiago Goncales12793312013-05-23 11:26:17 -070069_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
70_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070071
Steve Funge984a532013-11-25 17:09:25 -080072DF_COMMAND = 'df -k %s'
73DF_COMMAND_ANDROID = 'df %s'
74
Ryan Cui3045c5d2012-07-13 18:00:33 -070075def _UrlBaseName(url):
76 """Return the last component of the URL."""
77 return url.rstrip('/').rpartition('/')[-1]
78
79
David James88e6f032013-03-02 08:13:20 -080080class DeployFailure(results_lib.StepFailure):
81 """Raised whenever the deploy fails."""
82
83
Ryan Cui7193a7e2013-04-26 14:15:19 -070084DeviceInfo = collections.namedtuple(
85 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
86
87
Ryan Cui3045c5d2012-07-13 18:00:33 -070088class DeployChrome(object):
89 """Wraps the core deployment functionality."""
Ryan Cuia56a71e2012-10-18 18:40:35 -070090 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070091 """Initialize the class.
92
Mike Frysinger02e1e072013-11-10 22:11:34 -050093 Args:
Ryan Cuie535b172012-10-19 18:25:03 -070094 options: Optparse result structure.
95 tempdir: Scratch space for the class. Caller has responsibility to clean
96 it up.
Steve Funge984a532013-11-25 17:09:25 -080097 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070098 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070099 self.tempdir = tempdir
100 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -0700101 self.staging_dir = staging_dir
Ryan Cuiafd6c5c2012-07-30 17:48:22 -0700102 self.host = remote.RemoteAccess(options.to, tempdir, port=options.port)
David James88e6f032013-03-02 08:13:20 -0800103 self._rootfs_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700104
Steve Funge984a532013-11-25 17:09:25 -0800105 # Used to track whether deploying content_shell or chrome to a device.
106 self.content_shell = False
107 self.copy_paths = chrome_util.GetCopyPaths(False)
108 self.chrome_dir = _CHROME_DIR
109
Ryan Cui7193a7e2013-04-26 14:15:19 -0700110 def _GetRemoteMountFree(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800111 result = self.host.RemoteSh((DF_COMMAND if not self.content_shell
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800112 else DF_COMMAND_ANDROID) % remote_dir,
113 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700114 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800115 value = line.split()[3]
116 multipliers = {
117 'G': 1024 * 1024 * 1024,
118 'M': 1024 * 1024,
119 'K': 1024,
120 }
121 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700122
123 def _GetRemoteDirSize(self, remote_dir):
Steve Funge984a532013-11-25 17:09:25 -0800124 if self.content_shell:
125 # Content Shell devices currently do not contain the du binary.
126 logging.warning('Remote host does not contain du; cannot get remote '
127 'directory size to properly calculate available free '
128 'space.')
129 return 0
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800130 result = self.host.RemoteSh('du -ks %s' % remote_dir, capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700131 return int(result.output.split()[0])
132
133 def _GetStagingDirSize(self):
134 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800135 redirect_stdout=True,
136 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700137 return int(result.output.split()[0])
138
Ryan Cui3045c5d2012-07-13 18:00:33 -0700139 def _ChromeFileInUse(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800140 result = self.host.RemoteSh(LSOF_COMMAND % (self.options.target_dir,),
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800141 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700142 return result.returncode == 0
143
144 def _DisableRootfsVerification(self):
145 if not self.options.force:
146 logging.error('Detected that the device has rootfs verification enabled.')
147 logging.info('This script can automatically remove the rootfs '
148 'verification, which requires that it reboot the device.')
149 logging.info('Make sure the device is in developer mode!')
150 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700151 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800152 # Since we stopped Chrome earlier, it's good form to start it up again.
153 if self.options.startui:
154 logging.info('Starting Chrome...')
155 self.host.RemoteSh('start ui')
156 raise DeployFailure('Need rootfs verification to be disabled. '
157 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 logging.info('Removing rootfs verification from %s', self.options.to)
160 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
161 # Use --force to bypass the checks.
162 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
163 '--remove_rootfs_verification --force')
164 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
165 self.host.RemoteSh(cmd % partition, error_code_ok=True)
166
167 # A reboot in developer mode takes a while (and has delays), so the user
168 # will have time to read and act on the USB boot instructions below.
169 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
170 self.host.RemoteReboot()
171
David James88e6f032013-03-02 08:13:20 -0800172 # Now that the machine has been rebooted, we need to kill Chrome again.
173 self._KillProcsIfNeeded()
174
175 # Make sure the rootfs is writable now.
176 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700177
178 def _CheckUiJobStarted(self):
179 # status output is in the format:
180 # <job_name> <status> ['process' <pid>].
181 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800182 try:
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800183 result = self.host.RemoteSh('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800184 except cros_build_lib.RunCommandError as e:
185 if 'Unknown job' in e.result.error:
186 return False
187 else:
188 raise e
189
Ryan Cui3045c5d2012-07-13 18:00:33 -0700190 return result.output.split()[1].split('/')[0] == 'start'
191
192 def _KillProcsIfNeeded(self):
Steve Funge984a532013-11-25 17:09:25 -0800193 if self.content_shell:
194 logging.info('Shutting down content_shell...')
195 self.host.RemoteSh('stop content_shell')
196 return
197
Ryan Cui3045c5d2012-07-13 18:00:33 -0700198 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800199 logging.info('Shutting down Chrome...')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700200 self.host.RemoteSh('stop ui')
201
202 # Developers sometimes run session_manager manually, in which case we'll
203 # need to help shut the chrome processes down.
204 try:
David James3432acd2013-11-27 10:02:18 -0800205 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206 while self._ChromeFileInUse():
207 logging.warning('The chrome binary on the device is in use.')
208 logging.warning('Killing chrome and session_manager processes...\n')
209
210 self.host.RemoteSh("pkill 'chrome|session_manager'",
211 error_code_ok=True)
212 # Wait for processes to actually terminate
213 time.sleep(POST_KILL_WAIT)
214 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800215 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800216 msg = ('Could not kill processes after %s seconds. Please exit any '
217 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
218 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700219
David James88e6f032013-03-02 08:13:20 -0800220 def _MountRootfsAsWritable(self, error_code_ok=True):
221 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700222
David James88e6f032013-03-02 08:13:20 -0800223 If the command fails, and error_code_ok is True, then this function sets
224 self._rootfs_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700225
Mike Frysinger02e1e072013-11-10 22:11:34 -0500226 Args:
David James88e6f032013-03-02 08:13:20 -0800227 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
228 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800229 # TODO: Should migrate to use the remount functions in remote_access.
Steve Funge984a532013-11-25 17:09:25 -0800230 result = self.host.RemoteSh(MOUNT_RW_COMMAND if not self.content_shell
231 else MOUNT_RW_COMMAND_ANDROID,
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800232 error_code_ok=error_code_ok,
233 capture_output=True)
David James88e6f032013-03-02 08:13:20 -0800234 if result.returncode:
235 self._rootfs_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700236
Ryan Cui7193a7e2013-04-26 14:15:19 -0700237 def _GetDeviceInfo(self):
238 steps = [
239 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
240 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
241 ]
242 return_values = parallel.RunParallelSteps(steps, return_values=True)
243 return DeviceInfo(*return_values)
244
245 def _CheckDeviceFreeSpace(self, device_info):
246 """See if target device has enough space for Chrome.
247
Mike Frysinger02e1e072013-11-10 22:11:34 -0500248 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700249 device_info: A DeviceInfo named tuple.
250 """
251 effective_free = device_info.target_dir_size + device_info.target_fs_free
252 staging_size = self._GetStagingDirSize()
Steve Funge984a532013-11-25 17:09:25 -0800253 # For content shell deployments, which do not contain the du binary,
254 # do not raise DeployFailure since can't get exact free space available
255 # for staging files.
Ryan Cui7193a7e2013-04-26 14:15:19 -0700256 if effective_free < staging_size:
Steve Funge984a532013-11-25 17:09:25 -0800257 if self.content_shell:
258 logging.warning('Not enough free space on the device. If overwriting '
259 'files, deployment may still succeed. Required: %s '
260 'MiB, actual: %s MiB.', staging_size / 1024,
261 effective_free / 1024)
262 else:
263 raise DeployFailure(
264 'Not enough free space on the device. Required: %s MiB, '
265 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700266 if device_info.target_fs_free < (100 * 1024):
267 logging.warning('The device has less than 100MB free. deploy_chrome may '
268 'hang during the transfer.')
269
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800271 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800273 dest_path = _CHROME_DIR
274 if self.content_shell:
275 try:
276 self.host.Scp('%s/*' % os.path.abspath(self.staging_dir),
277 '%s/' % self.options.target_dir,
Steve Funge984a532013-11-25 17:09:25 -0800278 debug_level=logging.INFO,
279 verbose=self.options.verbose)
280 except cros_build_lib.RunCommandError as ex:
281 if ex.result.returncode != 1:
282 logging.error('Scp failure [%s]', ex.result.returncode)
283 raise DeployFailure(ex)
284 else:
285 # TODO(stevefung): Update Dropbear SSHD on device.
286 # http://crbug.com/329656
287 logging.info('Potential conflict with DropBear SSHD return status')
288
289 dest_path = _ANDROID_DIR
290 else:
291 self.host.Rsync('%s/' % os.path.abspath(self.staging_dir),
292 self.options.target_dir,
293 inplace=True, debug_level=logging.INFO,
294 verbose=self.options.verbose)
295
296 for p in self.copy_paths:
297 if p.owner:
298 self.host.RemoteSh('chown %s %s/%s' % (p.owner, dest_path,
299 p.src if not p.dest else p.dest))
300 if p.mode:
301 # Set mode if necessary.
302 self.host.RemoteSh('chmod %o %s/%s' % (p.mode, dest_path,
303 p.src if not p.dest else p.dest))
304
305
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800306 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800307 logging.info('Starting UI...')
308 if self.content_shell:
309 self.host.RemoteSh('start content_shell')
310 else:
311 self.host.RemoteSh('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700312
David James88e6f032013-03-02 08:13:20 -0800313 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700314 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800315 logging.info('Testing connection to the device...')
Steve Funge984a532013-11-25 17:09:25 -0800316 if self.content_shell:
317 # true command over ssh returns error code 255, so as workaround
318 # use `sleep 0` as no-op.
319 self.host.RemoteSh('sleep 0')
320 else:
321 self.host.RemoteSh('true')
David James88e6f032013-03-02 08:13:20 -0800322 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700323 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800324 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700325
Steve Funge984a532013-11-25 17:09:25 -0800326 def _CheckDeployType(self):
327 if self.options.build_dir and os.path.exists(
328 os.path.join(self.options.build_dir, 'system.unand')):
329 # Content shell deployment.
330 self.content_shell = True
331 self.options.build_dir = os.path.join(self.options.build_dir,
332 'system.unand/chrome/')
333 self.options.dostrip = False
334 self.options.target_dir = _ANDROID_DIR
335 self.copy_paths = chrome_util.GetCopyPaths(True)
336 elif self.options.local_pkg_path or self.options.gs_path:
337 # Package deployment.
338 pkg_path = self.options.local_pkg_path
339 if self.options.gs_path:
340 pkg_path = _FetchChromePackage(self.options.cache_dir, self.tempdir,
341 self.options.gs_path)
342
343 assert pkg_path
344 logging.info('Checking %s for content_shell...', pkg_path)
345 if pkg_path[-4:] == '.zip':
346 zip_pkg = zipfile.ZipFile(pkg_path)
347 if any('eureka_shell' in name for name in zip_pkg.namelist()):
348 self.content_shell = True
349 zip_pkg.close()
350 else:
351 tar = tarfile.open(pkg_path)
352 if any('eureka_shell' in member.name for member in tar.getmembers()):
353 self.content_shell = True
354 tar.close()
355
356 if self.content_shell:
357 self.options.dostrip = False
358 self.options.target_dir = _ANDROID_DIR
359 self.copy_paths = chrome_util.GetCopyPaths(True)
360 self.chrome_dir = _ANDROID_DIR
361
David James88e6f032013-03-02 08:13:20 -0800362 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800363 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
364 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800365
Thiago Goncales12793312013-05-23 11:26:17 -0700366 def _MountTarget(self):
367 logging.info('Mounting Chrome...')
368
369 # Create directory if does not exist
370 self.host.RemoteSh('mkdir -p --mode 0775 %s' % (self.options.mount_dir,))
371 self.host.RemoteSh(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
372 self.options.mount_dir))
373 # Chrome needs partition to have exec and suid flags set
374 self.host.RemoteSh(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
375
David James88e6f032013-03-02 08:13:20 -0800376 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800377 self._CheckDeployType()
378
David James88e6f032013-03-02 08:13:20 -0800379 # If requested, just do the staging step.
380 if self.options.staging_only:
381 self._PrepareStagingDir()
382 return 0
383
384 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800385 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800386 steps = [self._GetDeviceInfo, self._CheckConnection,
387 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
388 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700389 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
390 return_values=True)
391 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800392
393 # If we failed to mark the rootfs as writable, try disabling rootfs
394 # verification.
395 if self._rootfs_is_still_readonly.is_set():
396 self._DisableRootfsVerification()
397
Thiago Goncales12793312013-05-23 11:26:17 -0700398 if self.options.mount_dir is not None:
399 self._MountTarget()
400
David James88e6f032013-03-02 08:13:20 -0800401 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700402 self._Deploy()
403
404
Ryan Cuia56a71e2012-10-18 18:40:35 -0700405def ValidateGypDefines(_option, _opt, value):
406 """Convert GYP_DEFINES-formatted string to dictionary."""
407 return chrome_util.ProcessGypDefines(value)
408
409
410class CustomOption(commandline.Option):
411 """Subclass Option class to implement path evaluation."""
412 TYPES = commandline.Option.TYPES + ('gyp_defines',)
413 TYPE_CHECKER = commandline.Option.TYPE_CHECKER.copy()
414 TYPE_CHECKER['gyp_defines'] = ValidateGypDefines
415
416
Ryan Cuie535b172012-10-19 18:25:03 -0700417def _CreateParser():
418 """Create our custom parser."""
Ryan Cui504db722013-01-22 11:48:01 -0800419 parser = commandline.OptionParser(usage=_USAGE, option_class=CustomOption,
420 caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700421
Ryan Cuia56a71e2012-10-18 18:40:35 -0700422 # TODO(rcui): Have this use the UI-V2 format of having source and target
423 # device be specified as positional arguments.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700424 parser.add_option('--force', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800425 help='Skip all prompts (i.e., for disabling of rootfs '
426 'verification). This may result in the target '
427 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800428 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
429 parser.add_option('--board', default=sdk_board_env,
Ryan Cui686ec052013-02-12 16:39:41 -0800430 help="The board the Chrome build is targeted for. When in "
431 "a 'cros chrome-sdk' shell, defaults to the SDK "
432 "board.")
Ryan Cuia56a71e2012-10-18 18:40:35 -0700433 parser.add_option('--build-dir', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800434 help='The directory with Chrome build artifacts to deploy '
435 'from. Typically of format <chrome_root>/out/Debug. '
436 'When this option is used, the GYP_DEFINES '
437 'environment variable must be set.')
Pawel Osciak577773a2013-03-05 10:52:12 -0800438 parser.add_option('--target-dir', type='path',
439 help='Target directory on device to deploy Chrome into.',
Thiago Goncales12793312013-05-23 11:26:17 -0700440 default=None)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700441 parser.add_option('-g', '--gs-path', type='gs_path',
Ryan Cui686ec052013-02-12 16:39:41 -0800442 help='GS path that contains the chrome to deploy.')
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800443 parser.add_option('--nostartui', action='store_false', dest='startui',
444 default=True,
445 help="Don't restart the ui daemon after deployment.")
Ryan Cuif890a3e2013-03-07 18:57:06 -0800446 parser.add_option('--nostrip', action='store_false', dest='dostrip',
447 default=True,
448 help="Don't strip binaries during deployment. Warning: "
449 "the resulting binaries will be very large!")
Ryan Cui3045c5d2012-07-13 18:00:33 -0700450 parser.add_option('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
Ryan Cui686ec052013-02-12 16:39:41 -0800451 help='Port of the target device to connect to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700452 parser.add_option('-t', '--to',
Ryan Cui686ec052013-02-12 16:39:41 -0800453 help='The IP address of the CrOS device to deploy to.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700454 parser.add_option('-v', '--verbose', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800455 help='Show more debug output.')
Thiago Goncales12793312013-05-23 11:26:17 -0700456 parser.add_option('--mount-dir', type='path', default=None,
457 help='Deploy Chrome in target directory and bind it'
458 'to directory specified by this flag.')
459 parser.add_option('--mount', action='store_true', default=False,
460 help='Deploy Chrome to default target directory and bind it'
461 'to default mount directory.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700462
463 group = optparse.OptionGroup(parser, 'Advanced Options')
464 group.add_option('-l', '--local-pkg-path', type='path',
Ryan Cui686ec052013-02-12 16:39:41 -0800465 help='Path to local chrome prebuilt package to deploy.')
David Jamesa6e08892013-03-01 13:34:11 -0800466 group.add_option('--sloppy', action='store_true', default=False,
467 help='Ignore when mandatory artifacts are missing.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700468 group.add_option('--staging-flags', default=None, type='gyp_defines',
469 help='Extra flags to control staging. Valid flags are - %s'
470 % ', '.join(chrome_util.STAGING_FLAGS))
Ryan Cuief91e702013-02-04 12:06:36 -0800471 group.add_option('--strict', action='store_true', default=False,
Ryan Cui686ec052013-02-12 16:39:41 -0800472 help='Stage artifacts based on the GYP_DEFINES environment '
David Jamesa6e08892013-03-01 13:34:11 -0800473 'variable and --staging-flags, if set. Enforce that '
474 'all optional artifacts are deployed.')
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700475 group.add_option('--strip-flags', default=None,
476 help="Flags to call the 'strip' binutil tool with. "
477 "Overrides the default arguments.")
Ryan Cuief91e702013-02-04 12:06:36 -0800478
Ryan Cuia56a71e2012-10-18 18:40:35 -0700479 parser.add_option_group(group)
480
Ryan Cuif890a3e2013-03-07 18:57:06 -0800481 # GYP_DEFINES that Chrome was built with. Influences which files are staged
482 # when --build-dir is set. Defaults to reading from the GYP_DEFINES
483 # enviroment variable.
484 parser.add_option('--gyp-defines', default=None, type='gyp_defines',
485 help=optparse.SUPPRESS_HELP)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700486 # Path of an empty directory to stage chrome artifacts to. Defaults to a
487 # temporary directory that is removed when the script finishes. If the path
488 # is specified, then it will not be removed.
489 parser.add_option('--staging-dir', type='path', default=None,
490 help=optparse.SUPPRESS_HELP)
491 # Only prepare the staging directory, and skip deploying to the device.
492 parser.add_option('--staging-only', action='store_true', default=False,
493 help=optparse.SUPPRESS_HELP)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700494 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
495 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
496 # fetching the SDK toolchain.
497 parser.add_option('--strip-bin', default=None, help=optparse.SUPPRESS_HELP)
Ryan Cuie535b172012-10-19 18:25:03 -0700498 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700499
Ryan Cuie535b172012-10-19 18:25:03 -0700500
501def _ParseCommandLine(argv):
502 """Parse args, and run environment-independent checks."""
503 parser = _CreateParser()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700504 (options, args) = parser.parse_args(argv)
505
Ryan Cuia56a71e2012-10-18 18:40:35 -0700506 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
507 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800508 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700509 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
510 parser.error('Cannot specify both --build_dir and '
511 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800512 if options.build_dir and not options.board:
513 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700514 if options.gs_path and options.local_pkg_path:
515 parser.error('Cannot specify both --gs-path and --local-pkg-path')
516 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700517 parser.error('Need to specify --to')
Ryan Cuief91e702013-02-04 12:06:36 -0800518 if (options.strict or options.staging_flags) and not options.build_dir:
519 parser.error('--strict and --staging-flags require --build-dir to be '
520 'set.')
521 if options.staging_flags and not options.strict:
David Jamesa6e08892013-03-01 13:34:11 -0800522 parser.error('--staging-flags requires --strict to be set.')
523 if options.sloppy and options.strict:
524 parser.error('Cannot specify both --strict and --sloppy.')
Thiago Goncales12793312013-05-23 11:26:17 -0700525
526 if options.mount or options.mount_dir:
527 if not options.target_dir:
528 options.target_dir = _CHROME_DIR_MOUNT
529 else:
530 if not options.target_dir:
531 options.target_dir = _CHROME_DIR
532
533 if options.mount and not options.mount_dir:
534 options.mount_dir = _CHROME_DIR
535
Ryan Cui3045c5d2012-07-13 18:00:33 -0700536 return options, args
537
538
Ryan Cuie535b172012-10-19 18:25:03 -0700539def _PostParseCheck(options, _args):
Steve Funge984a532013-11-25 17:09:25 -0800540 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700541
542 Args:
Steve Funge984a532013-11-25 17:09:25 -0800543 options: The options object returned by optparse.
544 _args: The args object returned by optparse.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700545 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700546 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
547 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
548
Ryan Cuib623e7b2013-03-14 12:54:11 -0700549 if not options.gyp_defines:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700550 gyp_env = os.getenv('GYP_DEFINES', None)
551 if gyp_env is not None:
552 options.gyp_defines = chrome_util.ProcessGypDefines(gyp_env)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700553 logging.debug('GYP_DEFINES taken from environment: %s',
Ryan Cuia56a71e2012-10-18 18:40:35 -0700554 options.gyp_defines)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700555
556 if options.strict and not options.gyp_defines:
557 cros_build_lib.Die('When --strict is set, the GYP_DEFINES environment '
Ryan Cui686ec052013-02-12 16:39:41 -0800558 'variable must be set.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700559
Ryan Cui3c183c22013-04-29 18:04:11 -0700560 if options.build_dir:
561 chrome_path = os.path.join(options.build_dir, 'chrome')
562 if os.path.isfile(chrome_path):
563 deps = lddtree.ParseELF(chrome_path)
564 if 'libbase.so' in deps['libs']:
565 cros_build_lib.Warning(
566 'Detected a component build of Chrome. component build is '
567 'not working properly for Chrome OS. See crbug.com/196317. '
568 'Use at your own risk!')
Ryan Cuib623e7b2013-03-14 12:54:11 -0700569
Ryan Cuia56a71e2012-10-18 18:40:35 -0700570
Ryan Cui504db722013-01-22 11:48:01 -0800571def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700572 """Get the chrome prebuilt tarball from GS.
573
Mike Frysinger02e1e072013-11-10 22:11:34 -0500574 Returns:
575 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700576 """
David James9374aac2013-10-08 16:00:17 -0700577 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500578 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800579 files = [found for found in files if
580 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
581 if not files:
582 raise Exception('No chrome package found at %s' % gs_path)
583 elif len(files) > 1:
584 # - Users should provide us with a direct link to either a stripped or
585 # unstripped chrome package.
586 # - In the case of being provided with an archive directory, where both
587 # stripped and unstripped chrome available, use the stripped chrome
588 # package.
589 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
590 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
591 files = [f for f in files if not 'unstripped' in f]
592 assert len(files) == 1
593 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800594
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800595 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800596 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800597 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
598 chrome_path = os.path.join(tempdir, filename)
599 assert os.path.exists(chrome_path)
600 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700601
602
Ryan Cuif890a3e2013-03-07 18:57:06 -0800603@contextlib.contextmanager
604def _StripBinContext(options):
605 if not options.dostrip:
606 yield None
607 elif options.strip_bin:
608 yield options.strip_bin
609 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800610 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800611 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
612 with sdk.Prepare(components=components) as ctx:
613 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
614 constants.CHROME_ENV_FILE)
615 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
616 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
617 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800618 yield strip_bin
619
620
Steve Funge984a532013-11-25 17:09:25 -0800621def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
622 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800623 """Place the necessary files in the staging directory.
624
625 The staging directory is the directory used to rsync the build artifacts over
626 to the device. Only the necessary Chrome build artifacts are put into the
627 staging directory.
628 """
Ryan Cui5866be02013-03-18 14:12:00 -0700629 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400630 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800631 if options.build_dir:
632 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700633 strip_flags = (None if options.strip_flags is None else
634 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800635 chrome_util.StageChromeFromBuildDir(
636 staging_dir, options.build_dir, strip_bin, strict=options.strict,
David Jamesa6e08892013-03-01 13:34:11 -0800637 sloppy=options.sloppy, gyp_defines=options.gyp_defines,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700638 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800639 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700640 else:
641 pkg_path = options.local_pkg_path
642 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800643 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
644 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700645
646 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800647 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700648 # Extract only the ./opt/google/chrome contents, directly into the staging
649 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800650 if pkg_path[-4:] == '.zip':
651 cros_build_lib.DebugRunCommand(
652 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
653 staging_dir])
654 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
655 shutil.move(filename, staging_dir)
656 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
657 else:
658 cros_build_lib.DebugRunCommand(
659 ['tar', '--strip-components', '4', '--extract',
660 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
661 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700662
Ryan Cui71aa8de2013-04-19 16:12:55 -0700663
Ryan Cui3045c5d2012-07-13 18:00:33 -0700664def main(argv):
665 options, args = _ParseCommandLine(argv)
666 _PostParseCheck(options, args)
667
668 # Set cros_build_lib debug level to hide RunCommand spew.
669 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700670 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700671 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800672 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700673
Ryan Cui71aa8de2013-04-19 16:12:55 -0700674 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700675 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
676 if cmd_stats:
677 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700678
Ryan Cui71aa8de2013-04-19 16:12:55 -0700679 with osutils.TempDir(set_global=True) as tempdir:
680 staging_dir = options.staging_dir
681 if not staging_dir:
682 staging_dir = os.path.join(tempdir, 'chrome')
683
684 deploy = DeployChrome(options, tempdir, staging_dir)
685 try:
686 deploy.Perform()
687 except results_lib.StepFailure as ex:
688 raise SystemExit(str(ex).strip())