blob: 2bfc960a973053b949d18e93e827d148229a4299 [file] [log] [blame]
Ryan Cui3045c5d2012-07-13 18:00:33 -07001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Steve Funge984a532013-11-25 17:09:25 -08005"""Script that deploys a Chrome build to a device.
Ryan Cuia56a71e2012-10-18 18:40:35 -07006
7The script supports deploying Chrome from these sources:
8
91. A local build output directory, such as chromium/src/out/[Debug|Release].
102. A Chrome tarball uploaded by a trybot/official-builder to GoogleStorage.
113. A Chrome tarball existing locally.
12
13The script copies the necessary contents of the source location (tarball or
14build directory) and rsyncs the contents of the staging directory onto your
15device's rootfs.
16"""
Ryan Cui3045c5d2012-07-13 18:00:33 -070017
Mike Frysinger383367e2014-09-16 15:06:17 -040018from __future__ import print_function
19
Mike Frysingerc3061a62015-06-04 04:16:18 -040020import argparse
Ryan Cui7193a7e2013-04-26 14:15:19 -070021import collections
Ryan Cuif890a3e2013-03-07 18:57:06 -080022import contextlib
Ryan Cui7193a7e2013-04-26 14:15:19 -070023import functools
Steve Funge984a532013-11-25 17:09:25 -080024import glob
David James88e6f032013-03-02 08:13:20 -080025import multiprocessing
Ryan Cui3045c5d2012-07-13 18:00:33 -070026import os
Ryan Cui5b7c2ed2013-03-18 18:45:46 -070027import shlex
Steve Funge984a532013-11-25 17:09:25 -080028import shutil
Ryan Cui3045c5d2012-07-13 18:00:33 -070029import time
Ryan Cui3045c5d2012-07-13 18:00:33 -070030
Don Garrett88b8d782014-05-13 17:30:55 -070031from chromite.cbuildbot import constants
David James14e97772014-06-04 18:44:49 -070032from chromite.cbuildbot import failures_lib
David Pursellcfd58872015-03-19 09:15:48 -070033from chromite.cli.cros import cros_chrome_sdk
Ryan Cuia56a71e2012-10-18 18:40:35 -070034from chromite.lib import chrome_util
Ryan Cuie535b172012-10-19 18:25:03 -070035from chromite.lib import commandline
Ralph Nathan91874ca2015-03-19 13:29:41 -070036from chromite.lib import cros_build_lib
37from chromite.lib import cros_logging as logging
Ryan Cui777ff422012-12-07 13:12:54 -080038from chromite.lib import gs
Ryan Cui3045c5d2012-07-13 18:00:33 -070039from chromite.lib import osutils
David James88e6f032013-03-02 08:13:20 -080040from chromite.lib import parallel
Ryan Cui3045c5d2012-07-13 18:00:33 -070041from chromite.lib import remote_access as remote
Ryan Cui71aa8de2013-04-19 16:12:55 -070042from chromite.lib import stats
David James3432acd2013-11-27 10:02:18 -080043from chromite.lib import timeout_util
Steven Bennetts368c3e52016-09-23 13:05:21 -070044from gn_helpers import gn_helpers
Ryan Cui3045c5d2012-07-13 18:00:33 -070045
46
Ryan Cui3045c5d2012-07-13 18:00:33 -070047KERNEL_A_PARTITION = 2
48KERNEL_B_PARTITION = 4
49
50KILL_PROC_MAX_WAIT = 10
51POST_KILL_WAIT = 2
52
Ryan Cuie535b172012-10-19 18:25:03 -070053MOUNT_RW_COMMAND = 'mount -o remount,rw /'
Pawel Osciak577773a2013-03-05 10:52:12 -080054LSOF_COMMAND = 'lsof %s/chrome'
Ryan Cui3045c5d2012-07-13 18:00:33 -070055
Steve Funge984a532013-11-25 17:09:25 -080056_ANDROID_DIR = '/system/chrome'
57_ANDROID_DIR_EXTRACT_PATH = 'system/chrome/*'
58
David James2cb34002013-03-01 18:42:40 -080059_CHROME_DIR = '/opt/google/chrome'
Thiago Goncales12793312013-05-23 11:26:17 -070060_CHROME_DIR_MOUNT = '/mnt/stateful_partition/deploy_rootfs/opt/google/chrome'
David James2cb34002013-03-01 18:42:40 -080061
Pawel Osciakc1bb2742014-12-29 16:32:33 +090062_UMOUNT_DIR_IF_MOUNTPOINT_CMD = (
63 'if mountpoint -q %(dir)s; then umount %(dir)s; fi')
Thiago Goncales12793312013-05-23 11:26:17 -070064_BIND_TO_FINAL_DIR_CMD = 'mount --rbind %s %s'
65_SET_MOUNT_FLAGS_CMD = 'mount -o remount,exec,suid %s'
Ryan Cui3045c5d2012-07-13 18:00:33 -070066
Steve Funge984a532013-11-25 17:09:25 -080067DF_COMMAND = 'df -k %s'
Steve Funge984a532013-11-25 17:09:25 -080068
Mike Frysingere65f3752014-12-08 00:46:39 -050069
Ryan Cui3045c5d2012-07-13 18:00:33 -070070def _UrlBaseName(url):
71 """Return the last component of the URL."""
72 return url.rstrip('/').rpartition('/')[-1]
73
74
Yu-Ju Hongc54d3342014-05-14 12:42:06 -070075class DeployFailure(failures_lib.StepFailure):
David James88e6f032013-03-02 08:13:20 -080076 """Raised whenever the deploy fails."""
77
78
Ryan Cui7193a7e2013-04-26 14:15:19 -070079DeviceInfo = collections.namedtuple(
80 'DeviceInfo', ['target_dir_size', 'target_fs_free'])
81
82
Ryan Cui3045c5d2012-07-13 18:00:33 -070083class DeployChrome(object):
84 """Wraps the core deployment functionality."""
Mike Frysingere65f3752014-12-08 00:46:39 -050085
Ryan Cuia56a71e2012-10-18 18:40:35 -070086 def __init__(self, options, tempdir, staging_dir):
Ryan Cuie535b172012-10-19 18:25:03 -070087 """Initialize the class.
88
Mike Frysinger02e1e072013-11-10 22:11:34 -050089 Args:
Mike Frysingerc3061a62015-06-04 04:16:18 -040090 options: options object.
Ryan Cuie535b172012-10-19 18:25:03 -070091 tempdir: Scratch space for the class. Caller has responsibility to clean
92 it up.
Steve Funge984a532013-11-25 17:09:25 -080093 staging_dir: Directory to stage the files to.
Ryan Cuie535b172012-10-19 18:25:03 -070094 """
Ryan Cui3045c5d2012-07-13 18:00:33 -070095 self.tempdir = tempdir
96 self.options = options
Ryan Cuia56a71e2012-10-18 18:40:35 -070097 self.staging_dir = staging_dir
Robert Flack1dc7ea82014-11-26 13:50:24 -050098 if not self.options.staging_only:
Pawel Osciak141b2262014-12-21 10:27:05 +090099 self.device = remote.RemoteDevice(options.to, port=options.port,
100 ping=options.ping)
Robert Flack1dc7ea82014-11-26 13:50:24 -0500101 self._target_dir_is_still_readonly = multiprocessing.Event()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700102
kylechardd2371f2016-04-11 11:09:31 -0400103 if self.options.mash:
104 self.copy_paths = chrome_util.GetCopyPaths('mash')
105 else:
106 self.copy_paths = chrome_util.GetCopyPaths('chrome')
Steve Funge984a532013-11-25 17:09:25 -0800107 self.chrome_dir = _CHROME_DIR
108
Ryan Cui7193a7e2013-04-26 14:15:19 -0700109 def _GetRemoteMountFree(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500110 result = self.device.RunCommand(DF_COMMAND % remote_dir)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700111 line = result.output.splitlines()[1]
Steve Funge984a532013-11-25 17:09:25 -0800112 value = line.split()[3]
113 multipliers = {
114 'G': 1024 * 1024 * 1024,
115 'M': 1024 * 1024,
116 'K': 1024,
117 }
118 return int(value.rstrip('GMK')) * multipliers.get(value[-1], 1)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700119
120 def _GetRemoteDirSize(self, remote_dir):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500121 result = self.device.RunCommand('du -ks %s' % remote_dir,
122 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700123 return int(result.output.split()[0])
124
125 def _GetStagingDirSize(self):
126 result = cros_build_lib.DebugRunCommand(['du', '-ks', self.staging_dir],
Yu-Ju Hong3add4432014-01-30 11:46:15 -0800127 redirect_stdout=True,
128 capture_output=True)
Ryan Cui7193a7e2013-04-26 14:15:19 -0700129 return int(result.output.split()[0])
130
Ryan Cui3045c5d2012-07-13 18:00:33 -0700131 def _ChromeFileInUse(self):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500132 result = self.device.RunCommand(LSOF_COMMAND % (self.options.target_dir,),
133 error_code_ok=True, capture_output=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700134 return result.returncode == 0
135
136 def _DisableRootfsVerification(self):
137 if not self.options.force:
138 logging.error('Detected that the device has rootfs verification enabled.')
139 logging.info('This script can automatically remove the rootfs '
140 'verification, which requires that it reboot the device.')
141 logging.info('Make sure the device is in developer mode!')
142 logging.info('Skip this prompt by specifying --force.')
Brian Harring521e7242012-11-01 16:57:42 -0700143 if not cros_build_lib.BooleanPrompt('Remove roots verification?', False):
David James88e6f032013-03-02 08:13:20 -0800144 # Since we stopped Chrome earlier, it's good form to start it up again.
145 if self.options.startui:
146 logging.info('Starting Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500147 self.device.RunCommand('start ui')
David James88e6f032013-03-02 08:13:20 -0800148 raise DeployFailure('Need rootfs verification to be disabled. '
149 'Aborting.')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700150
151 logging.info('Removing rootfs verification from %s', self.options.to)
152 # Running in VM's cause make_dev_ssd's firmware sanity checks to fail.
153 # Use --force to bypass the checks.
154 cmd = ('/usr/share/vboot/bin/make_dev_ssd.sh --partitions %d '
155 '--remove_rootfs_verification --force')
156 for partition in (KERNEL_A_PARTITION, KERNEL_B_PARTITION):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500157 self.device.RunCommand(cmd % partition, error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700158
159 # A reboot in developer mode takes a while (and has delays), so the user
160 # will have time to read and act on the USB boot instructions below.
161 logging.info('Please remember to press Ctrl-U if you are booting from USB.')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500162 self.device.Reboot()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700163
David James88e6f032013-03-02 08:13:20 -0800164 # Now that the machine has been rebooted, we need to kill Chrome again.
165 self._KillProcsIfNeeded()
166
167 # Make sure the rootfs is writable now.
168 self._MountRootfsAsWritable(error_code_ok=False)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700169
170 def _CheckUiJobStarted(self):
171 # status output is in the format:
172 # <job_name> <status> ['process' <pid>].
173 # <status> is in the format <goal>/<state>.
Ryan Cuif2d1a582013-02-19 14:08:13 -0800174 try:
Robert Flack1dc7ea82014-11-26 13:50:24 -0500175 result = self.device.RunCommand('status ui', capture_output=True)
Ryan Cuif2d1a582013-02-19 14:08:13 -0800176 except cros_build_lib.RunCommandError as e:
177 if 'Unknown job' in e.result.error:
178 return False
179 else:
180 raise e
181
Ryan Cui3045c5d2012-07-13 18:00:33 -0700182 return result.output.split()[1].split('/')[0] == 'start'
183
184 def _KillProcsIfNeeded(self):
185 if self._CheckUiJobStarted():
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800186 logging.info('Shutting down Chrome...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500187 self.device.RunCommand('stop ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700188
189 # Developers sometimes run session_manager manually, in which case we'll
190 # need to help shut the chrome processes down.
191 try:
David James3432acd2013-11-27 10:02:18 -0800192 with timeout_util.Timeout(KILL_PROC_MAX_WAIT):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700193 while self._ChromeFileInUse():
194 logging.warning('The chrome binary on the device is in use.')
195 logging.warning('Killing chrome and session_manager processes...\n')
196
Robert Flack1dc7ea82014-11-26 13:50:24 -0500197 self.device.RunCommand("pkill 'chrome|session_manager'",
Mike Frysingere65f3752014-12-08 00:46:39 -0500198 error_code_ok=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700199 # Wait for processes to actually terminate
200 time.sleep(POST_KILL_WAIT)
201 logging.info('Rechecking the chrome binary...')
David James3432acd2013-11-27 10:02:18 -0800202 except timeout_util.TimeoutError:
David James88e6f032013-03-02 08:13:20 -0800203 msg = ('Could not kill processes after %s seconds. Please exit any '
204 'running chrome processes and try again.' % KILL_PROC_MAX_WAIT)
205 raise DeployFailure(msg)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700206
David James88e6f032013-03-02 08:13:20 -0800207 def _MountRootfsAsWritable(self, error_code_ok=True):
208 """Mount the rootfs as writable.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700209
Robert Flack1dc7ea82014-11-26 13:50:24 -0500210 If the command fails, and error_code_ok is True, and the target dir is not
211 writable then this function sets self._target_dir_is_still_readonly.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700212
Mike Frysinger02e1e072013-11-10 22:11:34 -0500213 Args:
David James88e6f032013-03-02 08:13:20 -0800214 error_code_ok: See remote.RemoteAccess.RemoteSh for details.
215 """
Yu-Ju Hong7e116ac2014-01-03 17:08:26 -0800216 # TODO: Should migrate to use the remount functions in remote_access.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500217 result = self.device.RunCommand(MOUNT_RW_COMMAND,
218 error_code_ok=error_code_ok,
219 capture_output=True)
220 if (result.returncode and
Mike Frysinger74ccd572015-05-21 21:18:20 -0400221 not self.device.IsDirWritable(self.options.target_dir)):
Robert Flack1dc7ea82014-11-26 13:50:24 -0500222 self._target_dir_is_still_readonly.set()
Ryan Cui3045c5d2012-07-13 18:00:33 -0700223
Ryan Cui7193a7e2013-04-26 14:15:19 -0700224 def _GetDeviceInfo(self):
225 steps = [
226 functools.partial(self._GetRemoteDirSize, self.options.target_dir),
227 functools.partial(self._GetRemoteMountFree, self.options.target_dir)
228 ]
229 return_values = parallel.RunParallelSteps(steps, return_values=True)
230 return DeviceInfo(*return_values)
231
232 def _CheckDeviceFreeSpace(self, device_info):
233 """See if target device has enough space for Chrome.
234
Mike Frysinger02e1e072013-11-10 22:11:34 -0500235 Args:
Ryan Cui7193a7e2013-04-26 14:15:19 -0700236 device_info: A DeviceInfo named tuple.
237 """
238 effective_free = device_info.target_dir_size + device_info.target_fs_free
239 staging_size = self._GetStagingDirSize()
240 if effective_free < staging_size:
Daniel Erat1ae46382014-08-14 10:23:39 -0700241 raise DeployFailure(
242 'Not enough free space on the device. Required: %s MiB, '
243 'actual: %s MiB.' % (staging_size / 1024, effective_free / 1024))
Ryan Cui7193a7e2013-04-26 14:15:19 -0700244 if device_info.target_fs_free < (100 * 1024):
245 logging.warning('The device has less than 100MB free. deploy_chrome may '
246 'hang during the transfer.')
247
Ryan Cui3045c5d2012-07-13 18:00:33 -0700248 def _Deploy(self):
Pawel Osciak577773a2013-03-05 10:52:12 -0800249 logging.info('Copying Chrome to %s on device...', self.options.target_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700250 # Show the output (status) for this command.
Steve Funge984a532013-11-25 17:09:25 -0800251 dest_path = _CHROME_DIR
Satoru Takabayashiab380bc2015-01-15 16:00:41 +0900252 if not self.device.HasRsync():
253 raise DeployFailure(
David Pursellcfd58872015-03-19 09:15:48 -0700254 'rsync is not found on the device.\n'
255 'Run dev_install on the device to get rsync installed')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500256 self.device.CopyToDevice('%s/' % os.path.abspath(self.staging_dir),
257 self.options.target_dir,
258 inplace=True, debug_level=logging.INFO,
259 verbose=self.options.verbose)
Steve Funge984a532013-11-25 17:09:25 -0800260
261 for p in self.copy_paths:
Steve Funge984a532013-11-25 17:09:25 -0800262 if p.mode:
263 # Set mode if necessary.
Robert Flack1dc7ea82014-11-26 13:50:24 -0500264 self.device.RunCommand('chmod %o %s/%s' % (
265 p.mode, dest_path, p.src if not p.dest else p.dest))
Steve Funge984a532013-11-25 17:09:25 -0800266
Ryan Cui82a1f2d2013-02-22 17:34:23 -0800267 if self.options.startui:
Steve Funge984a532013-11-25 17:09:25 -0800268 logging.info('Starting UI...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500269 self.device.RunCommand('start ui')
Ryan Cui3045c5d2012-07-13 18:00:33 -0700270
David James88e6f032013-03-02 08:13:20 -0800271 def _CheckConnection(self):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700272 try:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800273 logging.info('Testing connection to the device...')
Robert Flack1dc7ea82014-11-26 13:50:24 -0500274 self.device.RunCommand('true')
David James88e6f032013-03-02 08:13:20 -0800275 except cros_build_lib.RunCommandError as ex:
Ryan Cui3045c5d2012-07-13 18:00:33 -0700276 logging.error('Error connecting to the test device.')
David James88e6f032013-03-02 08:13:20 -0800277 raise DeployFailure(ex)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700278
Steve Funge984a532013-11-25 17:09:25 -0800279 def _CheckDeployType(self):
Steve Fung63d705d2014-03-16 03:14:03 -0700280 if self.options.build_dir:
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700281 def BinaryExists(filename):
282 """Checks if the passed-in file is present in the build directory."""
283 return os.path.exists(os.path.join(self.options.build_dir, filename))
284
Daniel Erat9813f0e2014-11-12 11:00:28 -0700285 # Handle non-Chrome deployments.
286 if not BinaryExists('chrome'):
287 if BinaryExists('envoy_shell'):
288 self.copy_paths = chrome_util.GetCopyPaths('envoy')
289 elif BinaryExists('app_shell'):
290 self.copy_paths = chrome_util.GetCopyPaths('app_shell')
291
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700292 # TODO(derat): Update _Deploy() and remove this after figuring out how
Daniel Erat9813f0e2014-11-12 11:00:28 -0700293 # {app,envoy}_shell should be executed.
Daniel Erat3fd6c7a2014-05-02 14:28:31 -0700294 self.options.startui = False
295
David James88e6f032013-03-02 08:13:20 -0800296 def _PrepareStagingDir(self):
Steve Funge984a532013-11-25 17:09:25 -0800297 _PrepareStagingDir(self.options, self.tempdir, self.staging_dir,
298 self.copy_paths, self.chrome_dir)
David James88e6f032013-03-02 08:13:20 -0800299
Thiago Goncales12793312013-05-23 11:26:17 -0700300 def _MountTarget(self):
301 logging.info('Mounting Chrome...')
302
303 # Create directory if does not exist
Robert Flack1dc7ea82014-11-26 13:50:24 -0500304 self.device.RunCommand('mkdir -p --mode 0775 %s' % (
Mike Frysingere65f3752014-12-08 00:46:39 -0500305 self.options.mount_dir,))
Pawel Osciakfb200f52014-12-21 13:42:05 +0900306 # Umount the existing mount on mount_dir if present first
307 self.device.RunCommand(_UMOUNT_DIR_IF_MOUNTPOINT_CMD %
308 {'dir': self.options.mount_dir})
Robert Flack1dc7ea82014-11-26 13:50:24 -0500309 self.device.RunCommand(_BIND_TO_FINAL_DIR_CMD % (self.options.target_dir,
310 self.options.mount_dir))
Thiago Goncales12793312013-05-23 11:26:17 -0700311 # Chrome needs partition to have exec and suid flags set
Robert Flack1dc7ea82014-11-26 13:50:24 -0500312 self.device.RunCommand(_SET_MOUNT_FLAGS_CMD % (self.options.mount_dir,))
313
314 def Cleanup(self):
315 """Clean up RemoteDevice."""
316 if not self.options.staging_only:
317 self.device.Cleanup()
Thiago Goncales12793312013-05-23 11:26:17 -0700318
David James88e6f032013-03-02 08:13:20 -0800319 def Perform(self):
Steve Funge984a532013-11-25 17:09:25 -0800320 self._CheckDeployType()
321
David James88e6f032013-03-02 08:13:20 -0800322 # If requested, just do the staging step.
323 if self.options.staging_only:
324 self._PrepareStagingDir()
325 return 0
326
327 # Run setup steps in parallel. If any step fails, RunParallelSteps will
David James48f6b332013-03-03 20:11:18 -0800328 # stop printing output at that point, and halt any running steps.
Steve Funge984a532013-11-25 17:09:25 -0800329 steps = [self._GetDeviceInfo, self._CheckConnection,
330 self._KillProcsIfNeeded, self._MountRootfsAsWritable,
331 self._PrepareStagingDir]
Ryan Cui7193a7e2013-04-26 14:15:19 -0700332 ret = parallel.RunParallelSteps(steps, halt_on_error=True,
333 return_values=True)
334 self._CheckDeviceFreeSpace(ret[0])
David James88e6f032013-03-02 08:13:20 -0800335
Robert Flack1dc7ea82014-11-26 13:50:24 -0500336 # If we're trying to deploy to a dir which is not writable and we failed
337 # to mark the rootfs as writable, try disabling rootfs verification.
338 if self._target_dir_is_still_readonly.is_set():
David James88e6f032013-03-02 08:13:20 -0800339 self._DisableRootfsVerification()
340
Thiago Goncales12793312013-05-23 11:26:17 -0700341 if self.options.mount_dir is not None:
342 self._MountTarget()
343
David James88e6f032013-03-02 08:13:20 -0800344 # Actually deploy Chrome to the device.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700345 self._Deploy()
346
347
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700348def ValidateStagingFlags(value):
349 """Convert formatted string to dictionary."""
350 return chrome_util.ProcessShellFlags(value)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700351
352
Steven Bennetts368c3e52016-09-23 13:05:21 -0700353def ValidateGnArgs(value):
354 """Convert GN_ARGS-formatted string to dictionary."""
355 return gn_helpers.FromGNArgs(value)
356
357
Ryan Cuie535b172012-10-19 18:25:03 -0700358def _CreateParser():
359 """Create our custom parser."""
Mike Frysingerc3061a62015-06-04 04:16:18 -0400360 parser = commandline.ArgumentParser(description=__doc__, caching=True)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700361
Ryan Cuia56a71e2012-10-18 18:40:35 -0700362 # TODO(rcui): Have this use the UI-V2 format of having source and target
363 # device be specified as positional arguments.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400364 parser.add_argument('--force', action='store_true', default=False,
365 help='Skip all prompts (i.e., for disabling of rootfs '
366 'verification). This may result in the target '
367 'machine being rebooted.')
Ryan Cuia0215a72013-02-14 16:20:45 -0800368 sdk_board_env = os.environ.get(cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV)
Mike Frysingerc3061a62015-06-04 04:16:18 -0400369 parser.add_argument('--board', default=sdk_board_env,
370 help="The board the Chrome build is targeted for. When "
371 "in a 'cros chrome-sdk' shell, defaults to the SDK "
372 "board.")
373 parser.add_argument('--build-dir', type='path',
374 help='The directory with Chrome build artifacts to '
375 'deploy from. Typically of format '
376 '<chrome_root>/out/Debug. When this option is used, '
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700377 'the GN_ARGS environment variable must be set.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400378 parser.add_argument('--target-dir', type='path',
379 default=None,
380 help='Target directory on device to deploy Chrome into.')
381 parser.add_argument('-g', '--gs-path', type='gs_path',
382 help='GS path that contains the chrome to deploy.')
383 parser.add_argument('--nostartui', action='store_false', dest='startui',
384 default=True,
385 help="Don't restart the ui daemon after deployment.")
386 parser.add_argument('--nostrip', action='store_false', dest='dostrip',
387 default=True,
388 help="Don't strip binaries during deployment. Warning: "
389 'the resulting binaries will be very large!')
390 parser.add_argument('-p', '--port', type=int, default=remote.DEFAULT_SSH_PORT,
391 help='Port of the target device to connect to.')
392 parser.add_argument('-t', '--to',
393 help='The IP address of the CrOS device to deploy to.')
394 parser.add_argument('-v', '--verbose', action='store_true', default=False,
395 help='Show more debug output.')
396 parser.add_argument('--mount-dir', type='path', default=None,
397 help='Deploy Chrome in target directory and bind it '
398 'to the directory specified by this flag.'
399 'Any existing mount on this directory will be '
400 'umounted first.')
401 parser.add_argument('--mount', action='store_true', default=False,
402 help='Deploy Chrome to default target directory and bind '
403 'it to the default mount directory.'
404 'Any existing mount on this directory will be '
405 'umounted first.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700406
Mike Frysingerc3061a62015-06-04 04:16:18 -0400407 group = parser.add_argument_group('Advanced Options')
408 group.add_argument('-l', '--local-pkg-path', type='path',
409 help='Path to local chrome prebuilt package to deploy.')
410 group.add_argument('--sloppy', action='store_true', default=False,
411 help='Ignore when mandatory artifacts are missing.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700412 group.add_argument('--staging-flags', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400413 help=('Extra flags to control staging. Valid flags are - '
414 '%s' % ', '.join(chrome_util.STAGING_FLAGS)))
Steven Bennetts46a84c32016-08-12 15:20:46 -0700415 # TODO(stevenjb): Remove --strict entirely once removed from the ebuild.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400416 group.add_argument('--strict', action='store_true', default=False,
Steven Bennetts46a84c32016-08-12 15:20:46 -0700417 help='Deprecated. Default behavior is "strict". Use '
418 '--sloppy to omit warnings for missing optional '
419 'files.')
Mike Frysingerc3061a62015-06-04 04:16:18 -0400420 group.add_argument('--strip-flags', default=None,
421 help="Flags to call the 'strip' binutil tool with. "
422 "Overrides the default arguments.")
423 group.add_argument('--ping', action='store_true', default=False,
424 help='Ping the device before connection attempt.')
kylechardd2371f2016-04-11 11:09:31 -0400425 group.add_argument('--mash', action='store_true', default=False,
426 help='Copy additional files for mus+ash. Will not fit in '
427 'the default target-dir.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700428
Mike Frysingerc3061a62015-06-04 04:16:18 -0400429 group = parser.add_argument_group(
430 'Metadata Overrides (Advanced)',
431 description='Provide all of these overrides in order to remove '
432 'dependencies on metadata.json existence.')
433 group.add_argument('--target-tc', action='store', default=None,
434 help='Override target toolchain name, e.g. '
435 'x86_64-cros-linux-gnu')
436 group.add_argument('--toolchain-url', action='store', default=None,
437 help='Override toolchain url format pattern, e.g. '
438 '2014/04/%%(target)s-2014.04.23.220740.tar.xz')
Aviv Keshet1c986f32014-04-24 13:20:49 -0700439
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700440 # DEPRECATED: --gyp-defines is ignored, but retained for backwards
441 # compatibility. TODO(stevenjb): Remove once eliminated from the ebuild.
442 parser.add_argument('--gyp-defines', default=None, type=ValidateStagingFlags,
Mike Frysingerc3061a62015-06-04 04:16:18 -0400443 help=argparse.SUPPRESS)
Steven Bennetts368c3e52016-09-23 13:05:21 -0700444
445 # GN_ARGS (args.gn) used to build Chrome. Influences which files are staged
446 # when --build-dir is set. Defaults to reading from the GN_ARGS env variable.
447 # CURRENLY IGNORED, ADDED FOR FORWARD COMPATABILITY.
448 parser.add_argument('--gn-args', default=None, type=ValidateGnArgs,
449 help=argparse.SUPPRESS)
450
Ryan Cuia56a71e2012-10-18 18:40:35 -0700451 # Path of an empty directory to stage chrome artifacts to. Defaults to a
452 # temporary directory that is removed when the script finishes. If the path
453 # is specified, then it will not be removed.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400454 parser.add_argument('--staging-dir', type='path', default=None,
455 help=argparse.SUPPRESS)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700456 # Only prepare the staging directory, and skip deploying to the device.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400457 parser.add_argument('--staging-only', action='store_true', default=False,
458 help=argparse.SUPPRESS)
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700459 # Path to a binutil 'strip' tool to strip binaries with. The passed-in path
460 # is used as-is, and not normalized. Used by the Chrome ebuild to skip
461 # fetching the SDK toolchain.
Mike Frysingerc3061a62015-06-04 04:16:18 -0400462 parser.add_argument('--strip-bin', default=None, help=argparse.SUPPRESS)
Ryan Cuie535b172012-10-19 18:25:03 -0700463 return parser
Ryan Cui3045c5d2012-07-13 18:00:33 -0700464
Ryan Cuie535b172012-10-19 18:25:03 -0700465
466def _ParseCommandLine(argv):
467 """Parse args, and run environment-independent checks."""
468 parser = _CreateParser()
Mike Frysingerc3061a62015-06-04 04:16:18 -0400469 options = parser.parse_args(argv)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700470
Ryan Cuia56a71e2012-10-18 18:40:35 -0700471 if not any([options.gs_path, options.local_pkg_path, options.build_dir]):
472 parser.error('Need to specify either --gs-path, --local-pkg-path, or '
Ryan Cuief91e702013-02-04 12:06:36 -0800473 '--build-dir')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700474 if options.build_dir and any([options.gs_path, options.local_pkg_path]):
475 parser.error('Cannot specify both --build_dir and '
476 '--gs-path/--local-pkg-patch')
Ryan Cui686ec052013-02-12 16:39:41 -0800477 if options.build_dir and not options.board:
478 parser.error('--board is required when --build-dir is specified.')
Ryan Cuia56a71e2012-10-18 18:40:35 -0700479 if options.gs_path and options.local_pkg_path:
480 parser.error('Cannot specify both --gs-path and --local-pkg-path')
481 if not (options.staging_only or options.to):
Ryan Cui3045c5d2012-07-13 18:00:33 -0700482 parser.error('Need to specify --to')
Steven Bennetts46a84c32016-08-12 15:20:46 -0700483 if options.staging_flags and not options.build_dir:
484 parser.error('--staging-flags require --build-dir to be set.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700485
Steven Bennetts46a84c32016-08-12 15:20:46 -0700486 if options.strict:
487 logging.warning('--strict is deprecated.')
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700488 if options.gyp_defines:
489 logging.warning('--gyp-defines is deprecated.')
Thiago Goncales12793312013-05-23 11:26:17 -0700490
491 if options.mount or options.mount_dir:
492 if not options.target_dir:
493 options.target_dir = _CHROME_DIR_MOUNT
494 else:
495 if not options.target_dir:
496 options.target_dir = _CHROME_DIR
497
498 if options.mount and not options.mount_dir:
499 options.mount_dir = _CHROME_DIR
500
Mike Frysingerc3061a62015-06-04 04:16:18 -0400501 return options
Ryan Cui3045c5d2012-07-13 18:00:33 -0700502
503
Mike Frysingerc3061a62015-06-04 04:16:18 -0400504def _PostParseCheck(options):
Steve Funge984a532013-11-25 17:09:25 -0800505 """Perform some usage validation (after we've parsed the arguments).
Ryan Cui3045c5d2012-07-13 18:00:33 -0700506
507 Args:
Mike Frysinger5fe3f222016-09-01 00:14:16 -0400508 options: The options object returned by the cli parser.
Ryan Cui3045c5d2012-07-13 18:00:33 -0700509 """
Ryan Cuia56a71e2012-10-18 18:40:35 -0700510 if options.local_pkg_path and not os.path.isfile(options.local_pkg_path):
511 cros_build_lib.Die('%s is not a file.', options.local_pkg_path)
512
Steven Bennetts368c3e52016-09-23 13:05:21 -0700513 if not options.gn_args:
514 gn_env = os.getenv('GN_ARGS')
515 if gn_env is not None:
516 options.gn_args = gn_helpers.FromGNArgs(gn_env)
517 logging.info('GN_ARGS taken from environment: %s', options.gn_args)
Ryan Cuib623e7b2013-03-14 12:54:11 -0700518
Steven Bennetts60600462016-05-12 10:40:20 -0700519 if not options.staging_flags:
520 use_env = os.getenv('USE')
521 if use_env is not None:
522 options.staging_flags = ' '.join(set(use_env.split()).intersection(
523 chrome_util.STAGING_FLAGS))
524 logging.info('Staging flags taken from USE in environment: %s',
525 options.staging_flags)
526
Ryan Cuia56a71e2012-10-18 18:40:35 -0700527
Ryan Cui504db722013-01-22 11:48:01 -0800528def _FetchChromePackage(cache_dir, tempdir, gs_path):
Ryan Cuia56a71e2012-10-18 18:40:35 -0700529 """Get the chrome prebuilt tarball from GS.
530
Mike Frysinger02e1e072013-11-10 22:11:34 -0500531 Returns:
532 Path to the fetched chrome tarball.
Ryan Cuia56a71e2012-10-18 18:40:35 -0700533 """
David James9374aac2013-10-08 16:00:17 -0700534 gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=True)
Mike Frysinger7ad4fd62014-02-11 16:53:56 -0500535 files = gs_ctx.LS(gs_path)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800536 files = [found for found in files if
537 _UrlBaseName(found).startswith('%s-' % constants.CHROME_PN)]
538 if not files:
539 raise Exception('No chrome package found at %s' % gs_path)
540 elif len(files) > 1:
541 # - Users should provide us with a direct link to either a stripped or
542 # unstripped chrome package.
543 # - In the case of being provided with an archive directory, where both
544 # stripped and unstripped chrome available, use the stripped chrome
545 # package.
546 # - Stripped chrome pkg is chromeos-chrome-<version>.tar.gz
547 # - Unstripped chrome pkg is chromeos-chrome-<version>-unstripped.tar.gz.
548 files = [f for f in files if not 'unstripped' in f]
549 assert len(files) == 1
550 logging.warning('Multiple chrome packages found. Using %s', files[0])
Ryan Cui777ff422012-12-07 13:12:54 -0800551
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800552 filename = _UrlBaseName(files[0])
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800553 logging.info('Fetching %s...', filename)
Ryan Cui4c2d42c2013-02-08 16:22:26 -0800554 gs_ctx.Copy(files[0], tempdir, print_cmd=False)
555 chrome_path = os.path.join(tempdir, filename)
556 assert os.path.exists(chrome_path)
557 return chrome_path
Ryan Cuia56a71e2012-10-18 18:40:35 -0700558
559
Ryan Cuif890a3e2013-03-07 18:57:06 -0800560@contextlib.contextmanager
561def _StripBinContext(options):
562 if not options.dostrip:
563 yield None
564 elif options.strip_bin:
565 yield options.strip_bin
566 else:
Ryan Cuia0215a72013-02-14 16:20:45 -0800567 sdk = cros_chrome_sdk.SDKFetcher(options.cache_dir, options.board)
Ryan Cui686ec052013-02-12 16:39:41 -0800568 components = (sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR)
Aviv Keshet1c986f32014-04-24 13:20:49 -0700569 with sdk.Prepare(components=components, target_tc=options.target_tc,
570 toolchain_url=options.toolchain_url) as ctx:
Ryan Cui686ec052013-02-12 16:39:41 -0800571 env_path = os.path.join(ctx.key_map[constants.CHROME_ENV_TAR].path,
572 constants.CHROME_ENV_FILE)
573 strip_bin = osutils.SourceEnvironment(env_path, ['STRIP'])['STRIP']
574 strip_bin = os.path.join(ctx.key_map[sdk.TARGET_TOOLCHAIN_KEY].path,
575 'bin', os.path.basename(strip_bin))
Ryan Cuif890a3e2013-03-07 18:57:06 -0800576 yield strip_bin
577
578
Steve Funge984a532013-11-25 17:09:25 -0800579def _PrepareStagingDir(options, tempdir, staging_dir, copy_paths=None,
580 chrome_dir=_CHROME_DIR):
Ryan Cuif890a3e2013-03-07 18:57:06 -0800581 """Place the necessary files in the staging directory.
582
583 The staging directory is the directory used to rsync the build artifacts over
584 to the device. Only the necessary Chrome build artifacts are put into the
585 staging directory.
586 """
Ryan Cui5866be02013-03-18 14:12:00 -0700587 osutils.SafeMakedirs(staging_dir)
Mike Frysinger60ec1012013-10-21 00:11:10 -0400588 os.chmod(staging_dir, 0o755)
Ryan Cuif890a3e2013-03-07 18:57:06 -0800589 if options.build_dir:
590 with _StripBinContext(options) as strip_bin:
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700591 strip_flags = (None if options.strip_flags is None else
592 shlex.split(options.strip_flags))
Ryan Cui686ec052013-02-12 16:39:41 -0800593 chrome_util.StageChromeFromBuildDir(
Steven Bennetts46a84c32016-08-12 15:20:46 -0700594 staging_dir, options.build_dir, strip_bin,
Steven Bennettsda8d9f02016-09-23 13:25:45 -0700595 sloppy=options.sloppy, gn_args=options.gn_args,
Ryan Cui5b7c2ed2013-03-18 18:45:46 -0700596 staging_flags=options.staging_flags,
Steve Funge984a532013-11-25 17:09:25 -0800597 strip_flags=strip_flags, copy_paths=copy_paths)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700598 else:
599 pkg_path = options.local_pkg_path
600 if options.gs_path:
Ryan Cui504db722013-01-22 11:48:01 -0800601 pkg_path = _FetchChromePackage(options.cache_dir, tempdir,
602 options.gs_path)
Ryan Cuia56a71e2012-10-18 18:40:35 -0700603
604 assert pkg_path
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800605 logging.info('Extracting %s...', pkg_path)
Ryan Cui5866be02013-03-18 14:12:00 -0700606 # Extract only the ./opt/google/chrome contents, directly into the staging
607 # dir, collapsing the directory hierarchy.
Steve Funge984a532013-11-25 17:09:25 -0800608 if pkg_path[-4:] == '.zip':
609 cros_build_lib.DebugRunCommand(
610 ['unzip', '-X', pkg_path, _ANDROID_DIR_EXTRACT_PATH, '-d',
611 staging_dir])
612 for filename in glob.glob(os.path.join(staging_dir, 'system/chrome/*')):
613 shutil.move(filename, staging_dir)
614 osutils.RmDir(os.path.join(staging_dir, 'system'), ignore_missing=True)
615 else:
616 cros_build_lib.DebugRunCommand(
617 ['tar', '--strip-components', '4', '--extract',
618 '--preserve-permissions', '--file', pkg_path, '.%s' % chrome_dir],
619 cwd=staging_dir)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700620
Ryan Cui71aa8de2013-04-19 16:12:55 -0700621
Ryan Cui3045c5d2012-07-13 18:00:33 -0700622def main(argv):
Mike Frysingerc3061a62015-06-04 04:16:18 -0400623 options = _ParseCommandLine(argv)
624 _PostParseCheck(options)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700625
626 # Set cros_build_lib debug level to hide RunCommand spew.
627 if options.verbose:
Ryan Cuia56a71e2012-10-18 18:40:35 -0700628 logging.getLogger().setLevel(logging.DEBUG)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700629 else:
Ryan Cui4d6b0db2013-02-28 15:13:24 -0800630 logging.getLogger().setLevel(logging.INFO)
Ryan Cui3045c5d2012-07-13 18:00:33 -0700631
Ryan Cui71aa8de2013-04-19 16:12:55 -0700632 with stats.UploadContext() as queue:
Ryan Cuicbd9bb62013-04-30 11:17:02 -0700633 cmd_stats = stats.Stats.SafeInit(cmd_line=argv, cmd_base='deploy_chrome')
634 if cmd_stats:
635 queue.put([cmd_stats, stats.StatsUploader.URL, 1])
Ryan Cuia56a71e2012-10-18 18:40:35 -0700636
Ryan Cui71aa8de2013-04-19 16:12:55 -0700637 with osutils.TempDir(set_global=True) as tempdir:
638 staging_dir = options.staging_dir
639 if not staging_dir:
640 staging_dir = os.path.join(tempdir, 'chrome')
641
642 deploy = DeployChrome(options, tempdir, staging_dir)
643 try:
644 deploy.Perform()
Yu-Ju Hongc54d3342014-05-14 12:42:06 -0700645 except failures_lib.StepFailure as ex:
Ryan Cui71aa8de2013-04-19 16:12:55 -0700646 raise SystemExit(str(ex).strip())
Robert Flack1dc7ea82014-11-26 13:50:24 -0500647 deploy.Cleanup()