Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 1 | #!/usr/bin/python -u |
| 2 | # |
| 3 | # -*- coding: utf-8 -*- |
| 4 | # |
| 5 | # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 6 | # Use of this source code is governed by a BSD-style license that can be |
| 7 | # found in the LICENSE file. |
| 8 | |
| 9 | ''' |
| 10 | The main factory flow that runs the factory test and finalizes a device. |
| 11 | ''' |
| 12 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 13 | import array |
| 14 | import fcntl |
| 15 | import glob |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 16 | import hashlib |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 17 | import logging |
| 18 | import os |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 19 | import cPickle as pickle |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 20 | import pipes |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 21 | import Queue |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 22 | import re |
| 23 | import signal |
| 24 | import subprocess |
| 25 | import sys |
| 26 | import tempfile |
| 27 | import threading |
| 28 | import time |
| 29 | import traceback |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 30 | import unittest |
| 31 | import uuid |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 32 | from collections import deque |
| 33 | from optparse import OptionParser |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 34 | from StringIO import StringIO |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 35 | |
| 36 | import factory_common |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 37 | from autotest_lib.client.bin.prespawner import Prespawner |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 38 | from autotest_lib.client.cros import factory |
| 39 | from autotest_lib.client.cros.factory import state |
| 40 | from autotest_lib.client.cros.factory import TestState |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 41 | from autotest_lib.client.cros.factory import utils |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 42 | from autotest_lib.client.cros.factory.event import Event |
| 43 | from autotest_lib.client.cros.factory.event import EventClient |
| 44 | from autotest_lib.client.cros.factory.event import EventServer |
Jon Salz | eb8d25f | 2012-05-22 15:17:32 +0800 | [diff] [blame] | 45 | from autotest_lib.client.cros.factory.event_log import EventLog |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 46 | from autotest_lib.client.cros.factory.invocation import TestInvocation |
| 47 | from autotest_lib.client.cros.factory.web_socket_manager import WebSocketManager |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 48 | |
| 49 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 50 | DEFAULT_TEST_LIST_PATH = os.path.join( |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 51 | factory.CLIENT_PATH , 'site_tests', 'suite_Factory', 'test_list') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 52 | HWID_CFG_PATH = '/usr/local/share/chromeos-hwid/cfg' |
| 53 | |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 54 | GOOFY_IN_CHROOT_WARNING = '\n' + ('*' * 70) + ''' |
| 55 | You are running Goofy inside the chroot. Autotests are not supported. |
| 56 | |
| 57 | To use Goofy in the chroot, first install an Xvnc server: |
| 58 | |
| 59 | sudo apt-get install tightvncserver |
| 60 | |
| 61 | ...and then start a VNC X server outside the chroot: |
| 62 | |
| 63 | vncserver :10 & |
| 64 | vncviewer :10 |
| 65 | |
| 66 | ...and run Goofy as follows: |
| 67 | |
| 68 | env --unset=XAUTHORITY DISPLAY=localhost:10 python goofy.py |
| 69 | ''' + ('*' * 70) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 70 | suppress_chroot_warning = False |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 71 | |
| 72 | def get_hwid_cfg(): |
| 73 | ''' |
| 74 | Returns the HWID config tag, or an empty string if none can be found. |
| 75 | ''' |
| 76 | if 'CROS_HWID' in os.environ: |
| 77 | return os.environ['CROS_HWID'] |
| 78 | if os.path.exists(HWID_CFG_PATH): |
| 79 | with open(HWID_CFG_PATH, 'rt') as hwid_cfg_handle: |
| 80 | return hwid_cfg_handle.read().strip() |
| 81 | return '' |
| 82 | |
| 83 | |
| 84 | def find_test_list(): |
| 85 | ''' |
| 86 | Returns the path to the active test list, based on the HWID config tag. |
| 87 | ''' |
| 88 | hwid_cfg = get_hwid_cfg() |
| 89 | |
| 90 | # Try in order: test_list, test_list.$hwid_cfg, test_list.all |
| 91 | if hwid_cfg: |
| 92 | test_list = '%s_%s' % (DEFAULT_TEST_LIST_PATH, hwid_cfg) |
| 93 | if os.path.exists(test_list): |
| 94 | logging.info('Using special test list: %s', test_list) |
| 95 | return test_list |
| 96 | logging.info('WARNING: no specific test list for config: %s', hwid_cfg) |
| 97 | |
| 98 | test_list = DEFAULT_TEST_LIST_PATH |
| 99 | if os.path.exists(test_list): |
| 100 | return test_list |
| 101 | |
| 102 | test_list = ('%s.all' % DEFAULT_TEST_LIST_PATH) |
| 103 | if os.path.exists(test_list): |
| 104 | logging.info('Using default test list: ' + test_list) |
| 105 | return test_list |
| 106 | logging.info('ERROR: Cannot find any test list.') |
| 107 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 108 | |
| 109 | class Environment(object): |
| 110 | ''' |
| 111 | Abstract base class for external test operations, e.g., run an autotest, |
| 112 | shutdown, or reboot. |
| 113 | |
| 114 | The Environment is assumed not to be thread-safe: callers must grab the lock |
| 115 | before calling any methods. This is primarily necessary because we mock out |
| 116 | this Environment with mox, and unfortunately mox is not thread-safe. |
| 117 | TODO(jsalz): Try to write a thread-safe wrapper for mox. |
| 118 | ''' |
| 119 | lock = threading.Lock() |
| 120 | |
| 121 | def shutdown(self, operation): |
| 122 | ''' |
| 123 | Shuts the machine down (from a ShutdownStep). |
| 124 | |
| 125 | Args: |
| 126 | operation: 'reboot' or 'halt'. |
| 127 | |
| 128 | Returns: |
| 129 | True if Goofy should gracefully exit, or False if Goofy |
| 130 | should just consider the shutdown to have suceeded (e.g., |
| 131 | in the chroot). |
| 132 | ''' |
| 133 | raise NotImplementedError() |
| 134 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 135 | def launch_chrome(self): |
| 136 | ''' |
| 137 | Launches Chrome. |
| 138 | |
| 139 | Returns: |
| 140 | The Chrome subprocess (or None if none). |
| 141 | ''' |
| 142 | raise NotImplementedError() |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 143 | |
| 144 | def spawn_autotest(self, name, args, env_additions, result_file): |
| 145 | ''' |
| 146 | Spawns a process to run an autotest. |
| 147 | |
| 148 | Args: |
| 149 | name: Name of the autotest to spawn. |
| 150 | args: Command-line arguments. |
| 151 | env_additions: Additions to the environment. |
| 152 | result_file: Expected location of the result file. |
| 153 | ''' |
| 154 | raise NotImplementedError() |
| 155 | |
| 156 | |
| 157 | class DUTEnvironment(Environment): |
| 158 | ''' |
| 159 | A real environment on a device under test. |
| 160 | ''' |
| 161 | def shutdown(self, operation): |
| 162 | assert operation in ['reboot', 'halt'] |
| 163 | logging.info('Shutting down: %s', operation) |
| 164 | subprocess.check_call('sync') |
| 165 | subprocess.check_call(operation) |
| 166 | time.sleep(30) |
| 167 | assert False, 'Never reached (should %s)' % operation |
| 168 | |
| 169 | def spawn_autotest(self, name, args, env_additions, result_file): |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 170 | return self.goofy.prespawner.spawn(args, env_additions) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 171 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 172 | def launch_chrome(self): |
| 173 | chrome_command = [ |
| 174 | '/opt/google/chrome/chrome', |
| 175 | '--user-data-dir=%s/factory-chrome-datadir' % |
| 176 | factory.get_log_root(), |
| 177 | '--aura-host-window-use-fullscreen', |
| 178 | '--kiosk', |
Jon Salz | 63585ea | 2012-05-21 15:03:32 +0800 | [diff] [blame] | 179 | ('--default-device-scale-factor=%d' % |
| 180 | self.goofy.options.ui_scale_factor), |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 181 | 'http://localhost:%d/' % state.DEFAULT_FACTORY_STATE_PORT, |
| 182 | ] |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 183 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 184 | chrome_log = os.path.join(factory.get_log_root(), 'factory.chrome.log') |
| 185 | chrome_log_file = open(chrome_log, "a") |
| 186 | logging.info('Launching Chrome; logs in %s' % chrome_log) |
| 187 | return subprocess.Popen(chrome_command, |
| 188 | stdout=chrome_log_file, |
| 189 | stderr=subprocess.STDOUT) |
| 190 | |
| 191 | class FakeChrootEnvironment(Environment): |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 192 | ''' |
| 193 | A chroot environment that doesn't actually shutdown or run autotests. |
| 194 | ''' |
| 195 | def shutdown(self, operation): |
| 196 | assert operation in ['reboot', 'halt'] |
| 197 | logging.warn('In chroot: skipping %s', operation) |
| 198 | return False |
| 199 | |
| 200 | def spawn_autotest(self, name, args, env_additions, result_file): |
| 201 | logging.warn('In chroot: skipping autotest %s', name) |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 202 | # Mark it as passed with 75% probability, or failed with 25% |
| 203 | # probability (depending on a hash of the autotest name). |
| 204 | pseudo_random = ord(hashlib.sha1(name).digest()[0]) / 256.0 |
| 205 | passed = pseudo_random > .25 |
| 206 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 207 | with open(result_file, 'w') as out: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 208 | pickle.dump((passed, '' if passed else 'Simulated failure'), out) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 209 | # Start a process that will return with a true exit status in |
| 210 | # 2 seconds (just like a happy autotest). |
| 211 | return subprocess.Popen(['sleep', '2']) |
| 212 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 213 | def launch_chrome(self): |
| 214 | logging.warn('In chroot; not launching Chrome. ' |
| 215 | 'Please open http://localhost:%d/ in Chrome.', |
| 216 | state.DEFAULT_FACTORY_STATE_PORT) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 217 | |
| 218 | |
| 219 | _inited_logging = False |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 220 | |
| 221 | class Goofy(object): |
| 222 | ''' |
| 223 | The main factory flow. |
| 224 | |
| 225 | Note that all methods in this class must be invoked from the main |
| 226 | (event) thread. Other threads, such as callbacks and TestInvocation |
| 227 | methods, should instead post events on the run queue. |
| 228 | |
| 229 | TODO: Unit tests. (chrome-os-partner:7409) |
| 230 | |
| 231 | Properties: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 232 | uuid: A unique UUID for this invocation of Goofy. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 233 | state_instance: An instance of FactoryState. |
| 234 | state_server: The FactoryState XML/RPC server. |
| 235 | state_server_thread: A thread running state_server. |
| 236 | event_server: The EventServer socket server. |
| 237 | event_server_thread: A thread running event_server. |
| 238 | event_client: A client to the event server. |
Hung-Te Lin | 6bb4855 | 2012-02-09 14:37:43 +0800 | [diff] [blame] | 239 | ui_process: The factory ui process object. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 240 | run_queue: A queue of callbacks to invoke from the main thread. |
| 241 | invocations: A map from FactoryTest objects to the corresponding |
| 242 | TestInvocations objects representing active tests. |
| 243 | tests_to_run: A deque of tests that should be run when the current |
| 244 | test(s) complete. |
| 245 | options: Command-line options. |
| 246 | args: Command-line args. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 247 | test_list: The test list. |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 248 | event_handlers: Map of Event.Type to the method used to handle that |
| 249 | event. If the method has an 'event' argument, the event is passed |
| 250 | to the handler. |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 251 | exceptions: Exceptions encountered in invocation threads. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 252 | ''' |
| 253 | def __init__(self): |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 254 | self.uuid = str(uuid.uuid4()) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 255 | self.state_instance = None |
| 256 | self.state_server = None |
| 257 | self.state_server_thread = None |
| 258 | self.event_server = None |
| 259 | self.event_server_thread = None |
| 260 | self.event_client = None |
Jon Salz | eb8d25f | 2012-05-22 15:17:32 +0800 | [diff] [blame] | 261 | self.event_log = None |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 262 | self.prespawner = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 263 | self.ui_process = None |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 264 | self.run_queue = Queue.Queue() |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 265 | self.invocations = {} |
| 266 | self.tests_to_run = deque() |
| 267 | self.visible_test = None |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 268 | self.chrome = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 269 | |
| 270 | self.options = None |
| 271 | self.args = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 272 | self.test_list = None |
| 273 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 274 | def test_or_root(event): |
| 275 | '''Returns the top-level parent for a test (the root node of the |
| 276 | tests that need to be run together if the given test path is to |
| 277 | be run).''' |
| 278 | try: |
| 279 | path = event.path |
Jon Salz | d2ed6cb | 2012-05-02 09:35:14 +0800 | [diff] [blame] | 280 | except AttributeError: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 281 | path = None |
| 282 | |
| 283 | if path: |
| 284 | return self.test_list.lookup_path(path).get_top_level_parent() |
| 285 | else: |
| 286 | return self.test_list |
| 287 | |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 288 | self.event_handlers = { |
| 289 | Event.Type.SWITCH_TEST: self.handle_switch_test, |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 290 | Event.Type.SHOW_NEXT_ACTIVE_TEST: |
| 291 | lambda event: self.show_next_active_test(), |
| 292 | Event.Type.RESTART_TESTS: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 293 | lambda event: self.restart_tests(root=test_or_root(event)), |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 294 | Event.Type.AUTO_RUN: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 295 | lambda event: self.auto_run(root=test_or_root(event)), |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 296 | Event.Type.RE_RUN_FAILED: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 297 | lambda event: self.re_run_failed(root=test_or_root(event)), |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 298 | Event.Type.REVIEW: |
| 299 | lambda event: self.show_review_information(), |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 300 | } |
| 301 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 302 | self.exceptions = [] |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 303 | self.web_socket_manager = None |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 304 | |
| 305 | def destroy(self): |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 306 | if self.chrome: |
| 307 | self.chrome.kill() |
| 308 | self.chrome = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 309 | if self.ui_process: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 310 | utils.kill_process_tree(self.ui_process, 'ui') |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 311 | self.ui_process = None |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 312 | if self.web_socket_manager: |
| 313 | logging.info('Stopping web sockets') |
| 314 | self.web_socket_manager.close() |
| 315 | self.web_socket_manager = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 316 | if self.state_server_thread: |
| 317 | logging.info('Stopping state server') |
| 318 | self.state_server.shutdown() |
| 319 | self.state_server_thread.join() |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 320 | self.state_server.server_close() |
| 321 | self.state_server_thread = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 322 | if self.event_server_thread: |
| 323 | logging.info('Stopping event server') |
| 324 | self.event_server.shutdown() # pylint: disable=E1101 |
| 325 | self.event_server_thread.join() |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 326 | self.event_server.server_close() |
| 327 | self.event_server_thread = None |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 328 | if self.prespawner: |
| 329 | logging.info('Stopping prespawner') |
| 330 | self.prespawner.stop() |
| 331 | self.prespawner = None |
| 332 | if self.event_client: |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 333 | logging.info('Closing event client') |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 334 | self.event_client.close() |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 335 | self.event_client = None |
Jon Salz | eb8d25f | 2012-05-22 15:17:32 +0800 | [diff] [blame] | 336 | if self.event_log: |
| 337 | self.event_log.Close() |
| 338 | self.event_log = None |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 339 | self.check_exceptions() |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 340 | logging.info('Done destroying Goofy') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 341 | |
| 342 | def start_state_server(self): |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 343 | self.state_instance, self.state_server = ( |
| 344 | state.create_server(bind_address='0.0.0.0')) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 345 | logging.info('Starting state server') |
| 346 | self.state_server_thread = threading.Thread( |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 347 | target=self.state_server.serve_forever, |
| 348 | name='StateServer') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 349 | self.state_server_thread.start() |
| 350 | |
| 351 | def start_event_server(self): |
| 352 | self.event_server = EventServer() |
| 353 | logging.info('Starting factory event server') |
| 354 | self.event_server_thread = threading.Thread( |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 355 | target=self.event_server.serve_forever, |
| 356 | name='EventServer') # pylint: disable=E1101 |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 357 | self.event_server_thread.start() |
| 358 | |
| 359 | self.event_client = EventClient( |
| 360 | callback=self.handle_event, event_loop=self.run_queue) |
| 361 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 362 | self.web_socket_manager = WebSocketManager(self.uuid) |
| 363 | self.state_server.add_handler("/event", |
| 364 | self.web_socket_manager.handle_web_socket) |
| 365 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 366 | def start_ui(self): |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 367 | ui_proc_args = [os.path.join(factory.CROS_FACTORY_LIB_PATH, 'ui'), |
| 368 | self.options.test_list] |
Jon Salz | 14bcbb0 | 2012-03-17 15:11:50 +0800 | [diff] [blame] | 369 | if self.options.verbose: |
| 370 | ui_proc_args.append('-v') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 371 | logging.info('Starting ui %s', ui_proc_args) |
| 372 | self.ui_process = subprocess.Popen(ui_proc_args) |
| 373 | logging.info('Waiting for UI to come up...') |
| 374 | self.event_client.wait( |
| 375 | lambda event: event.type == Event.Type.UI_READY) |
| 376 | logging.info('UI has started') |
| 377 | |
| 378 | def set_visible_test(self, test): |
| 379 | if self.visible_test == test: |
| 380 | return |
| 381 | |
| 382 | if test: |
| 383 | test.update_state(visible=True) |
| 384 | if self.visible_test: |
| 385 | self.visible_test.update_state(visible=False) |
| 386 | self.visible_test = test |
| 387 | |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 388 | def handle_shutdown_complete(self, test, state): |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 389 | ''' |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 390 | Handles the case where a shutdown was detected during a shutdown step. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 391 | |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 392 | @param test: The ShutdownStep. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 393 | @param state: The test state. |
| 394 | ''' |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 395 | state = test.update_state(increment_shutdown_count=1) |
| 396 | logging.info('Detected shutdown (%d of %d)', |
| 397 | state.shutdown_count, test.iterations) |
| 398 | if state.shutdown_count == test.iterations: |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 399 | # Good! |
| 400 | test.update_state(status=TestState.PASSED, error_msg='') |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 401 | elif state.shutdown_count > test.iterations: |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 402 | # Shut down too many times |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 403 | test.update_state(status=TestState.FAILED, |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 404 | error_msg='Too many shutdowns') |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 405 | elif utils.are_shift_keys_depressed(): |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 406 | logging.info('Shift keys are depressed; cancelling restarts') |
| 407 | # Abort shutdown |
| 408 | test.update_state( |
| 409 | status=TestState.FAILED, |
| 410 | error_msg='Shutdown aborted with double shift keys') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 411 | else: |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 412 | # Need to shutdown again |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 413 | self.env.shutdown('reboot') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 414 | |
| 415 | def init_states(self): |
| 416 | ''' |
| 417 | Initializes all states on startup. |
| 418 | ''' |
| 419 | for test in self.test_list.get_all_tests(): |
| 420 | # Make sure the state server knows about all the tests, |
| 421 | # defaulting to an untested state. |
| 422 | test.update_state(update_parent=False, visible=False) |
| 423 | |
| 424 | # Any 'active' tests should be marked as failed now. |
| 425 | for test in self.test_list.walk(): |
| 426 | state = test.get_state() |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 427 | if state.status != TestState.ACTIVE: |
| 428 | continue |
| 429 | if isinstance(test, factory.ShutdownStep): |
| 430 | # Shutdown while the test was active - that's good. |
| 431 | self.handle_shutdown_complete(test, state) |
| 432 | else: |
| 433 | test.update_state(status=TestState.FAILED, |
| 434 | error_msg='Unknown (shutdown?)') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 435 | |
| 436 | def show_next_active_test(self): |
| 437 | ''' |
| 438 | Rotates to the next visible active test. |
| 439 | ''' |
| 440 | self.reap_completed_tests() |
| 441 | active_tests = [ |
| 442 | t for t in self.test_list.walk() |
| 443 | if t.is_leaf() and t.get_state().status == TestState.ACTIVE] |
| 444 | if not active_tests: |
| 445 | return |
| 446 | |
| 447 | try: |
| 448 | next_test = active_tests[ |
| 449 | (active_tests.index(self.visible_test) + 1) % len(active_tests)] |
| 450 | except ValueError: # visible_test not present in active_tests |
| 451 | next_test = active_tests[0] |
| 452 | |
| 453 | self.set_visible_test(next_test) |
| 454 | |
| 455 | def handle_event(self, event): |
| 456 | ''' |
| 457 | Handles an event from the event server. |
| 458 | ''' |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 459 | handler = self.event_handlers.get(event.type) |
| 460 | if handler: |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 461 | handler(event) |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 462 | else: |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 463 | # We don't register handlers for all event types - just ignore |
| 464 | # this event. |
| 465 | logging.debug('Unbound event type %s', event.type) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 466 | |
| 467 | def run_next_test(self): |
| 468 | ''' |
| 469 | Runs the next eligible test (or tests) in self.tests_to_run. |
| 470 | ''' |
| 471 | self.reap_completed_tests() |
| 472 | while self.tests_to_run: |
| 473 | logging.debug('Tests to run: %s', |
| 474 | [x.path for x in self.tests_to_run]) |
| 475 | |
| 476 | test = self.tests_to_run[0] |
| 477 | |
| 478 | if test in self.invocations: |
| 479 | logging.info('Next test %s is already running', test.path) |
| 480 | self.tests_to_run.popleft() |
| 481 | return |
| 482 | |
| 483 | if self.invocations and not (test.backgroundable and all( |
| 484 | [x.backgroundable for x in self.invocations])): |
| 485 | logging.debug('Waiting for non-backgroundable tests to ' |
| 486 | 'complete before running %s', test.path) |
| 487 | return |
| 488 | |
| 489 | self.tests_to_run.popleft() |
| 490 | |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 491 | if isinstance(test, factory.ShutdownStep): |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 492 | test.update_state(status=TestState.ACTIVE, increment_count=1, |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 493 | error_msg='', shutdown_count=0) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 494 | # Save pending test list in the state server |
| 495 | self.state_instance.set_shared_data( |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 496 | 'tests_after_shutdown', |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 497 | [t.path for t in self.tests_to_run]) |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 498 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 499 | with self.env.lock: |
| 500 | shutdown_result = self.env.shutdown(test.operation) |
| 501 | if shutdown_result: |
| 502 | # That's all, folks! |
| 503 | self.run_queue.put(None) |
| 504 | return |
| 505 | else: |
| 506 | # Just pass (e.g., in the chroot). |
| 507 | test.update_state(status=TestState.PASSED) |
| 508 | self.state_instance.set_shared_data( |
| 509 | 'tests_after_shutdown', None) |
| 510 | continue |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 511 | |
| 512 | invoc = TestInvocation(self, test, on_completion=self.run_next_test) |
| 513 | self.invocations[test] = invoc |
| 514 | if self.visible_test is None and test.has_ui: |
| 515 | self.set_visible_test(test) |
| 516 | invoc.start() |
| 517 | |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 518 | def run_tests(self, subtrees, untested_only=False): |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 519 | ''' |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 520 | Runs tests under subtree. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 521 | |
| 522 | The tests are run in order unless one fails (then stops). |
| 523 | Backgroundable tests are run simultaneously; when a foreground test is |
| 524 | encountered, we wait for all active tests to finish before continuing. |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 525 | |
| 526 | @param subtrees: Node or nodes containing tests to run (may either be |
| 527 | a single test or a list). Duplicates will be ignored. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 528 | ''' |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 529 | if type(subtrees) != list: |
| 530 | subtrees = [subtrees] |
| 531 | |
| 532 | # Nodes we've seen so far, to avoid duplicates. |
| 533 | seen = set() |
| 534 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 535 | self.tests_to_run = deque() |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 536 | for subtree in subtrees: |
| 537 | for test in subtree.walk(): |
| 538 | if test in seen: |
| 539 | continue |
| 540 | seen.add(test) |
| 541 | |
| 542 | if not test.is_leaf(): |
| 543 | continue |
| 544 | if (untested_only and |
| 545 | test.get_state().status != TestState.UNTESTED): |
| 546 | continue |
| 547 | self.tests_to_run.append(test) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 548 | self.run_next_test() |
| 549 | |
| 550 | def reap_completed_tests(self): |
| 551 | ''' |
| 552 | Removes completed tests from the set of active tests. |
| 553 | |
| 554 | Also updates the visible test if it was reaped. |
| 555 | ''' |
| 556 | for t, v in dict(self.invocations).iteritems(): |
| 557 | if v.is_completed(): |
| 558 | del self.invocations[t] |
| 559 | |
| 560 | if (self.visible_test is None or |
| 561 | self.visible_test not in self.invocations): |
| 562 | self.set_visible_test(None) |
| 563 | # Make the first running test, if any, the visible test |
| 564 | for t in self.test_list.walk(): |
| 565 | if t in self.invocations: |
| 566 | self.set_visible_test(t) |
| 567 | break |
| 568 | |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 569 | def kill_active_tests(self, abort): |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 570 | ''' |
| 571 | Kills and waits for all active tests. |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 572 | |
| 573 | @param abort: True to change state of killed tests to FAILED, False for |
| 574 | UNTESTED. |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 575 | ''' |
| 576 | self.reap_completed_tests() |
| 577 | for test, invoc in self.invocations.items(): |
| 578 | factory.console.info('Killing active test %s...' % test.path) |
| 579 | invoc.abort_and_join() |
| 580 | factory.console.info('Killed %s' % test.path) |
| 581 | del self.invocations[test] |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 582 | if not abort: |
| 583 | test.update_state(status=TestState.UNTESTED) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 584 | self.reap_completed_tests() |
| 585 | |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 586 | def abort_active_tests(self): |
| 587 | self.kill_active_tests(True) |
| 588 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 589 | def main(self): |
Jon Salz | eb8d25f | 2012-05-22 15:17:32 +0800 | [diff] [blame] | 590 | try: |
| 591 | self.init() |
| 592 | self.event_log.AppendEvent('init', |
| 593 | success=True) |
| 594 | except: |
| 595 | if self.event_log: |
| 596 | try: |
| 597 | self.event_log.AppendEvent('init', |
| 598 | success=False, |
| 599 | trace=traceback.format_exc()) |
| 600 | except: |
| 601 | pass |
| 602 | raise |
| 603 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 604 | self.run() |
| 605 | |
| 606 | def init(self, args=None, env=None): |
| 607 | '''Initializes Goofy. |
Jon Salz | 74ad326 | 2012-03-16 14:40:55 +0800 | [diff] [blame] | 608 | |
| 609 | Args: |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 610 | args: A list of command-line arguments. Uses sys.argv if |
| 611 | args is None. |
| 612 | env: An Environment instance to use (or None to choose |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 613 | FakeChrootEnvironment or DUTEnvironment as appropriate). |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 614 | ''' |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 615 | parser = OptionParser() |
| 616 | parser.add_option('-v', '--verbose', dest='verbose', |
| 617 | action='store_true', |
| 618 | help='Enable debug logging') |
| 619 | parser.add_option('--print_test_list', dest='print_test_list', |
| 620 | metavar='FILE', |
| 621 | help='Read and print test list FILE, and exit') |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 622 | parser.add_option('--restart', dest='restart', |
| 623 | action='store_true', |
| 624 | help='Clear all test state') |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 625 | parser.add_option('--ui', dest='ui', type='choice', |
| 626 | choices=['none', 'gtk', 'chrome'], |
| 627 | default='gtk', |
| 628 | help='UI to use') |
Jon Salz | 63585ea | 2012-05-21 15:03:32 +0800 | [diff] [blame] | 629 | parser.add_option('--ui_scale_factor', dest='ui_scale_factor', |
| 630 | type='int', default=1, |
| 631 | help=('Factor by which to scale UI ' |
| 632 | '(Chrome UI only)')) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 633 | parser.add_option('--test_list', dest='test_list', |
| 634 | metavar='FILE', |
| 635 | help='Use FILE as test list') |
| 636 | (self.options, self.args) = parser.parse_args(args) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 637 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 638 | global _inited_logging |
| 639 | if not _inited_logging: |
| 640 | factory.init_logging('goofy', verbose=self.options.verbose) |
| 641 | _inited_logging = True |
Jon Salz | eb8d25f | 2012-05-22 15:17:32 +0800 | [diff] [blame] | 642 | self.event_log = EventLog('goofy') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 643 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 644 | if (not suppress_chroot_warning and |
| 645 | factory.in_chroot() and |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 646 | self.options.ui == 'gtk' and |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 647 | os.environ.get('DISPLAY') in [None, '', ':0', ':0.0']): |
| 648 | # That's not going to work! Tell the user how to run |
| 649 | # this way. |
| 650 | logging.warn(GOOFY_IN_CHROOT_WARNING) |
| 651 | time.sleep(1) |
| 652 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 653 | if env: |
| 654 | self.env = env |
| 655 | elif factory.in_chroot(): |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 656 | self.env = FakeChrootEnvironment() |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 657 | logging.warn( |
| 658 | 'Using chroot environment: will not actually run autotests') |
| 659 | else: |
| 660 | self.env = DUTEnvironment() |
Jon Salz | 323dd3d | 2012-04-09 18:40:43 +0800 | [diff] [blame] | 661 | self.env.goofy = self |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 662 | |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 663 | if self.options.restart: |
| 664 | state.clear_state() |
| 665 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 666 | if self.options.print_test_list: |
| 667 | print (factory.read_test_list(self.options.print_test_list). |
| 668 | __repr__(recursive=True)) |
| 669 | return |
| 670 | |
| 671 | logging.info('Started') |
| 672 | |
| 673 | self.start_state_server() |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 674 | self.state_instance.set_shared_data('hwid_cfg', get_hwid_cfg()) |
Jon Salz | 63585ea | 2012-05-21 15:03:32 +0800 | [diff] [blame] | 675 | self.state_instance.set_shared_data('ui_scale_factor', |
| 676 | self.options.ui_scale_factor) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 677 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 678 | self.options.test_list = (self.options.test_list or find_test_list()) |
| 679 | self.test_list = factory.read_test_list(self.options.test_list, |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 680 | self.state_instance) |
Jon Salz | 06fbeff | 2012-05-21 17:06:05 +0800 | [diff] [blame] | 681 | if not self.state_instance.has_shared_data('ui_lang'): |
| 682 | self.state_instance.set_shared_data('ui_lang', |
| 683 | self.test_list.options.ui_lang) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 684 | logging.info('TEST_LIST:\n%s', self.test_list.__repr__(recursive=True)) |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 685 | self.state_instance.test_list = self.test_list |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 686 | |
| 687 | self.init_states() |
| 688 | self.start_event_server() |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 689 | |
Jon Salz | b1b3909 | 2012-05-03 02:05:09 +0800 | [diff] [blame] | 690 | # Set CROS_UI since some behaviors in ui.py depend on the |
| 691 | # particular UI in use. TODO(jsalz): Remove this (and all |
| 692 | # places it is used) when the GTK UI is removed. |
| 693 | os.environ['CROS_UI'] = self.options.ui |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 694 | |
Jon Salz | b1b3909 | 2012-05-03 02:05:09 +0800 | [diff] [blame] | 695 | if self.options.ui == 'chrome': |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 696 | self.env.launch_chrome() |
| 697 | logging.info('Waiting for a web socket connection') |
| 698 | self.web_socket_manager.wait() |
Jon Salz | b1b3909 | 2012-05-03 02:05:09 +0800 | [diff] [blame] | 699 | |
| 700 | # Wait for the test widget size to be set; this is done in |
| 701 | # an asynchronous RPC so there is a small chance that the |
| 702 | # web socket might be opened first. |
| 703 | for i in range(100): # 10 s |
Jon Salz | 63585ea | 2012-05-21 15:03:32 +0800 | [diff] [blame] | 704 | try: |
| 705 | if self.state_instance.get_shared_data('test_widget_size'): |
| 706 | break |
| 707 | except KeyError: |
| 708 | pass # Retry |
Jon Salz | b1b3909 | 2012-05-03 02:05:09 +0800 | [diff] [blame] | 709 | time.sleep(0.1) # 100 ms |
| 710 | else: |
| 711 | logging.warn('Never received test_widget_size from UI') |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 712 | elif self.options.ui == 'gtk': |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 713 | self.start_ui() |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 714 | |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 715 | self.prespawner = Prespawner() |
Jon Salz | 323dd3d | 2012-04-09 18:40:43 +0800 | [diff] [blame] | 716 | self.prespawner.start() |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 717 | |
| 718 | def state_change_callback(test, state): |
| 719 | self.event_client.post_event( |
| 720 | Event(Event.Type.STATE_CHANGE, |
| 721 | path=test.path, state=state)) |
| 722 | self.test_list.state_change_callback = state_change_callback |
| 723 | |
| 724 | try: |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 725 | tests_after_shutdown = self.state_instance.get_shared_data( |
| 726 | 'tests_after_shutdown') |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 727 | except KeyError: |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 728 | tests_after_shutdown = None |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 729 | |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 730 | if tests_after_shutdown is not None: |
| 731 | logging.info('Resuming tests after shutdown: %s', |
| 732 | tests_after_shutdown) |
| 733 | self.state_instance.set_shared_data('tests_after_shutdown', None) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 734 | self.tests_to_run.extend( |
Jon Salz | 758e6cc | 2012-04-03 15:47:07 +0800 | [diff] [blame] | 735 | self.test_list.lookup_path(t) for t in tests_after_shutdown) |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 736 | self.run_queue.put(self.run_next_test) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 737 | else: |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 738 | if self.test_list.options.auto_run_on_start: |
| 739 | self.run_queue.put( |
| 740 | lambda: self.run_tests(self.test_list, untested_only=True)) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 741 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 742 | def run(self): |
| 743 | '''Runs Goofy.''' |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 744 | # Process events forever. |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 745 | while self.run_once(True): |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 746 | pass |
| 747 | |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 748 | def run_once(self, block=False): |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 749 | '''Runs all items pending in the event loop. |
| 750 | |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 751 | Args: |
| 752 | block: If true, block until at least one event is processed. |
| 753 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 754 | Returns: |
| 755 | True to keep going or False to shut down. |
| 756 | ''' |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 757 | events = [] |
| 758 | if block: |
| 759 | # Get at least one event |
| 760 | events.append(self.run_queue.get()) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 761 | while True: |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 762 | try: |
| 763 | events.append(self.run_queue.get_nowait()) |
| 764 | except Queue.Empty: |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 765 | break |
| 766 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 767 | for event in events: |
| 768 | if not event: |
| 769 | # Shutdown request. |
| 770 | self.run_queue.task_done() |
| 771 | return False |
| 772 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 773 | try: |
| 774 | event() |
| 775 | except Exception as e: # pylint: disable=W0703 |
| 776 | logging.error('Error in event loop: %s', e) |
| 777 | traceback.print_exc(sys.stderr) |
Jon Salz | 8375c2e | 2012-04-04 15:22:24 +0800 | [diff] [blame] | 778 | self.record_exception(traceback.format_exception_only( |
| 779 | *sys.exc_info()[:2])) |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 780 | # But keep going |
| 781 | finally: |
| 782 | self.run_queue.task_done() |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 783 | return True |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 784 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 785 | def run_tests_with_status(self, statuses_to_run, starting_at=None, |
| 786 | root=None): |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 787 | '''Runs all top-level tests with a particular status. |
| 788 | |
| 789 | All active tests, plus any tests to re-run, are reset. |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 790 | |
| 791 | Args: |
| 792 | starting_at: If provided, only auto-runs tests beginning with |
| 793 | this test. |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 794 | ''' |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 795 | root = root or self.test_list |
| 796 | |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 797 | if starting_at: |
| 798 | # Make sure they passed a test, not a string. |
| 799 | assert isinstance(starting_at, factory.FactoryTest) |
| 800 | |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 801 | tests_to_reset = [] |
| 802 | tests_to_run = [] |
| 803 | |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 804 | found_starting_at = False |
| 805 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 806 | for test in root.get_top_level_tests(): |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 807 | if starting_at: |
| 808 | if test == starting_at: |
| 809 | # We've found starting_at; do auto-run on all |
| 810 | # subsequent tests. |
| 811 | found_starting_at = True |
| 812 | if not found_starting_at: |
| 813 | # Don't start this guy yet |
| 814 | continue |
| 815 | |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 816 | status = test.get_state().status |
| 817 | if status == TestState.ACTIVE or status in statuses_to_run: |
| 818 | # Reset the test (later; we will need to abort |
| 819 | # all active tests first). |
| 820 | tests_to_reset.append(test) |
| 821 | if status in statuses_to_run: |
| 822 | tests_to_run.append(test) |
| 823 | |
| 824 | self.abort_active_tests() |
| 825 | |
| 826 | # Reset all statuses of the tests to run (in case any tests were active; |
| 827 | # we want them to be run again). |
| 828 | for test_to_reset in tests_to_reset: |
| 829 | for test in test_to_reset.walk(): |
| 830 | test.update_state(status=TestState.UNTESTED) |
| 831 | |
| 832 | self.run_tests(tests_to_run, untested_only=True) |
| 833 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 834 | def restart_tests(self, root=None): |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 835 | '''Restarts all tests.''' |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 836 | root = root or self.test_list |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 837 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 838 | self.abort_active_tests() |
| 839 | for test in root.walk(): |
| 840 | test.update_state(status=TestState.UNTESTED) |
| 841 | self.run_tests(root) |
| 842 | |
| 843 | def auto_run(self, starting_at=None, root=None): |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 844 | '''"Auto-runs" tests that have not been run yet. |
| 845 | |
| 846 | Args: |
| 847 | starting_at: If provide, only auto-runs tests beginning with |
| 848 | this test. |
| 849 | ''' |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 850 | root = root or self.test_list |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 851 | self.run_tests_with_status([TestState.UNTESTED, TestState.ACTIVE], |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 852 | starting_at=starting_at, |
| 853 | root=root) |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 854 | |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 855 | def re_run_failed(self, root=None): |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 856 | '''Re-runs failed tests.''' |
Jon Salz | 258a40c | 2012-04-19 12:34:01 +0800 | [diff] [blame] | 857 | root = root or self.test_list |
| 858 | self.run_tests_with_status([TestState.FAILED], root=root) |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 859 | |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 860 | def show_review_information(self): |
Hung-Te Lin | 9663236 | 2012-03-20 21:14:18 +0800 | [diff] [blame] | 861 | '''Event handler for showing review information screen. |
| 862 | |
| 863 | The information screene is rendered by main UI program (ui.py), so in |
| 864 | goofy we only need to kill all active tests, set them as untested, and |
| 865 | clear remaining tests. |
| 866 | ''' |
| 867 | self.kill_active_tests(False) |
| 868 | self.run_tests([]) |
| 869 | |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 870 | def handle_switch_test(self, event): |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 871 | '''Switches to a particular test. |
| 872 | |
| 873 | @param event: The SWITCH_TEST event. |
| 874 | ''' |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 875 | test = self.test_list.lookup_path(event.path) |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 876 | if not test: |
Jon Salz | 968e90b | 2012-03-18 16:12:43 +0800 | [diff] [blame] | 877 | logging.error('Unknown test %r', event.key) |
Jon Salz | 57717ca | 2012-04-04 16:47:25 +0800 | [diff] [blame] | 878 | return |
| 879 | |
| 880 | invoc = self.invocations.get(test) |
| 881 | if invoc and test.backgroundable: |
| 882 | # Already running: just bring to the front if it |
| 883 | # has a UI. |
| 884 | logging.info('Setting visible test to %s', test.path) |
| 885 | self.event_client.post_event( |
| 886 | Event(Event.Type.SET_VISIBLE_TEST, path=test.path)) |
| 887 | return |
| 888 | |
| 889 | self.abort_active_tests() |
| 890 | for t in test.walk(): |
| 891 | t.update_state(status=TestState.UNTESTED) |
| 892 | |
| 893 | if self.test_list.options.auto_run_on_keypress: |
| 894 | self.auto_run(starting_at=test) |
| 895 | else: |
| 896 | self.run_tests(test) |
Jon Salz | 0405ab5 | 2012-03-16 15:26:52 +0800 | [diff] [blame] | 897 | |
Jon Salz | 73e0fd0 | 2012-04-04 11:46:38 +0800 | [diff] [blame] | 898 | def wait(self): |
| 899 | '''Waits for all pending invocations. |
| 900 | |
| 901 | Useful for testing. |
| 902 | ''' |
| 903 | for k, v in self.invocations.iteritems(): |
| 904 | logging.info('Waiting for %s to complete...', k) |
| 905 | v.thread.join() |
| 906 | |
| 907 | def check_exceptions(self): |
| 908 | '''Raises an error if any exceptions have occurred in |
| 909 | invocation threads.''' |
| 910 | if self.exceptions: |
| 911 | raise RuntimeError('Exception in invocation thread: %r' % |
| 912 | self.exceptions) |
| 913 | |
| 914 | def record_exception(self, msg): |
| 915 | '''Records an exception in an invocation thread. |
| 916 | |
| 917 | An exception with the given message will be rethrown when |
| 918 | Goofy is destroyed.''' |
| 919 | self.exceptions.append(msg) |
| 920 | |
Hung-Te Lin | f2f78f7 | 2012-02-08 19:27:11 +0800 | [diff] [blame] | 921 | |
| 922 | if __name__ == '__main__': |
| 923 | Goofy().main() |