blob: ac14cf7ef2f38f4f60e777d45f6feb056978e6ba [file] [log] [blame]
Hung-Te Linf2f78f72012-02-08 19:27:11 +08001#!/usr/bin/python -u
2#
3# -*- coding: utf-8 -*-
4#
5# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
6# Use of this source code is governed by a BSD-style license that can be
7# found in the LICENSE file.
8
9'''
10The main factory flow that runs the factory test and finalizes a device.
11'''
12
Jon Salz73e0fd02012-04-04 11:46:38 +080013import array
14import fcntl
15import glob
Jon Salz258a40c2012-04-19 12:34:01 +080016import hashlib
Jon Salz0405ab52012-03-16 15:26:52 +080017import logging
18import os
Jon Salz258a40c2012-04-19 12:34:01 +080019import cPickle as pickle
Jon Salz0405ab52012-03-16 15:26:52 +080020import pipes
Jon Salz73e0fd02012-04-04 11:46:38 +080021import Queue
Jon Salz0405ab52012-03-16 15:26:52 +080022import re
23import signal
24import subprocess
25import sys
26import tempfile
27import threading
28import time
29import traceback
Jon Salz258a40c2012-04-19 12:34:01 +080030import unittest
31import uuid
Hung-Te Linf2f78f72012-02-08 19:27:11 +080032from collections import deque
33from optparse import OptionParser
Jon Salz258a40c2012-04-19 12:34:01 +080034from StringIO import StringIO
Hung-Te Linf2f78f72012-02-08 19:27:11 +080035
36import factory_common
Jon Salz8375c2e2012-04-04 15:22:24 +080037from autotest_lib.client.bin.prespawner import Prespawner
Hung-Te Linf2f78f72012-02-08 19:27:11 +080038from autotest_lib.client.cros import factory
39from autotest_lib.client.cros.factory import state
40from autotest_lib.client.cros.factory import TestState
Jon Salz258a40c2012-04-19 12:34:01 +080041from autotest_lib.client.cros.factory import utils
Hung-Te Linf2f78f72012-02-08 19:27:11 +080042from autotest_lib.client.cros.factory.event import Event
43from autotest_lib.client.cros.factory.event import EventClient
44from autotest_lib.client.cros.factory.event import EventServer
Jon Salzeb8d25f2012-05-22 15:17:32 +080045from autotest_lib.client.cros.factory.event_log import EventLog
Jon Salz258a40c2012-04-19 12:34:01 +080046from autotest_lib.client.cros.factory.invocation import TestInvocation
47from autotest_lib.client.cros.factory.web_socket_manager import WebSocketManager
Hung-Te Linf2f78f72012-02-08 19:27:11 +080048
49
Hung-Te Linf2f78f72012-02-08 19:27:11 +080050DEFAULT_TEST_LIST_PATH = os.path.join(
Jon Salz258a40c2012-04-19 12:34:01 +080051 factory.CLIENT_PATH , 'site_tests', 'suite_Factory', 'test_list')
Hung-Te Linf2f78f72012-02-08 19:27:11 +080052HWID_CFG_PATH = '/usr/local/share/chromeos-hwid/cfg'
53
Jon Salz758e6cc2012-04-03 15:47:07 +080054GOOFY_IN_CHROOT_WARNING = '\n' + ('*' * 70) + '''
55You are running Goofy inside the chroot. Autotests are not supported.
56
57To use Goofy in the chroot, first install an Xvnc server:
58
59 sudo apt-get install tightvncserver
60
61...and then start a VNC X server outside the chroot:
62
63 vncserver :10 &
64 vncviewer :10
65
66...and run Goofy as follows:
67
68 env --unset=XAUTHORITY DISPLAY=localhost:10 python goofy.py
69''' + ('*' * 70)
Jon Salz73e0fd02012-04-04 11:46:38 +080070suppress_chroot_warning = False
Hung-Te Linf2f78f72012-02-08 19:27:11 +080071
72def get_hwid_cfg():
73 '''
74 Returns the HWID config tag, or an empty string if none can be found.
75 '''
76 if 'CROS_HWID' in os.environ:
77 return os.environ['CROS_HWID']
78 if os.path.exists(HWID_CFG_PATH):
79 with open(HWID_CFG_PATH, 'rt') as hwid_cfg_handle:
80 return hwid_cfg_handle.read().strip()
81 return ''
82
83
84def find_test_list():
85 '''
86 Returns the path to the active test list, based on the HWID config tag.
87 '''
88 hwid_cfg = get_hwid_cfg()
89
90 # Try in order: test_list, test_list.$hwid_cfg, test_list.all
91 if hwid_cfg:
92 test_list = '%s_%s' % (DEFAULT_TEST_LIST_PATH, hwid_cfg)
93 if os.path.exists(test_list):
94 logging.info('Using special test list: %s', test_list)
95 return test_list
96 logging.info('WARNING: no specific test list for config: %s', hwid_cfg)
97
98 test_list = DEFAULT_TEST_LIST_PATH
99 if os.path.exists(test_list):
100 return test_list
101
102 test_list = ('%s.all' % DEFAULT_TEST_LIST_PATH)
103 if os.path.exists(test_list):
104 logging.info('Using default test list: ' + test_list)
105 return test_list
106 logging.info('ERROR: Cannot find any test list.')
107
Jon Salz73e0fd02012-04-04 11:46:38 +0800108
109class Environment(object):
110 '''
111 Abstract base class for external test operations, e.g., run an autotest,
112 shutdown, or reboot.
113
114 The Environment is assumed not to be thread-safe: callers must grab the lock
115 before calling any methods. This is primarily necessary because we mock out
116 this Environment with mox, and unfortunately mox is not thread-safe.
117 TODO(jsalz): Try to write a thread-safe wrapper for mox.
118 '''
119 lock = threading.Lock()
120
121 def shutdown(self, operation):
122 '''
123 Shuts the machine down (from a ShutdownStep).
124
125 Args:
126 operation: 'reboot' or 'halt'.
127
128 Returns:
129 True if Goofy should gracefully exit, or False if Goofy
130 should just consider the shutdown to have suceeded (e.g.,
131 in the chroot).
132 '''
133 raise NotImplementedError()
134
Jon Salz258a40c2012-04-19 12:34:01 +0800135 def launch_chrome(self):
136 '''
137 Launches Chrome.
138
139 Returns:
140 The Chrome subprocess (or None if none).
141 '''
142 raise NotImplementedError()
Jon Salz73e0fd02012-04-04 11:46:38 +0800143
144 def spawn_autotest(self, name, args, env_additions, result_file):
145 '''
146 Spawns a process to run an autotest.
147
148 Args:
149 name: Name of the autotest to spawn.
150 args: Command-line arguments.
151 env_additions: Additions to the environment.
152 result_file: Expected location of the result file.
153 '''
154 raise NotImplementedError()
155
156
157class DUTEnvironment(Environment):
158 '''
159 A real environment on a device under test.
160 '''
161 def shutdown(self, operation):
162 assert operation in ['reboot', 'halt']
163 logging.info('Shutting down: %s', operation)
164 subprocess.check_call('sync')
165 subprocess.check_call(operation)
166 time.sleep(30)
167 assert False, 'Never reached (should %s)' % operation
168
169 def spawn_autotest(self, name, args, env_additions, result_file):
Jon Salz8375c2e2012-04-04 15:22:24 +0800170 return self.goofy.prespawner.spawn(args, env_additions)
Jon Salz73e0fd02012-04-04 11:46:38 +0800171
Jon Salz258a40c2012-04-19 12:34:01 +0800172 def launch_chrome(self):
173 chrome_command = [
174 '/opt/google/chrome/chrome',
175 '--user-data-dir=%s/factory-chrome-datadir' %
176 factory.get_log_root(),
177 '--aura-host-window-use-fullscreen',
178 '--kiosk',
Jon Salz63585ea2012-05-21 15:03:32 +0800179 ('--default-device-scale-factor=%d' %
180 self.goofy.options.ui_scale_factor),
Jon Salz258a40c2012-04-19 12:34:01 +0800181 'http://localhost:%d/' % state.DEFAULT_FACTORY_STATE_PORT,
182 ]
Jon Salz73e0fd02012-04-04 11:46:38 +0800183
Jon Salz258a40c2012-04-19 12:34:01 +0800184 chrome_log = os.path.join(factory.get_log_root(), 'factory.chrome.log')
185 chrome_log_file = open(chrome_log, "a")
186 logging.info('Launching Chrome; logs in %s' % chrome_log)
187 return subprocess.Popen(chrome_command,
188 stdout=chrome_log_file,
189 stderr=subprocess.STDOUT)
190
191class FakeChrootEnvironment(Environment):
Jon Salz73e0fd02012-04-04 11:46:38 +0800192 '''
193 A chroot environment that doesn't actually shutdown or run autotests.
194 '''
195 def shutdown(self, operation):
196 assert operation in ['reboot', 'halt']
197 logging.warn('In chroot: skipping %s', operation)
198 return False
199
200 def spawn_autotest(self, name, args, env_additions, result_file):
201 logging.warn('In chroot: skipping autotest %s', name)
Jon Salz258a40c2012-04-19 12:34:01 +0800202 # Mark it as passed with 75% probability, or failed with 25%
203 # probability (depending on a hash of the autotest name).
204 pseudo_random = ord(hashlib.sha1(name).digest()[0]) / 256.0
205 passed = pseudo_random > .25
206
Jon Salz73e0fd02012-04-04 11:46:38 +0800207 with open(result_file, 'w') as out:
Jon Salz258a40c2012-04-19 12:34:01 +0800208 pickle.dump((passed, '' if passed else 'Simulated failure'), out)
Jon Salz73e0fd02012-04-04 11:46:38 +0800209 # Start a process that will return with a true exit status in
210 # 2 seconds (just like a happy autotest).
211 return subprocess.Popen(['sleep', '2'])
212
Jon Salz258a40c2012-04-19 12:34:01 +0800213 def launch_chrome(self):
214 logging.warn('In chroot; not launching Chrome. '
215 'Please open http://localhost:%d/ in Chrome.',
216 state.DEFAULT_FACTORY_STATE_PORT)
Jon Salz73e0fd02012-04-04 11:46:38 +0800217
218
219_inited_logging = False
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800220
221class Goofy(object):
222 '''
223 The main factory flow.
224
225 Note that all methods in this class must be invoked from the main
226 (event) thread. Other threads, such as callbacks and TestInvocation
227 methods, should instead post events on the run queue.
228
229 TODO: Unit tests. (chrome-os-partner:7409)
230
231 Properties:
Jon Salz258a40c2012-04-19 12:34:01 +0800232 uuid: A unique UUID for this invocation of Goofy.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800233 state_instance: An instance of FactoryState.
234 state_server: The FactoryState XML/RPC server.
235 state_server_thread: A thread running state_server.
236 event_server: The EventServer socket server.
237 event_server_thread: A thread running event_server.
238 event_client: A client to the event server.
Hung-Te Lin6bb48552012-02-09 14:37:43 +0800239 ui_process: The factory ui process object.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800240 run_queue: A queue of callbacks to invoke from the main thread.
241 invocations: A map from FactoryTest objects to the corresponding
242 TestInvocations objects representing active tests.
243 tests_to_run: A deque of tests that should be run when the current
244 test(s) complete.
245 options: Command-line options.
246 args: Command-line args.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800247 test_list: The test list.
Jon Salz0405ab52012-03-16 15:26:52 +0800248 event_handlers: Map of Event.Type to the method used to handle that
249 event. If the method has an 'event' argument, the event is passed
250 to the handler.
Jon Salz73e0fd02012-04-04 11:46:38 +0800251 exceptions: Exceptions encountered in invocation threads.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800252 '''
253 def __init__(self):
Jon Salz258a40c2012-04-19 12:34:01 +0800254 self.uuid = str(uuid.uuid4())
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800255 self.state_instance = None
256 self.state_server = None
257 self.state_server_thread = None
258 self.event_server = None
259 self.event_server_thread = None
260 self.event_client = None
Jon Salzeb8d25f2012-05-22 15:17:32 +0800261 self.event_log = None
Jon Salz8375c2e2012-04-04 15:22:24 +0800262 self.prespawner = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800263 self.ui_process = None
Jon Salz73e0fd02012-04-04 11:46:38 +0800264 self.run_queue = Queue.Queue()
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800265 self.invocations = {}
266 self.tests_to_run = deque()
267 self.visible_test = None
Jon Salz258a40c2012-04-19 12:34:01 +0800268 self.chrome = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800269
270 self.options = None
271 self.args = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800272 self.test_list = None
273
Jon Salz258a40c2012-04-19 12:34:01 +0800274 def test_or_root(event):
275 '''Returns the top-level parent for a test (the root node of the
276 tests that need to be run together if the given test path is to
277 be run).'''
278 try:
279 path = event.path
Jon Salzd2ed6cb2012-05-02 09:35:14 +0800280 except AttributeError:
Jon Salz258a40c2012-04-19 12:34:01 +0800281 path = None
282
283 if path:
284 return self.test_list.lookup_path(path).get_top_level_parent()
285 else:
286 return self.test_list
287
Jon Salz0405ab52012-03-16 15:26:52 +0800288 self.event_handlers = {
289 Event.Type.SWITCH_TEST: self.handle_switch_test,
Jon Salz968e90b2012-03-18 16:12:43 +0800290 Event.Type.SHOW_NEXT_ACTIVE_TEST:
291 lambda event: self.show_next_active_test(),
292 Event.Type.RESTART_TESTS:
Jon Salz258a40c2012-04-19 12:34:01 +0800293 lambda event: self.restart_tests(root=test_or_root(event)),
Jon Salz968e90b2012-03-18 16:12:43 +0800294 Event.Type.AUTO_RUN:
Jon Salz258a40c2012-04-19 12:34:01 +0800295 lambda event: self.auto_run(root=test_or_root(event)),
Jon Salz968e90b2012-03-18 16:12:43 +0800296 Event.Type.RE_RUN_FAILED:
Jon Salz258a40c2012-04-19 12:34:01 +0800297 lambda event: self.re_run_failed(root=test_or_root(event)),
Jon Salz968e90b2012-03-18 16:12:43 +0800298 Event.Type.REVIEW:
299 lambda event: self.show_review_information(),
Jon Salz0405ab52012-03-16 15:26:52 +0800300 }
301
Jon Salz73e0fd02012-04-04 11:46:38 +0800302 self.exceptions = []
Jon Salz258a40c2012-04-19 12:34:01 +0800303 self.web_socket_manager = None
Jon Salz73e0fd02012-04-04 11:46:38 +0800304
305 def destroy(self):
Jon Salz258a40c2012-04-19 12:34:01 +0800306 if self.chrome:
307 self.chrome.kill()
308 self.chrome = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800309 if self.ui_process:
Jon Salz258a40c2012-04-19 12:34:01 +0800310 utils.kill_process_tree(self.ui_process, 'ui')
Jon Salz73e0fd02012-04-04 11:46:38 +0800311 self.ui_process = None
Jon Salz258a40c2012-04-19 12:34:01 +0800312 if self.web_socket_manager:
313 logging.info('Stopping web sockets')
314 self.web_socket_manager.close()
315 self.web_socket_manager = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800316 if self.state_server_thread:
317 logging.info('Stopping state server')
318 self.state_server.shutdown()
319 self.state_server_thread.join()
Jon Salz73e0fd02012-04-04 11:46:38 +0800320 self.state_server.server_close()
321 self.state_server_thread = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800322 if self.event_server_thread:
323 logging.info('Stopping event server')
324 self.event_server.shutdown() # pylint: disable=E1101
325 self.event_server_thread.join()
Jon Salz73e0fd02012-04-04 11:46:38 +0800326 self.event_server.server_close()
327 self.event_server_thread = None
Jon Salz8375c2e2012-04-04 15:22:24 +0800328 if self.prespawner:
329 logging.info('Stopping prespawner')
330 self.prespawner.stop()
331 self.prespawner = None
332 if self.event_client:
Jon Salz258a40c2012-04-19 12:34:01 +0800333 logging.info('Closing event client')
Jon Salz8375c2e2012-04-04 15:22:24 +0800334 self.event_client.close()
Jon Salz258a40c2012-04-19 12:34:01 +0800335 self.event_client = None
Jon Salzeb8d25f2012-05-22 15:17:32 +0800336 if self.event_log:
337 self.event_log.Close()
338 self.event_log = None
Jon Salz73e0fd02012-04-04 11:46:38 +0800339 self.check_exceptions()
Jon Salz258a40c2012-04-19 12:34:01 +0800340 logging.info('Done destroying Goofy')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800341
342 def start_state_server(self):
Jon Salz258a40c2012-04-19 12:34:01 +0800343 self.state_instance, self.state_server = (
344 state.create_server(bind_address='0.0.0.0'))
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800345 logging.info('Starting state server')
346 self.state_server_thread = threading.Thread(
Jon Salz8375c2e2012-04-04 15:22:24 +0800347 target=self.state_server.serve_forever,
348 name='StateServer')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800349 self.state_server_thread.start()
350
351 def start_event_server(self):
352 self.event_server = EventServer()
353 logging.info('Starting factory event server')
354 self.event_server_thread = threading.Thread(
Jon Salz8375c2e2012-04-04 15:22:24 +0800355 target=self.event_server.serve_forever,
356 name='EventServer') # pylint: disable=E1101
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800357 self.event_server_thread.start()
358
359 self.event_client = EventClient(
360 callback=self.handle_event, event_loop=self.run_queue)
361
Jon Salz258a40c2012-04-19 12:34:01 +0800362 self.web_socket_manager = WebSocketManager(self.uuid)
363 self.state_server.add_handler("/event",
364 self.web_socket_manager.handle_web_socket)
365
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800366 def start_ui(self):
Jon Salz258a40c2012-04-19 12:34:01 +0800367 ui_proc_args = [os.path.join(factory.CROS_FACTORY_LIB_PATH, 'ui'),
368 self.options.test_list]
Jon Salz14bcbb02012-03-17 15:11:50 +0800369 if self.options.verbose:
370 ui_proc_args.append('-v')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800371 logging.info('Starting ui %s', ui_proc_args)
372 self.ui_process = subprocess.Popen(ui_proc_args)
373 logging.info('Waiting for UI to come up...')
374 self.event_client.wait(
375 lambda event: event.type == Event.Type.UI_READY)
376 logging.info('UI has started')
377
378 def set_visible_test(self, test):
379 if self.visible_test == test:
380 return
381
382 if test:
383 test.update_state(visible=True)
384 if self.visible_test:
385 self.visible_test.update_state(visible=False)
386 self.visible_test = test
387
Jon Salz74ad3262012-03-16 14:40:55 +0800388 def handle_shutdown_complete(self, test, state):
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800389 '''
Jon Salz74ad3262012-03-16 14:40:55 +0800390 Handles the case where a shutdown was detected during a shutdown step.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800391
Jon Salz74ad3262012-03-16 14:40:55 +0800392 @param test: The ShutdownStep.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800393 @param state: The test state.
394 '''
Jon Salz74ad3262012-03-16 14:40:55 +0800395 state = test.update_state(increment_shutdown_count=1)
396 logging.info('Detected shutdown (%d of %d)',
397 state.shutdown_count, test.iterations)
398 if state.shutdown_count == test.iterations:
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800399 # Good!
400 test.update_state(status=TestState.PASSED, error_msg='')
Jon Salz74ad3262012-03-16 14:40:55 +0800401 elif state.shutdown_count > test.iterations:
Jon Salz73e0fd02012-04-04 11:46:38 +0800402 # Shut down too many times
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800403 test.update_state(status=TestState.FAILED,
Jon Salz74ad3262012-03-16 14:40:55 +0800404 error_msg='Too many shutdowns')
Jon Salz258a40c2012-04-19 12:34:01 +0800405 elif utils.are_shift_keys_depressed():
Jon Salz73e0fd02012-04-04 11:46:38 +0800406 logging.info('Shift keys are depressed; cancelling restarts')
407 # Abort shutdown
408 test.update_state(
409 status=TestState.FAILED,
410 error_msg='Shutdown aborted with double shift keys')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800411 else:
Jon Salz74ad3262012-03-16 14:40:55 +0800412 # Need to shutdown again
Jon Salz73e0fd02012-04-04 11:46:38 +0800413 self.env.shutdown('reboot')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800414
415 def init_states(self):
416 '''
417 Initializes all states on startup.
418 '''
419 for test in self.test_list.get_all_tests():
420 # Make sure the state server knows about all the tests,
421 # defaulting to an untested state.
422 test.update_state(update_parent=False, visible=False)
423
424 # Any 'active' tests should be marked as failed now.
425 for test in self.test_list.walk():
426 state = test.get_state()
Hung-Te Lin96632362012-03-20 21:14:18 +0800427 if state.status != TestState.ACTIVE:
428 continue
429 if isinstance(test, factory.ShutdownStep):
430 # Shutdown while the test was active - that's good.
431 self.handle_shutdown_complete(test, state)
432 else:
433 test.update_state(status=TestState.FAILED,
434 error_msg='Unknown (shutdown?)')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800435
436 def show_next_active_test(self):
437 '''
438 Rotates to the next visible active test.
439 '''
440 self.reap_completed_tests()
441 active_tests = [
442 t for t in self.test_list.walk()
443 if t.is_leaf() and t.get_state().status == TestState.ACTIVE]
444 if not active_tests:
445 return
446
447 try:
448 next_test = active_tests[
449 (active_tests.index(self.visible_test) + 1) % len(active_tests)]
450 except ValueError: # visible_test not present in active_tests
451 next_test = active_tests[0]
452
453 self.set_visible_test(next_test)
454
455 def handle_event(self, event):
456 '''
457 Handles an event from the event server.
458 '''
Jon Salz0405ab52012-03-16 15:26:52 +0800459 handler = self.event_handlers.get(event.type)
460 if handler:
Jon Salz968e90b2012-03-18 16:12:43 +0800461 handler(event)
Jon Salz0405ab52012-03-16 15:26:52 +0800462 else:
Jon Salz968e90b2012-03-18 16:12:43 +0800463 # We don't register handlers for all event types - just ignore
464 # this event.
465 logging.debug('Unbound event type %s', event.type)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800466
467 def run_next_test(self):
468 '''
469 Runs the next eligible test (or tests) in self.tests_to_run.
470 '''
471 self.reap_completed_tests()
472 while self.tests_to_run:
473 logging.debug('Tests to run: %s',
474 [x.path for x in self.tests_to_run])
475
476 test = self.tests_to_run[0]
477
478 if test in self.invocations:
479 logging.info('Next test %s is already running', test.path)
480 self.tests_to_run.popleft()
481 return
482
483 if self.invocations and not (test.backgroundable and all(
484 [x.backgroundable for x in self.invocations])):
485 logging.debug('Waiting for non-backgroundable tests to '
486 'complete before running %s', test.path)
487 return
488
489 self.tests_to_run.popleft()
490
Jon Salz74ad3262012-03-16 14:40:55 +0800491 if isinstance(test, factory.ShutdownStep):
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800492 test.update_state(status=TestState.ACTIVE, increment_count=1,
Jon Salz74ad3262012-03-16 14:40:55 +0800493 error_msg='', shutdown_count=0)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800494 # Save pending test list in the state server
495 self.state_instance.set_shared_data(
Jon Salz74ad3262012-03-16 14:40:55 +0800496 'tests_after_shutdown',
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800497 [t.path for t in self.tests_to_run])
Jon Salz74ad3262012-03-16 14:40:55 +0800498
Jon Salz73e0fd02012-04-04 11:46:38 +0800499 with self.env.lock:
500 shutdown_result = self.env.shutdown(test.operation)
501 if shutdown_result:
502 # That's all, folks!
503 self.run_queue.put(None)
504 return
505 else:
506 # Just pass (e.g., in the chroot).
507 test.update_state(status=TestState.PASSED)
508 self.state_instance.set_shared_data(
509 'tests_after_shutdown', None)
510 continue
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800511
512 invoc = TestInvocation(self, test, on_completion=self.run_next_test)
513 self.invocations[test] = invoc
514 if self.visible_test is None and test.has_ui:
515 self.set_visible_test(test)
516 invoc.start()
517
Jon Salz0405ab52012-03-16 15:26:52 +0800518 def run_tests(self, subtrees, untested_only=False):
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800519 '''
Jon Salz0405ab52012-03-16 15:26:52 +0800520 Runs tests under subtree.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800521
522 The tests are run in order unless one fails (then stops).
523 Backgroundable tests are run simultaneously; when a foreground test is
524 encountered, we wait for all active tests to finish before continuing.
Jon Salz0405ab52012-03-16 15:26:52 +0800525
526 @param subtrees: Node or nodes containing tests to run (may either be
527 a single test or a list). Duplicates will be ignored.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800528 '''
Jon Salz0405ab52012-03-16 15:26:52 +0800529 if type(subtrees) != list:
530 subtrees = [subtrees]
531
532 # Nodes we've seen so far, to avoid duplicates.
533 seen = set()
534
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800535 self.tests_to_run = deque()
Jon Salz0405ab52012-03-16 15:26:52 +0800536 for subtree in subtrees:
537 for test in subtree.walk():
538 if test in seen:
539 continue
540 seen.add(test)
541
542 if not test.is_leaf():
543 continue
544 if (untested_only and
545 test.get_state().status != TestState.UNTESTED):
546 continue
547 self.tests_to_run.append(test)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800548 self.run_next_test()
549
550 def reap_completed_tests(self):
551 '''
552 Removes completed tests from the set of active tests.
553
554 Also updates the visible test if it was reaped.
555 '''
556 for t, v in dict(self.invocations).iteritems():
557 if v.is_completed():
558 del self.invocations[t]
559
560 if (self.visible_test is None or
561 self.visible_test not in self.invocations):
562 self.set_visible_test(None)
563 # Make the first running test, if any, the visible test
564 for t in self.test_list.walk():
565 if t in self.invocations:
566 self.set_visible_test(t)
567 break
568
Hung-Te Lin96632362012-03-20 21:14:18 +0800569 def kill_active_tests(self, abort):
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800570 '''
571 Kills and waits for all active tests.
Hung-Te Lin96632362012-03-20 21:14:18 +0800572
573 @param abort: True to change state of killed tests to FAILED, False for
574 UNTESTED.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800575 '''
576 self.reap_completed_tests()
577 for test, invoc in self.invocations.items():
578 factory.console.info('Killing active test %s...' % test.path)
579 invoc.abort_and_join()
580 factory.console.info('Killed %s' % test.path)
581 del self.invocations[test]
Hung-Te Lin96632362012-03-20 21:14:18 +0800582 if not abort:
583 test.update_state(status=TestState.UNTESTED)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800584 self.reap_completed_tests()
585
Hung-Te Lin96632362012-03-20 21:14:18 +0800586 def abort_active_tests(self):
587 self.kill_active_tests(True)
588
Jon Salz73e0fd02012-04-04 11:46:38 +0800589 def main(self):
Jon Salzeb8d25f2012-05-22 15:17:32 +0800590 try:
591 self.init()
592 self.event_log.AppendEvent('init',
593 success=True)
594 except:
595 if self.event_log:
596 try:
597 self.event_log.AppendEvent('init',
598 success=False,
599 trace=traceback.format_exc())
600 except:
601 pass
602 raise
603
Jon Salz73e0fd02012-04-04 11:46:38 +0800604 self.run()
605
606 def init(self, args=None, env=None):
607 '''Initializes Goofy.
Jon Salz74ad3262012-03-16 14:40:55 +0800608
609 Args:
Jon Salz73e0fd02012-04-04 11:46:38 +0800610 args: A list of command-line arguments. Uses sys.argv if
611 args is None.
612 env: An Environment instance to use (or None to choose
Jon Salz258a40c2012-04-19 12:34:01 +0800613 FakeChrootEnvironment or DUTEnvironment as appropriate).
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800614 '''
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800615 parser = OptionParser()
616 parser.add_option('-v', '--verbose', dest='verbose',
617 action='store_true',
618 help='Enable debug logging')
619 parser.add_option('--print_test_list', dest='print_test_list',
620 metavar='FILE',
621 help='Read and print test list FILE, and exit')
Jon Salz758e6cc2012-04-03 15:47:07 +0800622 parser.add_option('--restart', dest='restart',
623 action='store_true',
624 help='Clear all test state')
Jon Salz258a40c2012-04-19 12:34:01 +0800625 parser.add_option('--ui', dest='ui', type='choice',
626 choices=['none', 'gtk', 'chrome'],
627 default='gtk',
628 help='UI to use')
Jon Salz63585ea2012-05-21 15:03:32 +0800629 parser.add_option('--ui_scale_factor', dest='ui_scale_factor',
630 type='int', default=1,
631 help=('Factor by which to scale UI '
632 '(Chrome UI only)'))
Jon Salz73e0fd02012-04-04 11:46:38 +0800633 parser.add_option('--test_list', dest='test_list',
634 metavar='FILE',
635 help='Use FILE as test list')
636 (self.options, self.args) = parser.parse_args(args)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800637
Jon Salz73e0fd02012-04-04 11:46:38 +0800638 global _inited_logging
639 if not _inited_logging:
640 factory.init_logging('goofy', verbose=self.options.verbose)
641 _inited_logging = True
Jon Salzeb8d25f2012-05-22 15:17:32 +0800642 self.event_log = EventLog('goofy')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800643
Jon Salz73e0fd02012-04-04 11:46:38 +0800644 if (not suppress_chroot_warning and
645 factory.in_chroot() and
Jon Salz258a40c2012-04-19 12:34:01 +0800646 self.options.ui == 'gtk' and
Jon Salz758e6cc2012-04-03 15:47:07 +0800647 os.environ.get('DISPLAY') in [None, '', ':0', ':0.0']):
648 # That's not going to work! Tell the user how to run
649 # this way.
650 logging.warn(GOOFY_IN_CHROOT_WARNING)
651 time.sleep(1)
652
Jon Salz73e0fd02012-04-04 11:46:38 +0800653 if env:
654 self.env = env
655 elif factory.in_chroot():
Jon Salz258a40c2012-04-19 12:34:01 +0800656 self.env = FakeChrootEnvironment()
Jon Salz73e0fd02012-04-04 11:46:38 +0800657 logging.warn(
658 'Using chroot environment: will not actually run autotests')
659 else:
660 self.env = DUTEnvironment()
Jon Salz323dd3d2012-04-09 18:40:43 +0800661 self.env.goofy = self
Jon Salz73e0fd02012-04-04 11:46:38 +0800662
Jon Salz758e6cc2012-04-03 15:47:07 +0800663 if self.options.restart:
664 state.clear_state()
665
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800666 if self.options.print_test_list:
667 print (factory.read_test_list(self.options.print_test_list).
668 __repr__(recursive=True))
669 return
670
671 logging.info('Started')
672
673 self.start_state_server()
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800674 self.state_instance.set_shared_data('hwid_cfg', get_hwid_cfg())
Jon Salz63585ea2012-05-21 15:03:32 +0800675 self.state_instance.set_shared_data('ui_scale_factor',
676 self.options.ui_scale_factor)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800677
Jon Salz73e0fd02012-04-04 11:46:38 +0800678 self.options.test_list = (self.options.test_list or find_test_list())
679 self.test_list = factory.read_test_list(self.options.test_list,
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800680 self.state_instance)
Jon Salz06fbeff2012-05-21 17:06:05 +0800681 if not self.state_instance.has_shared_data('ui_lang'):
682 self.state_instance.set_shared_data('ui_lang',
683 self.test_list.options.ui_lang)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800684 logging.info('TEST_LIST:\n%s', self.test_list.__repr__(recursive=True))
Jon Salz258a40c2012-04-19 12:34:01 +0800685 self.state_instance.test_list = self.test_list
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800686
687 self.init_states()
688 self.start_event_server()
Jon Salz258a40c2012-04-19 12:34:01 +0800689
Jon Salzb1b39092012-05-03 02:05:09 +0800690 # Set CROS_UI since some behaviors in ui.py depend on the
691 # particular UI in use. TODO(jsalz): Remove this (and all
692 # places it is used) when the GTK UI is removed.
693 os.environ['CROS_UI'] = self.options.ui
Jon Salz258a40c2012-04-19 12:34:01 +0800694
Jon Salzb1b39092012-05-03 02:05:09 +0800695 if self.options.ui == 'chrome':
Jon Salz258a40c2012-04-19 12:34:01 +0800696 self.env.launch_chrome()
697 logging.info('Waiting for a web socket connection')
698 self.web_socket_manager.wait()
Jon Salzb1b39092012-05-03 02:05:09 +0800699
700 # Wait for the test widget size to be set; this is done in
701 # an asynchronous RPC so there is a small chance that the
702 # web socket might be opened first.
703 for i in range(100): # 10 s
Jon Salz63585ea2012-05-21 15:03:32 +0800704 try:
705 if self.state_instance.get_shared_data('test_widget_size'):
706 break
707 except KeyError:
708 pass # Retry
Jon Salzb1b39092012-05-03 02:05:09 +0800709 time.sleep(0.1) # 100 ms
710 else:
711 logging.warn('Never received test_widget_size from UI')
Jon Salz258a40c2012-04-19 12:34:01 +0800712 elif self.options.ui == 'gtk':
Jon Salz73e0fd02012-04-04 11:46:38 +0800713 self.start_ui()
Jon Salz258a40c2012-04-19 12:34:01 +0800714
Jon Salz8375c2e2012-04-04 15:22:24 +0800715 self.prespawner = Prespawner()
Jon Salz323dd3d2012-04-09 18:40:43 +0800716 self.prespawner.start()
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800717
718 def state_change_callback(test, state):
719 self.event_client.post_event(
720 Event(Event.Type.STATE_CHANGE,
721 path=test.path, state=state))
722 self.test_list.state_change_callback = state_change_callback
723
724 try:
Jon Salz758e6cc2012-04-03 15:47:07 +0800725 tests_after_shutdown = self.state_instance.get_shared_data(
726 'tests_after_shutdown')
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800727 except KeyError:
Jon Salz758e6cc2012-04-03 15:47:07 +0800728 tests_after_shutdown = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800729
Jon Salz758e6cc2012-04-03 15:47:07 +0800730 if tests_after_shutdown is not None:
731 logging.info('Resuming tests after shutdown: %s',
732 tests_after_shutdown)
733 self.state_instance.set_shared_data('tests_after_shutdown', None)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800734 self.tests_to_run.extend(
Jon Salz758e6cc2012-04-03 15:47:07 +0800735 self.test_list.lookup_path(t) for t in tests_after_shutdown)
Jon Salz73e0fd02012-04-04 11:46:38 +0800736 self.run_queue.put(self.run_next_test)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800737 else:
Jon Salz57717ca2012-04-04 16:47:25 +0800738 if self.test_list.options.auto_run_on_start:
739 self.run_queue.put(
740 lambda: self.run_tests(self.test_list, untested_only=True))
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800741
Jon Salz73e0fd02012-04-04 11:46:38 +0800742 def run(self):
743 '''Runs Goofy.'''
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800744 # Process events forever.
Jon Salz57717ca2012-04-04 16:47:25 +0800745 while self.run_once(True):
Jon Salz73e0fd02012-04-04 11:46:38 +0800746 pass
747
Jon Salz57717ca2012-04-04 16:47:25 +0800748 def run_once(self, block=False):
Jon Salz73e0fd02012-04-04 11:46:38 +0800749 '''Runs all items pending in the event loop.
750
Jon Salz57717ca2012-04-04 16:47:25 +0800751 Args:
752 block: If true, block until at least one event is processed.
753
Jon Salz73e0fd02012-04-04 11:46:38 +0800754 Returns:
755 True to keep going or False to shut down.
756 '''
Jon Salz57717ca2012-04-04 16:47:25 +0800757 events = []
758 if block:
759 # Get at least one event
760 events.append(self.run_queue.get())
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800761 while True:
Jon Salz73e0fd02012-04-04 11:46:38 +0800762 try:
763 events.append(self.run_queue.get_nowait())
764 except Queue.Empty:
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800765 break
766
Jon Salz73e0fd02012-04-04 11:46:38 +0800767 for event in events:
768 if not event:
769 # Shutdown request.
770 self.run_queue.task_done()
771 return False
772
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800773 try:
774 event()
775 except Exception as e: # pylint: disable=W0703
776 logging.error('Error in event loop: %s', e)
777 traceback.print_exc(sys.stderr)
Jon Salz8375c2e2012-04-04 15:22:24 +0800778 self.record_exception(traceback.format_exception_only(
779 *sys.exc_info()[:2]))
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800780 # But keep going
781 finally:
782 self.run_queue.task_done()
Jon Salz73e0fd02012-04-04 11:46:38 +0800783 return True
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800784
Jon Salz258a40c2012-04-19 12:34:01 +0800785 def run_tests_with_status(self, statuses_to_run, starting_at=None,
786 root=None):
Jon Salz0405ab52012-03-16 15:26:52 +0800787 '''Runs all top-level tests with a particular status.
788
789 All active tests, plus any tests to re-run, are reset.
Jon Salz57717ca2012-04-04 16:47:25 +0800790
791 Args:
792 starting_at: If provided, only auto-runs tests beginning with
793 this test.
Jon Salz0405ab52012-03-16 15:26:52 +0800794 '''
Jon Salz258a40c2012-04-19 12:34:01 +0800795 root = root or self.test_list
796
Jon Salz57717ca2012-04-04 16:47:25 +0800797 if starting_at:
798 # Make sure they passed a test, not a string.
799 assert isinstance(starting_at, factory.FactoryTest)
800
Jon Salz0405ab52012-03-16 15:26:52 +0800801 tests_to_reset = []
802 tests_to_run = []
803
Jon Salz57717ca2012-04-04 16:47:25 +0800804 found_starting_at = False
805
Jon Salz258a40c2012-04-19 12:34:01 +0800806 for test in root.get_top_level_tests():
Jon Salz57717ca2012-04-04 16:47:25 +0800807 if starting_at:
808 if test == starting_at:
809 # We've found starting_at; do auto-run on all
810 # subsequent tests.
811 found_starting_at = True
812 if not found_starting_at:
813 # Don't start this guy yet
814 continue
815
Jon Salz0405ab52012-03-16 15:26:52 +0800816 status = test.get_state().status
817 if status == TestState.ACTIVE or status in statuses_to_run:
818 # Reset the test (later; we will need to abort
819 # all active tests first).
820 tests_to_reset.append(test)
821 if status in statuses_to_run:
822 tests_to_run.append(test)
823
824 self.abort_active_tests()
825
826 # Reset all statuses of the tests to run (in case any tests were active;
827 # we want them to be run again).
828 for test_to_reset in tests_to_reset:
829 for test in test_to_reset.walk():
830 test.update_state(status=TestState.UNTESTED)
831
832 self.run_tests(tests_to_run, untested_only=True)
833
Jon Salz258a40c2012-04-19 12:34:01 +0800834 def restart_tests(self, root=None):
Jon Salz0405ab52012-03-16 15:26:52 +0800835 '''Restarts all tests.'''
Jon Salz258a40c2012-04-19 12:34:01 +0800836 root = root or self.test_list
Jon Salz0405ab52012-03-16 15:26:52 +0800837
Jon Salz258a40c2012-04-19 12:34:01 +0800838 self.abort_active_tests()
839 for test in root.walk():
840 test.update_state(status=TestState.UNTESTED)
841 self.run_tests(root)
842
843 def auto_run(self, starting_at=None, root=None):
Jon Salz57717ca2012-04-04 16:47:25 +0800844 '''"Auto-runs" tests that have not been run yet.
845
846 Args:
847 starting_at: If provide, only auto-runs tests beginning with
848 this test.
849 '''
Jon Salz258a40c2012-04-19 12:34:01 +0800850 root = root or self.test_list
Jon Salz57717ca2012-04-04 16:47:25 +0800851 self.run_tests_with_status([TestState.UNTESTED, TestState.ACTIVE],
Jon Salz258a40c2012-04-19 12:34:01 +0800852 starting_at=starting_at,
853 root=root)
Jon Salz0405ab52012-03-16 15:26:52 +0800854
Jon Salz258a40c2012-04-19 12:34:01 +0800855 def re_run_failed(self, root=None):
Jon Salz0405ab52012-03-16 15:26:52 +0800856 '''Re-runs failed tests.'''
Jon Salz258a40c2012-04-19 12:34:01 +0800857 root = root or self.test_list
858 self.run_tests_with_status([TestState.FAILED], root=root)
Jon Salz0405ab52012-03-16 15:26:52 +0800859
Jon Salz968e90b2012-03-18 16:12:43 +0800860 def show_review_information(self):
Hung-Te Lin96632362012-03-20 21:14:18 +0800861 '''Event handler for showing review information screen.
862
863 The information screene is rendered by main UI program (ui.py), so in
864 goofy we only need to kill all active tests, set them as untested, and
865 clear remaining tests.
866 '''
867 self.kill_active_tests(False)
868 self.run_tests([])
869
Jon Salz0405ab52012-03-16 15:26:52 +0800870 def handle_switch_test(self, event):
Jon Salz968e90b2012-03-18 16:12:43 +0800871 '''Switches to a particular test.
872
873 @param event: The SWITCH_TEST event.
874 '''
Jon Salz0405ab52012-03-16 15:26:52 +0800875 test = self.test_list.lookup_path(event.path)
Jon Salz57717ca2012-04-04 16:47:25 +0800876 if not test:
Jon Salz968e90b2012-03-18 16:12:43 +0800877 logging.error('Unknown test %r', event.key)
Jon Salz57717ca2012-04-04 16:47:25 +0800878 return
879
880 invoc = self.invocations.get(test)
881 if invoc and test.backgroundable:
882 # Already running: just bring to the front if it
883 # has a UI.
884 logging.info('Setting visible test to %s', test.path)
885 self.event_client.post_event(
886 Event(Event.Type.SET_VISIBLE_TEST, path=test.path))
887 return
888
889 self.abort_active_tests()
890 for t in test.walk():
891 t.update_state(status=TestState.UNTESTED)
892
893 if self.test_list.options.auto_run_on_keypress:
894 self.auto_run(starting_at=test)
895 else:
896 self.run_tests(test)
Jon Salz0405ab52012-03-16 15:26:52 +0800897
Jon Salz73e0fd02012-04-04 11:46:38 +0800898 def wait(self):
899 '''Waits for all pending invocations.
900
901 Useful for testing.
902 '''
903 for k, v in self.invocations.iteritems():
904 logging.info('Waiting for %s to complete...', k)
905 v.thread.join()
906
907 def check_exceptions(self):
908 '''Raises an error if any exceptions have occurred in
909 invocation threads.'''
910 if self.exceptions:
911 raise RuntimeError('Exception in invocation thread: %r' %
912 self.exceptions)
913
914 def record_exception(self, msg):
915 '''Records an exception in an invocation thread.
916
917 An exception with the given message will be rethrown when
918 Goofy is destroyed.'''
919 self.exceptions.append(msg)
920
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800921
922if __name__ == '__main__':
923 Goofy().main()