blob: 001b3ea785f53cc3caad25be44ece960235b8edb [file] [log] [blame]
Hung-Te Linf2f78f72012-02-08 19:27:11 +08001#!/usr/bin/python -u
Hung-Te Linf2f78f72012-02-08 19:27:11 +08002# -*- coding: utf-8 -*-
3#
Jon Salz37eccbd2012-05-25 16:06:52 +08004# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Hung-Te Linf2f78f72012-02-08 19:27:11 +08005# Use of this source code is governed by a BSD-style license that can be
6# found in the LICENSE file.
7
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08008"""The main factory flow that runs the factory test and finalizes a device."""
Hung-Te Linf2f78f72012-02-08 19:27:11 +08009
Joel Kitchingb85ed7f2014-10-08 18:24:39 +080010from __future__ import print_function
11
Jon Salze12c2b32013-06-25 16:24:34 +080012import glob
Wei-Han Chenc17b4112016-11-22 14:56:51 +080013import itertools
Jon Salz0405ab52012-03-16 15:26:52 +080014import logging
Wei-Han Chenc17b4112016-11-22 14:56:51 +080015from optparse import OptionParser
Jon Salz0405ab52012-03-16 15:26:52 +080016import os
Jon Salze12c2b32013-06-25 16:24:34 +080017import shutil
Jon Salz77c151e2012-08-28 07:20:37 +080018import signal
Jon Salz0405ab52012-03-16 15:26:52 +080019import sys
Jon Salzeff94182013-06-19 15:06:28 +080020import syslog
Jon Salz0405ab52012-03-16 15:26:52 +080021import threading
22import time
23import traceback
Jon Salz258a40c2012-04-19 12:34:01 +080024import uuid
Jon Salzb10cf512012-08-09 17:29:21 +080025from xmlrpclib import Binary
Hung-Te Linf2f78f72012-02-08 19:27:11 +080026
Jon Salz0697cbf2012-07-04 15:14:04 +080027import factory_common # pylint: disable=W0611
Hung-Te Linb6287242016-05-18 14:39:05 +080028from cros.factory.device import device_utils
Vic Yangd80ea752014-09-24 16:07:14 +080029from cros.factory.goofy.goofy_base import GoofyBase
30from cros.factory.goofy.goofy_rpc import GoofyRPC
Earl Ouacbe99c2017-02-21 16:04:19 +080031from cros.factory.goofy import goofy_server
Vic Yangd80ea752014-09-24 16:07:14 +080032from cros.factory.goofy.invocation import TestInvocation
33from cros.factory.goofy.link_manager import PresenterLinkManager
Earl Oua3bca122016-10-21 16:00:30 +080034from cros.factory.goofy.plugins import plugin_controller
Vic Yange2c76a82014-10-30 12:48:19 -070035from cros.factory.goofy import prespawner
Wei-Ning Huang38b75f02015-02-25 18:25:14 +080036from cros.factory.goofy.terminal_manager import TerminalManager
Earl Oua3bca122016-10-21 16:00:30 +080037from cros.factory.goofy import test_environment
Wei-Han Chenc17b4112016-11-22 14:56:51 +080038from cros.factory.goofy.test_list_iterator import TestListIterator
Earl Oua3bca122016-10-21 16:00:30 +080039from cros.factory.goofy import updater
Vic Yangd80ea752014-09-24 16:07:14 +080040from cros.factory.goofy.web_socket_manager import WebSocketManager
Hung-Te Linb6287242016-05-18 14:39:05 +080041from cros.factory.test.e2e_test.common import AutomationMode
42from cros.factory.test.e2e_test.common import AutomationModePrompt
43from cros.factory.test.e2e_test.common import ParseAutomationMode
Earl Ouacbe99c2017-02-21 16:04:19 +080044from cros.factory.test.env import goofy_proxy
Hung-Te Linb6287242016-05-18 14:39:05 +080045from cros.factory.test.env import paths
Jon Salz83591782012-06-26 11:09:58 +080046from cros.factory.test.event import Event
47from cros.factory.test.event import EventClient
48from cros.factory.test.event import EventServer
Hung-Te Linb6287242016-05-18 14:39:05 +080049from cros.factory.test import event_log
50from cros.factory.test.event_log import EventLog
Hung-Te Linb6287242016-05-18 14:39:05 +080051from cros.factory.test.event_log import GetBootSequence
Hung-Te Lin91492a12014-11-25 18:56:30 +080052from cros.factory.test.event_log_watcher import EventLogWatcher
Earl Oua3bca122016-10-21 16:00:30 +080053from cros.factory.test import factory
jcliangcd688182012-08-20 21:01:26 +080054from cros.factory.test.factory import TestState
Hung-Te Lin3f096842016-01-13 17:37:06 +080055from cros.factory.test.rules import phase
Earl Oua3bca122016-10-21 16:00:30 +080056from cros.factory.test import shopfloor
57from cros.factory.test import state
Wei-Han Chen2ebb92d2016-01-12 14:51:41 +080058from cros.factory.test.test_lists import test_lists
Earl Oua3bca122016-10-21 16:00:30 +080059from cros.factory.test import testlog
60from cros.factory.test import testlog_goofy
Hung-Te Linb6287242016-05-18 14:39:05 +080061from cros.factory.tools.key_filter import KeyFilter
Wei-Han Chen78f35f62017-03-06 20:11:20 +080062from cros.factory.utils import config_utils
Hung-Te Linf707b242016-01-08 23:11:42 +080063from cros.factory.utils import debug_utils
Jon Salz2af235d2013-06-24 14:47:21 +080064from cros.factory.utils import file_utils
Joel Kitchingb85ed7f2014-10-08 18:24:39 +080065from cros.factory.utils import net_utils
Hung-Te Lin4e6357c2016-01-08 14:32:00 +080066from cros.factory.utils import process_utils
67from cros.factory.utils import sys_utils
68from cros.factory.utils import time_utils
Hung-Te Linf707b242016-01-08 23:11:42 +080069from cros.factory.utils import type_utils
Hung-Te Linf2f78f72012-02-08 19:27:11 +080070
71
Hung-Te Linf2f78f72012-02-08 19:27:11 +080072HWID_CFG_PATH = '/usr/local/share/chromeos-hwid/cfg'
Joel Kitching625ff0f2016-05-16 14:59:40 -070073CACHES_DIR = os.path.join(paths.GetStateRoot(), 'caches')
Hung-Te Linf2f78f72012-02-08 19:27:11 +080074
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +080075CLEANUP_LOGS_PAUSED = '/var/lib/cleanup_logs_paused'
76
Jon Salz5c344f62012-07-13 14:31:16 +080077# Value for tests_after_shutdown that forces auto-run (e.g., after
78# a factory update, when the available set of tests might change).
79FORCE_AUTO_RUN = 'force_auto_run'
80
Wei-Han Chenc17b4112016-11-22 14:56:51 +080081# Key to load the test list iterator after shutdown test
82TESTS_AFTER_SHUTDOWN = 'tests_after_shutdown'
83
Justin Chuang83813982013-05-13 01:26:32 +080084
Ricky Liang45c73e72015-01-15 15:00:30 +080085MAX_CRASH_FILE_SIZE = 64 * 1024
Jon Salze12c2b32013-06-25 16:24:34 +080086
Hung-Te Linf707b242016-01-08 23:11:42 +080087Status = type_utils.Enum(['UNINITIALIZED', 'INITIALIZING', 'RUNNING',
Wei-Han Chen2ebb92d2016-01-12 14:51:41 +080088 'TERMINATING', 'TERMINATED'])
Jon Salzd7550792013-07-12 05:49:27 +080089
Hung-Te Linf2f78f72012-02-08 19:27:11 +080090def get_hwid_cfg():
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +080091 """Returns the HWID config tag, or an empty string if none can be found."""
Jon Salz0697cbf2012-07-04 15:14:04 +080092 if 'CROS_HWID' in os.environ:
93 return os.environ['CROS_HWID']
94 if os.path.exists(HWID_CFG_PATH):
Ricky Liang45c73e72015-01-15 15:00:30 +080095 with open(HWID_CFG_PATH, 'r') as hwid_cfg_handle:
Jon Salz0697cbf2012-07-04 15:14:04 +080096 return hwid_cfg_handle.read().strip()
97 return ''
Hung-Te Linf2f78f72012-02-08 19:27:11 +080098
Jon Salz73e0fd02012-04-04 11:46:38 +080099_inited_logging = False
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800100
Ricky Liang45c73e72015-01-15 15:00:30 +0800101
Peter Ammon1e1ec572014-06-26 17:56:32 -0700102class Goofy(GoofyBase):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800103 """The main factory flow.
Jon Salz0697cbf2012-07-04 15:14:04 +0800104
105 Note that all methods in this class must be invoked from the main
106 (event) thread. Other threads, such as callbacks and TestInvocation
107 methods, should instead post events on the run queue.
108
109 TODO: Unit tests. (chrome-os-partner:7409)
110
111 Properties:
112 uuid: A unique UUID for this invocation of Goofy.
113 state_instance: An instance of FactoryState.
114 state_server: The FactoryState XML/RPC server.
115 state_server_thread: A thread running state_server.
116 event_server: The EventServer socket server.
117 event_server_thread: A thread running event_server.
118 event_client: A client to the event server.
Earl Oua3bca122016-10-21 16:00:30 +0800119 plugin_controller: The PluginController object.
Jon Salz0697cbf2012-07-04 15:14:04 +0800120 invocations: A map from FactoryTest objects to the corresponding
121 TestInvocations objects representing active tests.
Jon Salz0697cbf2012-07-04 15:14:04 +0800122 options: Command-line options.
123 args: Command-line args.
124 test_list: The test list.
Jon Salz128b0932013-07-03 16:55:26 +0800125 test_lists: All new-style test lists.
Ricky Liang4bff3e32014-02-20 18:46:11 +0800126 run_id: The identifier for latest test run.
127 scheduled_run_tests: The list of tests scheduled for latest test run.
Jon Salz0697cbf2012-07-04 15:14:04 +0800128 event_handlers: Map of Event.Type to the method used to handle that
129 event. If the method has an 'event' argument, the event is passed
130 to the handler.
Jon Salz416f9cc2013-05-10 18:32:50 +0800131 hooks: A Hooks object containing hooks for various Goofy actions.
Jon Salzd7550792013-07-12 05:49:27 +0800132 status: The current Goofy status (a member of the Status enum).
Peter Ammon948b7172014-07-15 12:43:06 -0700133 link_manager: Instance of PresenterLinkManager for communicating
134 with GoofyPresenter
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800135 """
Ricky Liang45c73e72015-01-15 15:00:30 +0800136
Jon Salz0697cbf2012-07-04 15:14:04 +0800137 def __init__(self):
Peter Ammon1e1ec572014-06-26 17:56:32 -0700138 super(Goofy, self).__init__()
Jon Salz0697cbf2012-07-04 15:14:04 +0800139 self.uuid = str(uuid.uuid4())
140 self.state_instance = None
Earl Ouacbe99c2017-02-21 16:04:19 +0800141 self.goofy_server = None
142 self.goofy_server_thread = None
Jon Salz16d10542012-07-23 12:18:45 +0800143 self.goofy_rpc = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800144 self.event_server = None
145 self.event_server_thread = None
146 self.event_client = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800147 self.log_watcher = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800148 self.event_log = None
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800149 self.testlog = None
Vic Yange2c76a82014-10-30 12:48:19 -0700150 self.autotest_prespawner = None
Earl Oua3bca122016-10-21 16:00:30 +0800151 self.plugin_controller = None
Vic Yange2c76a82014-10-30 12:48:19 -0700152 self.pytest_prespawner = None
Vic Yanga3cecf82014-12-26 00:44:21 -0800153 self._ui_initialized = False
Jon Salzc79a9982012-08-30 04:42:01 +0800154 self.dummy_shopfloor = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800155 self.invocations = {}
Jon Salz0697cbf2012-07-04 15:14:04 +0800156 self.visible_test = None
157 self.chrome = None
Jon Salz416f9cc2013-05-10 18:32:50 +0800158 self.hooks = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800159
160 self.options = None
161 self.args = None
162 self.test_list = None
Jon Salz128b0932013-07-03 16:55:26 +0800163 self.test_lists = None
Ricky Liang4bff3e32014-02-20 18:46:11 +0800164 self.run_id = None
165 self.scheduled_run_tests = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800166 self.env = None
Jon Salzb22d1172012-08-06 10:38:57 +0800167 self.last_idle = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800168 self.last_shutdown_time = None
cychiang21886742012-07-05 15:16:32 +0800169 self.last_update_check = None
Cheng-Yi Chiang194d3c02015-03-16 14:37:15 +0800170 self._suppress_periodic_update_messages = False
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +0800171 self._suppress_event_log_error_messages = False
Earl Ouab979142016-10-25 16:48:06 +0800172 self.exclusive_resources = set()
Dean Liao592e4d52013-01-10 20:06:39 +0800173 self.key_filter = None
Jon Salzd7550792013-07-12 05:49:27 +0800174 self.status = Status.UNINITIALIZED
Ricky Liang36512a32014-07-25 11:47:04 +0800175 self.ready_for_ui_connection = False
Peter Ammon1e1ec572014-06-26 17:56:32 -0700176 self.link_manager = None
Hung-Te Linef7f2be2015-07-20 20:38:51 +0800177 self.is_restart_requested = False
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800178 self.test_list_iterator = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800179
Hung-Te Lin6a72c642015-12-13 22:09:09 +0800180 # TODO(hungte) Support controlling remote DUT.
Hung-Te Linb6287242016-05-18 14:39:05 +0800181 self.dut = device_utils.CreateDUTInterface()
Hung-Te Lin6a72c642015-12-13 22:09:09 +0800182
Jon Salz85a39882012-07-05 16:45:04 +0800183 def test_or_root(event, parent_or_group=True):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800184 """Returns the test affected by a particular event.
Jon Salz85a39882012-07-05 16:45:04 +0800185
186 Args:
187 event: The event containing an optional 'path' attribute.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800188 parent_or_group: If True, returns the top-level parent for a test (the
Jon Salz85a39882012-07-05 16:45:04 +0800189 root node of the tests that need to be run together if the given test
190 path is to be run).
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800191 """
Jon Salz0697cbf2012-07-04 15:14:04 +0800192 try:
193 path = event.path
194 except AttributeError:
195 path = None
196
197 if path:
Jon Salz85a39882012-07-05 16:45:04 +0800198 test = self.test_list.lookup_path(path)
199 if parent_or_group:
200 test = test.get_top_level_parent_or_group()
201 return test
Jon Salz0697cbf2012-07-04 15:14:04 +0800202 else:
203 return self.test_list
204
205 self.event_handlers = {
Ricky Liang45c73e72015-01-15 15:00:30 +0800206 Event.Type.SWITCH_TEST: self.handle_switch_test,
Ricky Liang45c73e72015-01-15 15:00:30 +0800207 Event.Type.RESTART_TESTS:
208 lambda event: self.restart_tests(root=test_or_root(event)),
209 Event.Type.AUTO_RUN:
210 lambda event: self.auto_run(root=test_or_root(event)),
Ricky Liang45c73e72015-01-15 15:00:30 +0800211 Event.Type.RUN_TESTS_WITH_STATUS:
212 lambda event: self.run_tests_with_status(
213 event.status,
214 root=test_or_root(event)),
Ricky Liang45c73e72015-01-15 15:00:30 +0800215 Event.Type.UPDATE_SYSTEM_INFO:
216 lambda event: self.update_system_info(),
217 Event.Type.STOP:
218 lambda event: self.stop(root=test_or_root(event, False),
219 fail=getattr(event, 'fail', False),
220 reason=getattr(event, 'reason', None)),
221 Event.Type.SET_VISIBLE_TEST:
222 lambda event: self.set_visible_test(
223 self.test_list.lookup_path(event.path)),
224 Event.Type.CLEAR_STATE:
225 lambda event: self.clear_state(
226 self.test_list.lookup_path(event.path)),
Wei-Ning Huang38b75f02015-02-25 18:25:14 +0800227 Event.Type.KEY_FILTER_MODE: self.handle_key_filter_mode,
Jon Salz0697cbf2012-07-04 15:14:04 +0800228 }
229
Jon Salz0697cbf2012-07-04 15:14:04 +0800230 self.web_socket_manager = None
Wei-Ning Huang38b75f02015-02-25 18:25:14 +0800231 self.terminal_manager = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800232
233 def destroy(self):
Ricky Liang74237a02014-09-18 15:11:23 +0800234 """Performs any shutdown tasks. Overrides base class method."""
Jon Salzd7550792013-07-12 05:49:27 +0800235 self.status = Status.TERMINATING
Jon Salz0697cbf2012-07-04 15:14:04 +0800236 if self.chrome:
237 self.chrome.kill()
238 self.chrome = None
Jon Salzc79a9982012-08-30 04:42:01 +0800239 if self.dummy_shopfloor:
240 self.dummy_shopfloor.kill()
241 self.dummy_shopfloor = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800242 if self.web_socket_manager:
243 logging.info('Stopping web sockets')
244 self.web_socket_manager.close()
245 self.web_socket_manager = None
Earl Ouacbe99c2017-02-21 16:04:19 +0800246 if self.goofy_server_thread:
247 logging.info('Stopping goofy server')
248 self.goofy_server.shutdown()
249 self.goofy_server_thread.join()
250 self.goofy_server.server_close()
251 self.goofy_server_thread = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800252 if self.state_instance:
253 self.state_instance.close()
254 if self.event_server_thread:
255 logging.info('Stopping event server')
256 self.event_server.shutdown() # pylint: disable=E1101
257 self.event_server_thread.join()
258 self.event_server.server_close()
259 self.event_server_thread = None
260 if self.log_watcher:
261 if self.log_watcher.IsThreadStarted():
262 self.log_watcher.StopWatchThread()
263 self.log_watcher = None
Vic Yange2c76a82014-10-30 12:48:19 -0700264 if self.autotest_prespawner:
265 logging.info('Stopping autotest prespawner')
266 self.autotest_prespawner.stop()
267 self.autotest_prespawner = None
268 if self.pytest_prespawner:
269 logging.info('Stopping pytest prespawner')
270 self.pytest_prespawner.stop()
271 self.pytest_prespawner = None
Jon Salz0697cbf2012-07-04 15:14:04 +0800272 if self.event_client:
273 logging.info('Closing event client')
274 self.event_client.close()
275 self.event_client = None
276 if self.event_log:
277 self.event_log.Close()
278 self.event_log = None
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800279 if self.testlog:
280 self.testlog.Close()
281 self.testlog = None
Dean Liao592e4d52013-01-10 20:06:39 +0800282 if self.key_filter:
283 self.key_filter.Stop()
Peter Ammon1e1ec572014-06-26 17:56:32 -0700284 if self.link_manager:
285 self.link_manager.Stop()
286 self.link_manager = None
Earl Oua3bca122016-10-21 16:00:30 +0800287 if self.plugin_controller:
288 self.plugin_controller.StopAndDestroyAllPlugins()
289 self.plugin_controller = None
Dean Liao592e4d52013-01-10 20:06:39 +0800290
Peter Ammon1e1ec572014-06-26 17:56:32 -0700291 super(Goofy, self).destroy()
Jon Salz0697cbf2012-07-04 15:14:04 +0800292 logging.info('Done destroying Goofy')
Jon Salzd7550792013-07-12 05:49:27 +0800293 self.status = Status.TERMINATED
Jon Salz0697cbf2012-07-04 15:14:04 +0800294
Earl Ouacbe99c2017-02-21 16:04:19 +0800295 def start_goofy_server(self):
296 self.goofy_server = goofy_server.GoofyServer(
297 (goofy_proxy.DEFAULT_GOOFY_ADDRESS, goofy_proxy.DEFAULT_GOOFY_PORT))
298 logging.info('Starting goofy server')
299 self.goofy_server_thread = threading.Thread(
300 target=self.goofy_server.serve_forever,
301 name='GoofyServer')
302 self.goofy_server_thread.start()
303
304 # Setup static file path
305 self.goofy_server.RegisterPath(
306 '/', os.path.join(paths.FACTORY_PACKAGE_PATH, 'goofy/static'))
307
308 def init_state_instance(self):
Jon Salz2af235d2013-06-24 14:47:21 +0800309 # Before starting state server, remount stateful partitions with
310 # no commit flag. The default commit time (commit=600) makes corruption
311 # too likely.
Hung-Te Lin1968d9c2016-01-08 22:55:46 +0800312 sys_utils.ResetCommitTime()
Earl Ouacbe99c2017-02-21 16:04:19 +0800313 self.state_instance = state.FactoryState()
314 self.goofy_server.AddRPCInstance(goofy_proxy.STATE_URL, self.state_instance)
Jon Salz2af235d2013-06-24 14:47:21 +0800315
Earl Ouacbe99c2017-02-21 16:04:19 +0800316 # Setup Goofy RPC.
317 # TODO(shunhsingou): separate goofy_rpc and state server instead of
318 # injecting goofy_rpc functions into state.
Jon Salz16d10542012-07-23 12:18:45 +0800319 self.goofy_rpc = GoofyRPC(self)
320 self.goofy_rpc.RegisterMethods(self.state_instance)
Jon Salz0697cbf2012-07-04 15:14:04 +0800321
322 def start_event_server(self):
323 self.event_server = EventServer()
324 logging.info('Starting factory event server')
325 self.event_server_thread = threading.Thread(
Ricky Liang45c73e72015-01-15 15:00:30 +0800326 target=self.event_server.serve_forever,
327 name='EventServer') # pylint: disable=E1101
Jon Salz0697cbf2012-07-04 15:14:04 +0800328 self.event_server_thread.start()
329
330 self.event_client = EventClient(
Ricky Liang45c73e72015-01-15 15:00:30 +0800331 callback=self.handle_event, event_loop=self.run_queue)
Jon Salz0697cbf2012-07-04 15:14:04 +0800332
333 self.web_socket_manager = WebSocketManager(self.uuid)
Earl Ouacbe99c2017-02-21 16:04:19 +0800334 self.goofy_server.AddHTTPGetHandler(
335 '/event', self.web_socket_manager.handle_web_socket)
Jon Salz0697cbf2012-07-04 15:14:04 +0800336
Wei-Ning Huang38b75f02015-02-25 18:25:14 +0800337 def start_terminal_server(self):
338 self.terminal_manager = TerminalManager()
Earl Ouacbe99c2017-02-21 16:04:19 +0800339 self.goofy_server.AddHTTPGetHandler(
340 '/pty', self.terminal_manager.handle_web_socket)
Wei-Ning Huang38b75f02015-02-25 18:25:14 +0800341
Jon Salz0697cbf2012-07-04 15:14:04 +0800342 def set_visible_test(self, test):
343 if self.visible_test == test:
344 return
Jon Salz2f2d42c2012-07-30 12:30:34 +0800345 if test and not test.has_ui:
346 return
Jon Salz0697cbf2012-07-04 15:14:04 +0800347
348 if test:
349 test.update_state(visible=True)
350 if self.visible_test:
351 self.visible_test.update_state(visible=False)
352 self.visible_test = test
353
Ricky Liang48e47f92014-02-26 19:31:51 +0800354 def log_startup_messages(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800355 """Logs the tail of var/log/messages and mosys and EC console logs."""
Jon Salzd4306c82012-11-30 15:16:36 +0800356 # TODO(jsalz): This is mostly a copy-and-paste of code in init_states,
357 # for factory-3004.B only. Consolidate and merge back to ToT.
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +0800358 if sys_utils.InChroot():
Jon Salzd4306c82012-11-30 15:16:36 +0800359 return
360
361 try:
Hung-Te Lin1a4e30c2016-01-08 23:25:10 +0800362 var_log_messages = sys_utils.GetVarLogMessagesBeforeReboot()
Jon Salzd4306c82012-11-30 15:16:36 +0800363 logging.info(
Ricky Liang45c73e72015-01-15 15:00:30 +0800364 'Tail of /var/log/messages before last reboot:\n'
365 '%s', ('\n'.join(
366 ' ' + x for x in var_log_messages)))
Jon Salzd4306c82012-11-30 15:16:36 +0800367 except: # pylint: disable=W0702
368 logging.exception('Unable to grok /var/log/messages')
369
370 try:
Hung-Te Lin4e6357c2016-01-08 14:32:00 +0800371 mosys_log = process_utils.Spawn(
Jon Salzd4306c82012-11-30 15:16:36 +0800372 ['mosys', 'eventlog', 'list'],
373 read_stdout=True, log_stderr_on_error=True).stdout_data
374 logging.info('System eventlog from mosys:\n%s\n', mosys_log)
375 except: # pylint: disable=W0702
376 logging.exception('Unable to read mosys eventlog')
377
Dean Liao88b93192014-10-23 19:37:41 +0800378 self.log_ec_console()
379 self.log_ec_panic_info()
380
381 @staticmethod
382 def log_ec_console():
383 """Logs EC console log into logging.info.
384
385 It logs an error message in logging.exception if an exception is raised
386 when getting EC console log.
387 For unsupported device, it logs unsupport message in logging.info
388
389 Returns:
390 EC console log string.
391 """
Jon Salzd4306c82012-11-30 15:16:36 +0800392 try:
Hung-Te Linb6287242016-05-18 14:39:05 +0800393 ec_console_log = device_utils.CreateDUTInterface().ec.GetECConsoleLog()
Jon Salzd4306c82012-11-30 15:16:36 +0800394 logging.info('EC console log after reboot:\n%s\n', ec_console_log)
Dean Liao88b93192014-10-23 19:37:41 +0800395 return ec_console_log
396 except NotImplementedError:
397 logging.info('EC console log not supported')
Jon Salzd4306c82012-11-30 15:16:36 +0800398 except: # pylint: disable=W0702
399 logging.exception('Error retrieving EC console log')
400
Dean Liao88b93192014-10-23 19:37:41 +0800401 @staticmethod
402 def log_ec_panic_info():
403 """Logs EC panic info into logging.info.
404
405 It logs an error message in logging.exception if an exception is raised
406 when getting EC panic info.
407 For unsupported device, it logs unsupport message in logging.info
408
409 Returns:
410 EC panic info string.
411 """
Vic Yang079f9872013-07-01 11:32:00 +0800412 try:
Hung-Te Linb6287242016-05-18 14:39:05 +0800413 ec_panic_info = device_utils.CreateDUTInterface().ec.GetECPanicInfo()
Vic Yang079f9872013-07-01 11:32:00 +0800414 logging.info('EC panic info after reboot:\n%s\n', ec_panic_info)
Dean Liao88b93192014-10-23 19:37:41 +0800415 return ec_panic_info
416 except NotImplementedError:
417 logging.info('EC panic info is not supported')
Vic Yang079f9872013-07-01 11:32:00 +0800418 except: # pylint: disable=W0702
419 logging.exception('Error retrieving EC panic info')
420
Ricky Liang48e47f92014-02-26 19:31:51 +0800421 def shutdown(self, operation):
422 """Starts shutdown procedure.
423
424 Args:
Vic (Chun-Ju) Yang05b0d952014-04-28 17:39:09 +0800425 operation: The shutdown operation (reboot, full_reboot, or halt).
Ricky Liang48e47f92014-02-26 19:31:51 +0800426 """
427 active_tests = []
428 for test in self.test_list.walk():
429 if not test.is_leaf():
430 continue
431
432 test_state = test.get_state()
433 if test_state.status == TestState.ACTIVE:
434 active_tests.append(test)
435
Ricky Liang48e47f92014-02-26 19:31:51 +0800436 if not (len(active_tests) == 1 and
437 isinstance(active_tests[0], factory.ShutdownStep)):
438 logging.error(
439 'Calling Goofy shutdown outside of the shutdown factory test')
440 return
441
442 logging.info('Start Goofy shutdown (%s)', operation)
443 # Save pending test list in the state server
444 self.state_instance.set_shared_data(
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800445 TESTS_AFTER_SHUTDOWN, self.test_list_iterator)
Ricky Liang48e47f92014-02-26 19:31:51 +0800446 # Save shutdown time
447 self.state_instance.set_shared_data('shutdown_time', time.time())
448
449 with self.env.lock:
450 self.event_log.Log('shutdown', operation=operation)
451 shutdown_result = self.env.shutdown(operation)
452 if shutdown_result:
453 # That's all, folks!
Peter Ammon1e1ec572014-06-26 17:56:32 -0700454 self.run_enqueue(None)
Ricky Liang48e47f92014-02-26 19:31:51 +0800455 else:
456 # Just pass (e.g., in the chroot).
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800457 self.state_instance.set_shared_data(TESTS_AFTER_SHUTDOWN, None)
Ricky Liang48e47f92014-02-26 19:31:51 +0800458 # Send event with no fields to indicate that there is no
459 # longer a pending shutdown.
460 self.event_client.post_event(Event(Event.Type.PENDING_SHUTDOWN))
461
462 def handle_shutdown_complete(self, test):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800463 """Handles the case where a shutdown was detected during a shutdown step.
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800464
Ricky Liang6fe218c2013-12-27 15:17:17 +0800465 Args:
466 test: The ShutdownStep.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800467 """
Jon Salz0697cbf2012-07-04 15:14:04 +0800468 test_state = test.update_state(increment_shutdown_count=1)
469 logging.info('Detected shutdown (%d of %d)',
Ricky Liang48e47f92014-02-26 19:31:51 +0800470 test_state.shutdown_count, test.iterations)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800471
Ricky Liang48e47f92014-02-26 19:31:51 +0800472 tests_after_shutdown = self.state_instance.get_shared_data(
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800473 TESTS_AFTER_SHUTDOWN, optional=True)
474
475 # Make this shutdown test the next test to run. This is to continue on
476 # post-shutdown verification in the shutdown step.
Ricky Liang48e47f92014-02-26 19:31:51 +0800477 if not tests_after_shutdown:
Ricky Liang48e47f92014-02-26 19:31:51 +0800478 self.state_instance.set_shared_data(
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800479 TESTS_AFTER_SHUTDOWN, TestListIterator(test))
480 else:
481 # unset inited, so we will start from the reboot test.
482 tests_after_shutdown.inited = False
483 self.state_instance.set_shared_data(
484 TESTS_AFTER_SHUTDOWN, tests_after_shutdown)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800485
Ricky Liang48e47f92014-02-26 19:31:51 +0800486 # Set 'post_shutdown' to inform shutdown test that a shutdown just occurred.
Ricky Liangb7eb8772014-09-15 18:05:22 +0800487 self.state_instance.set_shared_data(
488 state.POST_SHUTDOWN_TAG % test.path,
489 self.state_instance.get_test_state(test.path).invocation)
Jon Salz258a40c2012-04-19 12:34:01 +0800490
Jon Salz0697cbf2012-07-04 15:14:04 +0800491 def init_states(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800492 """Initializes all states on startup."""
Jon Salz0697cbf2012-07-04 15:14:04 +0800493 for test in self.test_list.get_all_tests():
494 # Make sure the state server knows about all the tests,
495 # defaulting to an untested state.
496 test.update_state(update_parent=False, visible=False)
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800497
Jon Salz0697cbf2012-07-04 15:14:04 +0800498 var_log_messages = None
Vic Yanga9c32212012-08-16 20:07:54 +0800499 mosys_log = None
Vic Yange4c275d2012-08-28 01:50:20 +0800500 ec_console_log = None
Vic Yang079f9872013-07-01 11:32:00 +0800501 ec_panic_info = None
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800502
Jon Salz0697cbf2012-07-04 15:14:04 +0800503 # Any 'active' tests should be marked as failed now.
504 for test in self.test_list.walk():
Jon Salza6711d72012-07-18 14:33:03 +0800505 if not test.is_leaf():
506 # Don't bother with parents; they will be updated when their
507 # children are updated.
508 continue
509
Jon Salz0697cbf2012-07-04 15:14:04 +0800510 test_state = test.get_state()
511 if test_state.status != TestState.ACTIVE:
512 continue
513 if isinstance(test, factory.ShutdownStep):
514 # Shutdown while the test was active - that's good.
Ricky Liang48e47f92014-02-26 19:31:51 +0800515 self.handle_shutdown_complete(test)
Jon Salz0697cbf2012-07-04 15:14:04 +0800516 else:
517 # Unexpected shutdown. Grab /var/log/messages for context.
518 if var_log_messages is None:
519 try:
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800520 var_log_messages = sys_utils.GetVarLogMessagesBeforeReboot()
Jon Salz0697cbf2012-07-04 15:14:04 +0800521 # Write it to the log, to make it easier to
522 # correlate with /var/log/messages.
523 logging.info(
Ricky Liang45c73e72015-01-15 15:00:30 +0800524 'Unexpected shutdown. '
525 'Tail of /var/log/messages before last reboot:\n'
526 '%s', ('\n'.join(
527 ' ' + x for x in var_log_messages)))
Jon Salz0697cbf2012-07-04 15:14:04 +0800528 except: # pylint: disable=W0702
529 logging.exception('Unable to grok /var/log/messages')
530 var_log_messages = []
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800531
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +0800532 if mosys_log is None and not sys_utils.InChroot():
Jon Salz008f4ea2012-08-28 05:39:45 +0800533 try:
Hung-Te Lin4e6357c2016-01-08 14:32:00 +0800534 mosys_log = process_utils.Spawn(
Jon Salz008f4ea2012-08-28 05:39:45 +0800535 ['mosys', 'eventlog', 'list'],
536 read_stdout=True, log_stderr_on_error=True).stdout_data
537 # Write it to the log also.
538 logging.info('System eventlog from mosys:\n%s\n', mosys_log)
539 except: # pylint: disable=W0702
540 logging.exception('Unable to read mosys eventlog')
Vic Yanga9c32212012-08-16 20:07:54 +0800541
Vic Yange4c275d2012-08-28 01:50:20 +0800542 if ec_console_log is None:
Dean Liao88b93192014-10-23 19:37:41 +0800543 ec_console_log = self.log_ec_console()
Vic Yange4c275d2012-08-28 01:50:20 +0800544
Vic Yang079f9872013-07-01 11:32:00 +0800545 if ec_panic_info is None:
Dean Liao88b93192014-10-23 19:37:41 +0800546 ec_panic_info = self.log_ec_panic_info()
Vic Yang079f9872013-07-01 11:32:00 +0800547
Jon Salz0697cbf2012-07-04 15:14:04 +0800548 error_msg = 'Unexpected shutdown while test was running'
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800549 # TODO(itspeter): Add testlog to collect expired session infos.
Jon Salz0697cbf2012-07-04 15:14:04 +0800550 self.event_log.Log('end_test',
Ricky Liang45c73e72015-01-15 15:00:30 +0800551 path=test.path,
552 status=TestState.FAILED,
553 invocation=test.get_state().invocation,
554 error_msg=error_msg,
555 var_log_messages='\n'.join(var_log_messages),
556 mosys_log=mosys_log)
Jon Salz0697cbf2012-07-04 15:14:04 +0800557 test.update_state(
Ricky Liang45c73e72015-01-15 15:00:30 +0800558 status=TestState.FAILED,
559 error_msg=error_msg)
Chun-Ta Lin87c2dac2015-05-02 01:35:01 -0700560 # Trigger the OnTestFailure callback.
Claire Changd1961a22015-08-05 16:15:55 +0800561 self.run_queue.put(lambda: self.test_fail(test))
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800562
Jon Salz50efe942012-07-26 11:54:10 +0800563 if not test.never_fails:
564 # For "never_fails" tests (such as "Start"), don't cancel
565 # pending tests, since reboot is expected.
566 factory.console.info('Unexpected shutdown while test %s '
567 'running; cancelling any pending tests',
568 test.path)
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800569 # cancel pending tests by replace the iterator with an empty one
570 self.state_instance.set_shared_data(
571 TESTS_AFTER_SHUTDOWN,
572 TestListIterator(None))
Jon Salz008f4ea2012-08-28 05:39:45 +0800573
Jon Salz0697cbf2012-07-04 15:14:04 +0800574 def handle_event(self, event):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800575 """Handles an event from the event server."""
Jon Salz0697cbf2012-07-04 15:14:04 +0800576 handler = self.event_handlers.get(event.type)
577 if handler:
578 handler(event)
579 else:
580 # We don't register handlers for all event types - just ignore
581 # this event.
582 logging.debug('Unbound event type %s', event.type)
Jon Salz4f6c7172012-06-11 20:45:36 +0800583
Vic Yangaabf9fd2013-04-09 18:56:13 +0800584 def check_critical_factory_note(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800585 """Returns True if the last factory note is critical."""
Vic Yangaabf9fd2013-04-09 18:56:13 +0800586 notes = self.state_instance.get_shared_data('factory_note', True)
587 return notes and notes[-1]['level'] == 'CRITICAL'
588
Hung-Te Linef7f2be2015-07-20 20:38:51 +0800589 def schedule_restart(self):
590 """Schedules a restart event when any invocation is completed."""
591 self.is_restart_requested = True
592
593 def invocation_completion(self):
594 """Callback when an invocation is completed."""
595 if self.is_restart_requested:
596 logging.info('Restart by scheduled event.')
597 self.is_restart_requested = False
598 self.restart_tests()
599 else:
600 self.run_next_test()
601
Jon Salz0697cbf2012-07-04 15:14:04 +0800602 def run_next_test(self):
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800603 """Runs the next eligible test.
henryhsu4cc6b022014-04-22 17:12:42 +0800604
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800605 self.test_list_iterator (a TestListIterator object) will determine which
606 test should be run.
henryhsu4cc6b022014-04-22 17:12:42 +0800607 """
Jon Salz0697cbf2012-07-04 15:14:04 +0800608 self.reap_completed_tests()
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800609
610 if self.invocations:
611 # there are tests still running, we cannot start new tests
Vic Yangaabf9fd2013-04-09 18:56:13 +0800612 return
Jon Salz94eb56f2012-06-12 18:01:12 +0800613
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800614 if self.check_critical_factory_note():
615 logging.info('has critical factory note, stop running')
616 self.test_list_iterator.stop()
617 return
Jon Salz94eb56f2012-06-12 18:01:12 +0800618
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800619 while True:
620 try:
621 path = self.test_list_iterator.next()
622 test = self.test_list.lookup_path(path)
623 except StopIteration:
624 logging.info('no next test, stop running')
Jon Salz0697cbf2012-07-04 15:14:04 +0800625 return
Jon Salz94eb56f2012-06-12 18:01:12 +0800626
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800627 # check if we have run all required tests
Jon Salz304a75d2012-07-06 11:14:15 +0800628 untested = set()
Jon Salza1412922012-07-23 16:04:17 +0800629 for requirement in test.require_run:
630 for i in requirement.test.walk():
631 if i == test:
Jon Salz304a75d2012-07-06 11:14:15 +0800632 # We've hit this test itself; stop checking
633 break
Jon Salza1412922012-07-23 16:04:17 +0800634 if ((i.get_state().status == TestState.UNTESTED) or
635 (requirement.passed and i.get_state().status !=
636 TestState.PASSED)):
Jon Salz304a75d2012-07-06 11:14:15 +0800637 # Found an untested test; move on to the next
638 # element in require_run.
Jon Salza1412922012-07-23 16:04:17 +0800639 untested.add(i)
Jon Salz304a75d2012-07-06 11:14:15 +0800640 break
641
642 if untested:
643 untested_paths = ', '.join(sorted([x.path for x in untested]))
644 if self.state_instance.get_shared_data('engineering_mode',
645 optional=True):
646 # In engineering mode, we'll let it go.
647 factory.console.warn('In engineering mode; running '
648 '%s even though required tests '
649 '[%s] have not completed',
650 test.path, untested_paths)
651 else:
652 # Not in engineering mode; mark it failed.
653 error_msg = ('Required tests [%s] have not been run yet'
654 % untested_paths)
655 factory.console.error('Not running %s: %s',
656 test.path, error_msg)
657 test.update_state(status=TestState.FAILED,
658 error_msg=error_msg)
659 continue
660
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800661 # okay, let's run the test
Ricky Liang48e47f92014-02-26 19:31:51 +0800662 if (isinstance(test, factory.ShutdownStep) and
Ricky Liangb7eb8772014-09-15 18:05:22 +0800663 self.state_instance.get_shared_data(
664 state.POST_SHUTDOWN_TAG % test.path, optional=True)):
Ricky Liang48e47f92014-02-26 19:31:51 +0800665 # Invoking post shutdown method of shutdown test. We should retain the
666 # iterations_left and retries_left of the original test state.
667 test_state = self.state_instance.get_test_state(test.path)
668 self._run_test(test, test_state.iterations_left,
669 test_state.retries_left)
670 else:
671 # Starts a new test run; reset iterations and retries.
672 self._run_test(test, test.iterations, test.retries)
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800673 return # to leave while
Jon Salz1acc8742012-07-17 17:45:55 +0800674
Cheng-Yi Chiangce05c002013-04-04 02:13:17 +0800675 def _run_test(self, test, iterations_left=None, retries_left=None):
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800676 """Invokes the test.
677
678 The argument `test` should be either a leaf test (no subtests) or a parallel
679 test (all subtests should be run in parallel).
680 """
Vic Yanga3cecf82014-12-26 00:44:21 -0800681 if not self._ui_initialized and not test.is_no_host():
682 self.init_ui()
Jon Salz1acc8742012-07-17 17:45:55 +0800683
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800684 if test.is_leaf():
685 invoc = TestInvocation(
686 self, test, on_completion=self.invocation_completion,
687 on_test_failure=lambda: self.test_fail(test))
688 new_state = test.update_state(
689 status=TestState.ACTIVE, increment_count=1, error_msg='',
690 invocation=invoc.uuid, iterations_left=iterations_left,
691 retries_left=retries_left,
692 visible=(self.visible_test == test))
693 invoc.count = new_state.count
694 self.invocations[test] = invoc
695 if self.visible_test is None and test.has_ui:
696 self.set_visible_test(test)
697 self.check_plugins()
698 invoc.start()
699 else:
700 assert test.is_parallel()
701 for subtest in test.subtests:
702 # TODO(stimim): what if the subtests *must* be run in parallel?
703 # for example, stressapptest and countdown test.
704
705 # Make sure we don't need to skip it:
706 if not self.test_list_iterator.check_skip(subtest):
707 self._run_test(subtest, subtest.iterations, subtest.retries)
Jon Salz5f2a0672012-05-22 17:14:06 +0800708
Earl Oua3bca122016-10-21 16:00:30 +0800709 def check_plugins(self):
710 """Check plugins to be paused or resumed."""
711 exclusive_resources = set()
712 for test in self.invocations:
713 exclusive_resources = exclusive_resources.union(
714 test.get_exclusive_resources())
715 self.plugin_controller.PauseAndResumePluginByResource(exclusive_resources)
716
cychiang21886742012-07-05 15:16:32 +0800717 def check_for_updates(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800718 """Schedules an asynchronous check for updates if necessary."""
cychiang21886742012-07-05 15:16:32 +0800719 if not self.test_list.options.update_period_secs:
720 # Not enabled.
721 return
722
723 now = time.time()
724 if self.last_update_check and (
725 now - self.last_update_check <
726 self.test_list.options.update_period_secs):
727 # Not yet time for another check.
728 return
729
730 self.last_update_check = now
731
732 def handle_check_for_update(reached_shopfloor, md5sum, needs_update):
733 if reached_shopfloor:
734 new_update_md5sum = md5sum if needs_update else None
Hung-Te Line594e5d2015-12-16 02:36:05 +0800735 if self.dut.info.update_md5sum != new_update_md5sum:
cychiang21886742012-07-05 15:16:32 +0800736 logging.info('Received new update MD5SUM: %s', new_update_md5sum)
Hung-Te Line594e5d2015-12-16 02:36:05 +0800737 self.dut.info.Overrides('update_md5sum', new_update_md5sum)
Peter Ammon1e1ec572014-06-26 17:56:32 -0700738 self.run_enqueue(self.update_system_info)
Cheng-Yi Chiang194d3c02015-03-16 14:37:15 +0800739 else:
740 if not self._suppress_periodic_update_messages:
741 logging.warning('Suppress error messages for periodic update checking'
742 ' after the first one.')
743 self._suppress_periodic_update_messages = True
cychiang21886742012-07-05 15:16:32 +0800744
745 updater.CheckForUpdateAsync(
Ricky Liang45c73e72015-01-15 15:00:30 +0800746 handle_check_for_update,
Cheng-Yi Chiang194d3c02015-03-16 14:37:15 +0800747 self.test_list.options.shopfloor_timeout_secs,
748 self._suppress_periodic_update_messages)
cychiang21886742012-07-05 15:16:32 +0800749
Jon Salza6711d72012-07-18 14:33:03 +0800750 def cancel_pending_tests(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800751 """Cancels any tests in the run queue."""
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800752 self.run_tests(None)
Jon Salza6711d72012-07-18 14:33:03 +0800753
Ricky Liang4bff3e32014-02-20 18:46:11 +0800754 def restore_active_run_state(self):
755 """Restores active run id and the list of scheduled tests."""
756 self.run_id = self.state_instance.get_shared_data('run_id', optional=True)
757 self.scheduled_run_tests = self.state_instance.get_shared_data(
758 'scheduled_run_tests', optional=True)
759
760 def set_active_run_state(self):
761 """Sets active run id and the list of scheduled tests."""
762 self.run_id = str(uuid.uuid4())
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800763 # try our best to predict which tests will be run.
764 self.scheduled_run_tests = self.test_list_iterator.get_pending_tests()
Ricky Liang4bff3e32014-02-20 18:46:11 +0800765 self.state_instance.set_shared_data('run_id', self.run_id)
766 self.state_instance.set_shared_data('scheduled_run_tests',
767 self.scheduled_run_tests)
768
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800769 def run_tests(self, subtree, status_filter=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800770 """Runs tests under subtree.
Jon Salz258a40c2012-04-19 12:34:01 +0800771
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800772 Run tests under a given subtree.
Jon Salzb1b39092012-05-03 02:05:09 +0800773
Ricky Liang6fe218c2013-12-27 15:17:17 +0800774 Args:
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800775 subtree: root of subtree to run or None to run nothing.
Chih-Yu Huang85dc63c2015-08-12 15:21:28 +0800776 status_filter: List of available test states. Only run the tests which
777 states are in the list. Set to None if all test states are available.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800778 """
Hung-Te Lin410f70a2015-12-15 14:53:42 +0800779 self.dut.hooks.OnTestStart()
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800780 self.test_list_iterator = TestListIterator(
781 subtree, status_filter, self.test_list)
782 if subtree is not None:
Ricky Liang4bff3e32014-02-20 18:46:11 +0800783 self.set_active_run_state()
Jon Salz0697cbf2012-07-04 15:14:04 +0800784 self.run_next_test()
Hung-Te Linf2f78f72012-02-08 19:27:11 +0800785
Jon Salz0697cbf2012-07-04 15:14:04 +0800786 def reap_completed_tests(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800787 """Removes completed tests from the set of active tests.
Jon Salz0697cbf2012-07-04 15:14:04 +0800788
789 Also updates the visible test if it was reaped.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800790 """
Cheng-Yi Chiang5ac22ca2013-04-12 17:45:26 +0800791 test_completed = False
Jon Salz0697cbf2012-07-04 15:14:04 +0800792 for t, v in dict(self.invocations).iteritems():
793 if v.is_completed():
Cheng-Yi Chiang5ac22ca2013-04-12 17:45:26 +0800794 test_completed = True
Jon Salz1acc8742012-07-17 17:45:55 +0800795 new_state = t.update_state(**v.update_state_on_completion)
Jon Salz0697cbf2012-07-04 15:14:04 +0800796 del self.invocations[t]
797
Johny Lin62ed2a32015-05-13 11:57:12 +0800798 # Stop on failure if flag is true and there is no retry chances.
Chun-Ta Lin54e17e42012-09-06 22:05:13 +0800799 if (self.test_list.options.stop_on_failure and
Johny Lin62ed2a32015-05-13 11:57:12 +0800800 new_state.retries_left < 0 and
Chun-Ta Lin54e17e42012-09-06 22:05:13 +0800801 new_state.status == TestState.FAILED):
802 # Clean all the tests to cause goofy to stop.
Ricky Liang45c73e72015-01-15 15:00:30 +0800803 factory.console.info('Stop on failure triggered. Empty the queue.')
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800804 self.cancel_pending_tests()
Chun-Ta Lin54e17e42012-09-06 22:05:13 +0800805
Jon Salz1acc8742012-07-17 17:45:55 +0800806 if new_state.iterations_left and new_state.status == TestState.PASSED:
807 # Play it again, Sam!
808 self._run_test(t)
Cheng-Yi Chiangce05c002013-04-04 02:13:17 +0800809 # new_state.retries_left is obtained after update.
810 # For retries_left == 0, test can still be run for the last time.
811 elif (new_state.retries_left >= 0 and
812 new_state.status == TestState.FAILED):
813 # Still have to retry, Sam!
814 self._run_test(t)
Jon Salz1acc8742012-07-17 17:45:55 +0800815
Cheng-Yi Chiang5ac22ca2013-04-12 17:45:26 +0800816 if test_completed:
Vic Yangf01c59f2013-04-19 17:37:56 +0800817 self.log_watcher.KickWatchThread()
Cheng-Yi Chiang5ac22ca2013-04-12 17:45:26 +0800818
Jon Salz0697cbf2012-07-04 15:14:04 +0800819 if (self.visible_test is None or
Jon Salz85a39882012-07-05 16:45:04 +0800820 self.visible_test not in self.invocations):
Jon Salz0697cbf2012-07-04 15:14:04 +0800821 self.set_visible_test(None)
822 # Make the first running test, if any, the visible test
823 for t in self.test_list.walk():
824 if t in self.invocations:
825 self.set_visible_test(t)
826 break
827
Jon Salz6dc031d2013-06-19 13:06:23 +0800828 def kill_active_tests(self, abort, root=None, reason=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800829 """Kills and waits for all active tests.
Jon Salz0697cbf2012-07-04 15:14:04 +0800830
Jon Salz85a39882012-07-05 16:45:04 +0800831 Args:
832 abort: True to change state of killed tests to FAILED, False for
Jon Salz0697cbf2012-07-04 15:14:04 +0800833 UNTESTED.
Jon Salz85a39882012-07-05 16:45:04 +0800834 root: If set, only kills tests with root as an ancestor.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800835 reason: If set, the abort reason.
836 """
Jon Salz0697cbf2012-07-04 15:14:04 +0800837 self.reap_completed_tests()
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800838 # since we remove objects while iterating, make a copy
839 for test, invoc in dict(self.invocations).iteritems():
Jon Salz85a39882012-07-05 16:45:04 +0800840 if root and not test.has_ancestor(root):
841 continue
842
Ricky Liang45c73e72015-01-15 15:00:30 +0800843 factory.console.info('Killing active test %s...', test.path)
Jon Salz6dc031d2013-06-19 13:06:23 +0800844 invoc.abort_and_join(reason)
Ricky Liang45c73e72015-01-15 15:00:30 +0800845 factory.console.info('Killed %s', test.path)
Jon Salz1acc8742012-07-17 17:45:55 +0800846 test.update_state(**invoc.update_state_on_completion)
Jon Salz0697cbf2012-07-04 15:14:04 +0800847 del self.invocations[test]
Jon Salz1acc8742012-07-17 17:45:55 +0800848
Jon Salz0697cbf2012-07-04 15:14:04 +0800849 if not abort:
850 test.update_state(status=TestState.UNTESTED)
851 self.reap_completed_tests()
852
Jon Salz6dc031d2013-06-19 13:06:23 +0800853 def stop(self, root=None, fail=False, reason=None):
854 self.kill_active_tests(fail, root, reason)
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800855
856 if not root:
857 self.test_list_iterator.stop()
858 else:
859 # only skip tests under `root`
860 self.test_list_iterator = itertools.dropwhile(
861 lambda path: self.test_list.lookup_path(path).has_ancestor(root),
862 self.test_list_iterator)
Jon Salz85a39882012-07-05 16:45:04 +0800863 self.run_next_test()
Jon Salz0697cbf2012-07-04 15:14:04 +0800864
Jon Salz4712ac72013-02-07 17:12:05 +0800865 def clear_state(self, root=None):
Jon Salzd7550792013-07-12 05:49:27 +0800866 if root is None:
867 root = self.test_list
Jon Salz6dc031d2013-06-19 13:06:23 +0800868 self.stop(root, reason='Clearing test state')
Jon Salz4712ac72013-02-07 17:12:05 +0800869 for f in root.walk():
870 if f.is_leaf():
871 f.update_state(status=TestState.UNTESTED)
872
Jon Salz6dc031d2013-06-19 13:06:23 +0800873 def abort_active_tests(self, reason=None):
874 self.kill_active_tests(True, reason=reason)
Jon Salz0697cbf2012-07-04 15:14:04 +0800875
876 def main(self):
Jon Salzeff94182013-06-19 15:06:28 +0800877 syslog.openlog('goofy')
878
Jon Salz0697cbf2012-07-04 15:14:04 +0800879 try:
Jon Salzd7550792013-07-12 05:49:27 +0800880 self.status = Status.INITIALIZING
Jon Salz0697cbf2012-07-04 15:14:04 +0800881 self.init()
882 self.event_log.Log('goofy_init',
Ricky Liang45c73e72015-01-15 15:00:30 +0800883 success=True)
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800884 testlog.Log(
Joel Kitching9eb203a2016-04-21 15:36:30 +0800885 testlog.StationInit({
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800886 'stationDeviceId': testlog_goofy.GetDeviceID(),
Joel Kitching21bc69b2016-07-13 08:29:52 -0700887 'stationInstallationId': testlog_goofy.GetInstallationID(),
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800888 'count': testlog_goofy.GetInitCount(),
Joel Kitching9eb203a2016-04-21 15:36:30 +0800889 'success': True}))
Jon Salz0697cbf2012-07-04 15:14:04 +0800890 except:
Joel Kitching9eb203a2016-04-21 15:36:30 +0800891 try:
892 if self.event_log:
Jon Salz0697cbf2012-07-04 15:14:04 +0800893 self.event_log.Log('goofy_init',
Ricky Liang45c73e72015-01-15 15:00:30 +0800894 success=False,
895 trace=traceback.format_exc())
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800896 if self.testlog:
897 testlog.Log(
Joel Kitching9eb203a2016-04-21 15:36:30 +0800898 testlog.StationInit({
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800899 'stationDeviceId': testlog_goofy.GetDeviceID(),
Joel Kitching21bc69b2016-07-13 08:29:52 -0700900 'stationInstallationId': testlog_goofy.GetInstallationID(),
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800901 'count': testlog_goofy.GetInitCount(),
Joel Kitching9eb203a2016-04-21 15:36:30 +0800902 'success': False,
903 'failureMessage': traceback.format_exc()}))
904 except: # pylint: disable=W0702
905 pass
Jon Salz0697cbf2012-07-04 15:14:04 +0800906 raise
907
Jon Salzd7550792013-07-12 05:49:27 +0800908 self.status = Status.RUNNING
Jon Salzeff94182013-06-19 15:06:28 +0800909 syslog.syslog('Goofy (factory test harness) starting')
Chun-Ta Lin5d12b592015-06-30 00:54:23 -0700910 syslog.syslog('Boot sequence = %d' % GetBootSequence())
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +0800911 syslog.syslog('Goofy init count = %d' % testlog_goofy.GetInitCount())
Jon Salz0697cbf2012-07-04 15:14:04 +0800912 self.run()
913
914 def update_system_info(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800915 """Updates system info."""
Hung-Te Line594e5d2015-12-16 02:36:05 +0800916 info = self.dut.info.GetAll()
917 self.state_instance.set_shared_data('system_info', info)
Jon Salz0697cbf2012-07-04 15:14:04 +0800918 self.event_client.post_event(Event(Event.Type.SYSTEM_INFO,
Hung-Te Line594e5d2015-12-16 02:36:05 +0800919 system_info=info))
920 logging.info('System info: %r', info)
Jon Salz0697cbf2012-07-04 15:14:04 +0800921
Jon Salzeb42f0d2012-07-27 19:14:04 +0800922 def update_factory(self, auto_run_on_restart=False, post_update_hook=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800923 """Commences updating factory software.
Jon Salzeb42f0d2012-07-27 19:14:04 +0800924
925 Args:
926 auto_run_on_restart: Auto-run when the machine comes back up.
927 post_update_hook: Code to call after update but immediately before
928 restart.
929
930 Returns:
931 Never if the update was successful (we just reboot).
932 False if the update was unnecessary (no update available).
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +0800933 """
Jon Salz6dc031d2013-06-19 13:06:23 +0800934 self.kill_active_tests(False, reason='Factory software update')
Jon Salza6711d72012-07-18 14:33:03 +0800935 self.cancel_pending_tests()
Jon Salz0697cbf2012-07-04 15:14:04 +0800936
Jon Salz5c344f62012-07-13 14:31:16 +0800937 def pre_update_hook():
938 if auto_run_on_restart:
Wei-Han Chenc17b4112016-11-22 14:56:51 +0800939 self.state_instance.set_shared_data(TESTS_AFTER_SHUTDOWN,
Jon Salz5c344f62012-07-13 14:31:16 +0800940 FORCE_AUTO_RUN)
941 self.state_instance.close()
942
Jon Salzeb42f0d2012-07-27 19:14:04 +0800943 if updater.TryUpdate(pre_update_hook=pre_update_hook):
944 if post_update_hook:
945 post_update_hook()
946 self.env.shutdown('reboot')
Jon Salz0697cbf2012-07-04 15:14:04 +0800947
Ricky Liang8fecf412014-05-22 10:56:14 +0800948 def handle_sigint(self, dummy_signum, dummy_frame): # pylint: disable=W0613
Jon Salz77c151e2012-08-28 07:20:37 +0800949 logging.error('Received SIGINT')
Peter Ammon1e1ec572014-06-26 17:56:32 -0700950 self.run_enqueue(None)
Jon Salz77c151e2012-08-28 07:20:37 +0800951 raise KeyboardInterrupt()
952
Ricky Liang8fecf412014-05-22 10:56:14 +0800953 def handle_sigterm(self, dummy_signum, dummy_frame): # pylint: disable=W0613
954 logging.error('Received SIGTERM')
Hung-Te Lin94ca4742014-07-09 20:13:50 +0800955 self.env.terminate()
956 self.run_queue.put(None)
Ricky Liang8fecf412014-05-22 10:56:14 +0800957 raise RuntimeError('Received SIGTERM')
958
Jon Salze12c2b32013-06-25 16:24:34 +0800959 def find_kcrashes(self):
960 """Finds kcrash files, logs them, and marks them as seen."""
961 seen_crashes = set(
962 self.state_instance.get_shared_data('seen_crashes', optional=True)
963 or [])
964
965 for path in glob.glob('/var/spool/crash/*'):
966 if not os.path.isfile(path):
967 continue
968 if path in seen_crashes:
969 continue
970 try:
971 stat = os.stat(path)
Hung-Te Lin4e6357c2016-01-08 14:32:00 +0800972 mtime = time_utils.TimeString(stat.st_mtime)
Jon Salze12c2b32013-06-25 16:24:34 +0800973 logging.info(
974 'Found new crash file %s (%d bytes at %s)',
975 path, stat.st_size, mtime)
976 extra_log_args = {}
977
978 try:
979 _, ext = os.path.splitext(path)
980 if ext in ['.kcrash', '.meta']:
981 ext = ext.replace('.', '')
982 with open(path) as f:
983 data = f.read(MAX_CRASH_FILE_SIZE)
984 tell = f.tell()
985 logging.info(
986 'Contents of %s%s:%s',
987 path,
988 ('' if tell == stat.st_size
989 else '(truncated to %d bytes)' % MAX_CRASH_FILE_SIZE),
990 ('\n' + data).replace('\n', '\n ' + ext + '> '))
991 extra_log_args['data'] = data
992
993 # Copy to /var/factory/kcrash for posterity
Joel Kitching625ff0f2016-05-16 14:59:40 -0700994 kcrash_dir = paths.GetFactoryRoot('kcrash')
Hung-Te Lin4e6357c2016-01-08 14:32:00 +0800995 file_utils.TryMakeDirs(kcrash_dir)
Jon Salze12c2b32013-06-25 16:24:34 +0800996 shutil.copy(path, kcrash_dir)
997 logging.info('Copied to %s',
998 os.path.join(kcrash_dir, os.path.basename(path)))
999 finally:
1000 # Even if something goes wrong with the above, still try to
1001 # log to event log
1002 self.event_log.Log('crash_file',
1003 path=path, size=stat.st_size, mtime=mtime,
1004 **extra_log_args)
1005 except: # pylint: disable=W0702
1006 logging.exception('Unable to handle crash files %s', path)
1007 seen_crashes.add(path)
1008
1009 self.state_instance.set_shared_data('seen_crashes', list(seen_crashes))
1010
Jon Salz128b0932013-07-03 16:55:26 +08001011 def GetTestList(self, test_list_id):
1012 """Returns the test list with the given ID.
1013
1014 Raises:
1015 TestListError: The test list ID is not valid.
1016 """
1017 try:
1018 return self.test_lists[test_list_id]
1019 except KeyError:
1020 raise test_lists.TestListError(
1021 '%r is not a valid test list ID (available IDs are [%s])' % (
1022 test_list_id, ', '.join(sorted(self.test_lists.keys()))))
1023
1024 def InitTestLists(self):
Joel Kitching50a63ea2016-02-22 13:15:09 +08001025 """Reads in all test lists and sets the active test list.
1026
1027 Returns:
1028 True if the active test list could be set, False if failed.
1029 """
1030 startup_errors = []
1031 self.test_lists, failed_files = test_lists.BuildAllTestLists(
Ricky Liang27051552014-05-04 14:22:26 +08001032 force_generic=(self.options.automation_mode is not None))
Jon Salzd7550792013-07-12 05:49:27 +08001033 logging.info('Loaded test lists: [%s]',
1034 test_lists.DescribeTestLists(self.test_lists))
Jon Salz128b0932013-07-03 16:55:26 +08001035
Joel Kitching50a63ea2016-02-22 13:15:09 +08001036 # Check for any syntax errors in test list files.
1037 if failed_files:
1038 logging.info('Failed test list files: [%s]',
1039 ' '.join(failed_files.keys()))
1040 for f, exc_info in failed_files.iteritems():
1041 logging.error('Error in test list file: %s', f,
1042 exc_info=exc_info)
1043
1044 # Limit the stack trace to the very last entry.
1045 exc_type, exc_value, exc_traceback = exc_info
1046 while exc_traceback and exc_traceback.tb_next:
1047 exc_traceback = exc_traceback.tb_next
1048
1049 exc_string = ''.join(
1050 traceback.format_exception(
1051 exc_type, exc_value, exc_traceback)).rstrip()
1052 startup_errors.append('Error in test list file (%s):\n%s'
1053 % (f, exc_string))
1054
Jon Salz128b0932013-07-03 16:55:26 +08001055 if not self.options.test_list:
1056 self.options.test_list = test_lists.GetActiveTestListId()
1057
Joel Kitching50a63ea2016-02-22 13:15:09 +08001058 # Check for a non-existent test list ID.
1059 try:
Wei-Han Chen84fee7c2016-08-26 21:56:25 +08001060 self.test_list = self.GetTestList(self.options.test_list)
Joel Kitching50a63ea2016-02-22 13:15:09 +08001061 logging.info('Active test list: %s', self.test_list.test_list_id)
1062 except test_lists.TestListError as e:
1063 logging.exception('Invalid active test list: %s',
1064 self.options.test_list)
1065 startup_errors.append(e.message)
Jon Salz128b0932013-07-03 16:55:26 +08001066
Joel Kitching50a63ea2016-02-22 13:15:09 +08001067 # We may have failed loading the active test list.
1068 if self.test_list:
Joel Kitching50a63ea2016-02-22 13:15:09 +08001069 self.test_list.state_instance = self.state_instance
Jon Salz128b0932013-07-03 16:55:26 +08001070
Joel Kitching50a63ea2016-02-22 13:15:09 +08001071 # Prepare DUT link.
1072 if self.test_list.options.dut_options:
1073 logging.info('dut_options set by %s: %r', self.test_list.test_list_id,
1074 self.test_list.options.dut_options)
Hung-Te Linb6287242016-05-18 14:39:05 +08001075 device_utils.PrepareDUTLink(**self.test_list.options.dut_options)
Wei-Han Chene8a025f2016-01-14 16:42:02 +08001076
Joel Kitching50a63ea2016-02-22 13:15:09 +08001077 # Show all startup errors.
1078 if startup_errors:
1079 self.state_instance.set_shared_data(
1080 'startup_error', '\n\n'.join(startup_errors))
1081
1082 # Only return False if failed to load the active test list.
1083 return bool(self.test_list)
Jon Salz128b0932013-07-03 16:55:26 +08001084
Shuo-Peng Liao268b40b2013-07-01 15:58:59 +08001085 def init_hooks(self):
1086 """Initializes hooks.
1087
1088 Must run after self.test_list ready.
1089 """
Shuo-Peng Liao52b90da2013-06-30 17:00:06 +08001090 module, cls = self.test_list.options.hooks_class.rsplit('.', 1)
1091 self.hooks = getattr(__import__(module, fromlist=[cls]), cls)()
1092 assert isinstance(self.hooks, factory.Hooks), (
Ricky Liang45c73e72015-01-15 15:00:30 +08001093 'hooks should be of type Hooks but is %r' % type(self.hooks))
Shuo-Peng Liao52b90da2013-06-30 17:00:06 +08001094 self.hooks.test_list = self.test_list
Shuo-Peng Liao268b40b2013-07-01 15:58:59 +08001095 self.hooks.OnCreatedTestList()
Shuo-Peng Liao52b90da2013-06-30 17:00:06 +08001096
Vic Yanga3cecf82014-12-26 00:44:21 -08001097 def init_ui(self):
1098 """Initialize UI."""
1099 self._ui_initialized = True
1100 if self.options.ui == 'chrome':
Hung-Te Lin8f6a3782015-01-06 22:58:32 +08001101 if self.options.monolithic:
Hung-Te Lin7bd55312014-12-30 16:43:36 +08001102 self.env.launch_chrome()
1103 else:
1104 # The presenter is responsible for launching Chrome. Let's just
1105 # wait here.
1106 self.env.controller_ready_for_ui()
Vic Yanga3cecf82014-12-26 00:44:21 -08001107 logging.info('Waiting for a web socket connection')
1108 self.web_socket_manager.wait()
1109
1110 # Wait for the test widget size to be set; this is done in
1111 # an asynchronous RPC so there is a small chance that the
1112 # web socket might be opened first.
1113 for _ in range(100): # 10 s
1114 try:
1115 if self.state_instance.get_shared_data('test_widget_size'):
1116 break
1117 except KeyError:
1118 pass # Retry
1119 time.sleep(0.1) # 100 ms
1120 else:
1121 logging.warn('Never received test_widget_size from UI')
1122
Jon Salz0697cbf2012-07-04 15:14:04 +08001123 def init(self, args=None, env=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001124 """Initializes Goofy.
Jon Salz0697cbf2012-07-04 15:14:04 +08001125
1126 Args:
1127 args: A list of command-line arguments. Uses sys.argv if
1128 args is None.
1129 env: An Environment instance to use (or None to choose
1130 FakeChrootEnvironment or DUTEnvironment as appropriate).
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001131 """
Jon Salz0697cbf2012-07-04 15:14:04 +08001132 parser = OptionParser()
1133 parser.add_option('-v', '--verbose', dest='verbose',
Jon Salz8fa8e832012-07-13 19:04:09 +08001134 action='store_true',
1135 help='Enable debug logging')
Jon Salz0697cbf2012-07-04 15:14:04 +08001136 parser.add_option('--print_test_list', dest='print_test_list',
Wei-Han Chen84fee7c2016-08-26 21:56:25 +08001137 metavar='TEST_LIST_ID',
1138 help='Print the content of TEST_LIST_ID and exit')
Jon Salz0697cbf2012-07-04 15:14:04 +08001139 parser.add_option('--restart', dest='restart',
Jon Salz8fa8e832012-07-13 19:04:09 +08001140 action='store_true',
1141 help='Clear all test state')
Jon Salz0697cbf2012-07-04 15:14:04 +08001142 parser.add_option('--ui', dest='ui', type='choice',
Jon Salz7b5482e2014-08-04 17:48:41 +08001143 choices=['none', 'chrome'],
Jon Salz2f881df2013-02-01 17:00:35 +08001144 default='chrome',
Jon Salz8fa8e832012-07-13 19:04:09 +08001145 help='UI to use')
Jon Salz0697cbf2012-07-04 15:14:04 +08001146 parser.add_option('--ui_scale_factor', dest='ui_scale_factor',
Jon Salz8fa8e832012-07-13 19:04:09 +08001147 type='int', default=1,
1148 help=('Factor by which to scale UI '
1149 '(Chrome UI only)'))
Jon Salz0697cbf2012-07-04 15:14:04 +08001150 parser.add_option('--test_list', dest='test_list',
Wei-Han Chen84fee7c2016-08-26 21:56:25 +08001151 metavar='TEST_LIST_ID',
1152 help='Use test list whose id is TEST_LIST_ID')
Jon Salzc79a9982012-08-30 04:42:01 +08001153 parser.add_option('--dummy_shopfloor', action='store_true',
1154 help='Use a dummy shopfloor server')
Ricky Liang6fe218c2013-12-27 15:17:17 +08001155 parser.add_option('--automation-mode',
1156 choices=[m.lower() for m in AutomationMode],
Ricky Liang45c73e72015-01-15 15:00:30 +08001157 default='none', help='Factory test automation mode.')
Ricky Liang117484a2014-04-14 11:14:41 +08001158 parser.add_option('--no-auto-run-on-start', dest='auto_run_on_start',
1159 action='store_false', default=True,
1160 help=('do not automatically run the test list on goofy '
1161 'start; this is only valid when factory test '
1162 'automation is enabled'))
Chun-Ta Lina8dd3172014-11-26 16:15:13 +08001163 parser.add_option('--handshake_timeout', dest='handshake_timeout',
1164 type='float', default=0.3,
1165 help=('RPC timeout when doing handshake between device '
1166 'and presenter.'))
Vic Yang7d693c42014-09-14 09:52:39 +08001167 parser.add_option('--standalone', dest='standalone',
1168 action='store_true', default=False,
1169 help=('Assume the presenter is running on the same '
1170 'machines.'))
Hung-Te Lin8f6a3782015-01-06 22:58:32 +08001171 parser.add_option('--monolithic', dest='monolithic',
1172 action='store_true', default=False,
1173 help='Run in monolithic mode (without presenter)')
Jon Salz0697cbf2012-07-04 15:14:04 +08001174 (self.options, self.args) = parser.parse_args(args)
1175
Hung-Te Lina846f602014-07-04 20:32:22 +08001176 signal.signal(signal.SIGINT, self.handle_sigint)
1177 # TODO(hungte) SIGTERM does not work properly without Telemetry and should
1178 # be fixed.
Hung-Te Lina846f602014-07-04 20:32:22 +08001179
Jon Salz46b89562012-07-05 11:49:22 +08001180 # Make sure factory directories exist.
Joel Kitching625ff0f2016-05-16 14:59:40 -07001181 paths.GetLogRoot()
1182 paths.GetStateRoot()
1183 paths.GetTestDataRoot()
Jon Salz46b89562012-07-05 11:49:22 +08001184
Jon Salz0697cbf2012-07-04 15:14:04 +08001185 global _inited_logging # pylint: disable=W0603
1186 if not _inited_logging:
1187 factory.init_logging('goofy', verbose=self.options.verbose)
1188 _inited_logging = True
Jon Salz8fa8e832012-07-13 19:04:09 +08001189
Wei-Han Chen78f35f62017-03-06 20:11:20 +08001190 try:
1191 goofy_default_options = config_utils.LoadConfig(validate_schema=False)
1192 for key, value in goofy_default_options.iteritems():
1193 if getattr(self.options, key, None) is None:
1194 logging.info('self.options.%s = %r', key, value)
1195 setattr(self.options, key, value)
1196 except Exception:
1197 logging.exception('failed to load goofy overriding options')
1198
Jon Salz0f996602012-10-03 15:26:48 +08001199 if self.options.print_test_list:
Wei-Han Chen84fee7c2016-08-26 21:56:25 +08001200 test_list = test_lists.BuildTestList(self.options.print_test_list)
1201 print(test_list.__repr__(recursive=True))
Jon Salz0f996602012-10-03 15:26:48 +08001202 sys.exit(0)
1203
Jon Salzee85d522012-07-17 14:34:46 +08001204 event_log.IncrementBootSequence()
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +08001205 testlog_goofy.IncrementInitCount()
1206
Jon Salzd15bbcf2013-05-21 17:33:57 +08001207 # Don't defer logging the initial event, so we can make sure
1208 # that device_id, reimage_id, etc. are all set up.
1209 self.event_log = EventLog('goofy', defer=False)
Chun-Ta Lin53cbbd52016-06-08 21:42:19 +08001210 self.testlog = testlog.Testlog(
1211 log_root=paths.GetLogRoot(), uuid=self.uuid)
1212 # Direct the logging calls to testlog as well.
1213 testlog.CapturePythonLogging(
1214 callback=self.testlog.primary_json.Log,
1215 level=logging.getLogger().getEffectiveLevel())
Jon Salz0697cbf2012-07-04 15:14:04 +08001216
Jon Salz0697cbf2012-07-04 15:14:04 +08001217 if env:
1218 self.env = env
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +08001219 elif sys_utils.InChroot():
Jon Salz0697cbf2012-07-04 15:14:04 +08001220 self.env = test_environment.FakeChrootEnvironment()
1221 logging.warn(
Ricky Liang45c73e72015-01-15 15:00:30 +08001222 'Using chroot environment: will not actually run autotests')
Hung-Te Lina846f602014-07-04 20:32:22 +08001223 elif self.options.ui == 'chrome':
Ricky Liang09d66d82014-09-25 11:20:54 +08001224 self.env = test_environment.DUTEnvironment()
Jon Salz0697cbf2012-07-04 15:14:04 +08001225 self.env.goofy = self
Vic Yanga4931152014-08-11 16:36:24 -07001226 # web_socket_manager will be initialized later
1227 # pylint: disable=W0108
1228 self.env.has_sockets = lambda: self.web_socket_manager.has_sockets()
Jon Salz0697cbf2012-07-04 15:14:04 +08001229
1230 if self.options.restart:
1231 state.clear_state()
1232
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +08001233 if self.options.ui_scale_factor != 1 and sys_utils.InQEMU():
Jon Salz0697cbf2012-07-04 15:14:04 +08001234 logging.warn(
Ricky Liang45c73e72015-01-15 15:00:30 +08001235 'In QEMU; ignoring ui_scale_factor argument')
Jon Salz0697cbf2012-07-04 15:14:04 +08001236 self.options.ui_scale_factor = 1
1237
1238 logging.info('Started')
1239
Hung-Te Lin8f6a3782015-01-06 22:58:32 +08001240 if not self.options.monolithic:
Hung-Te Lin7bd55312014-12-30 16:43:36 +08001241 self.link_manager = PresenterLinkManager(
1242 check_interval=1,
1243 handshake_timeout=self.options.handshake_timeout,
1244 standalone=self.options.standalone)
Peter Ammon1e1ec572014-06-26 17:56:32 -07001245
Earl Ouacbe99c2017-02-21 16:04:19 +08001246 self.start_goofy_server()
1247 self.init_state_instance()
Jon Salz0697cbf2012-07-04 15:14:04 +08001248 self.state_instance.set_shared_data('hwid_cfg', get_hwid_cfg())
1249 self.state_instance.set_shared_data('ui_scale_factor',
Ricky Liang09216dc2013-02-22 17:26:45 +08001250 self.options.ui_scale_factor)
Jon Salz0697cbf2012-07-04 15:14:04 +08001251 self.last_shutdown_time = (
Ricky Liang45c73e72015-01-15 15:00:30 +08001252 self.state_instance.get_shared_data('shutdown_time', optional=True))
Jon Salz0697cbf2012-07-04 15:14:04 +08001253 self.state_instance.del_shared_data('shutdown_time', optional=True)
Jon Salzb19ea072013-02-07 16:35:00 +08001254 self.state_instance.del_shared_data('startup_error', optional=True)
Jon Salz0697cbf2012-07-04 15:14:04 +08001255
Ricky Liang6fe218c2013-12-27 15:17:17 +08001256 self.options.automation_mode = ParseAutomationMode(
1257 self.options.automation_mode)
1258 self.state_instance.set_shared_data('automation_mode',
1259 self.options.automation_mode)
1260 self.state_instance.set_shared_data(
1261 'automation_mode_prompt',
1262 AutomationModePrompt[self.options.automation_mode])
1263
Joel Kitching50a63ea2016-02-22 13:15:09 +08001264 success = False
1265 exc_info = None
Jon Salz128b0932013-07-03 16:55:26 +08001266 try:
Joel Kitching50a63ea2016-02-22 13:15:09 +08001267 success = self.InitTestLists()
Jon Salz128b0932013-07-03 16:55:26 +08001268 except: # pylint: disable=W0702
Joel Kitching50a63ea2016-02-22 13:15:09 +08001269 exc_info = sys.exc_info()
1270
1271 if not success:
1272 if exc_info:
1273 logging.exception('Unable to initialize test lists')
1274 self.state_instance.set_shared_data(
1275 'startup_error',
1276 'Unable to initialize test lists\n%s' % (
1277 traceback.format_exc()))
Jon Salzb19ea072013-02-07 16:35:00 +08001278 if self.options.ui == 'chrome':
1279 # Create an empty test list with default options so that the rest of
1280 # startup can proceed.
1281 self.test_list = factory.FactoryTestList(
1282 [], self.state_instance, factory.Options())
1283 else:
1284 # Bail with an error; no point in starting up.
1285 sys.exit('No valid test list; exiting.')
1286
Shuo-Peng Liao268b40b2013-07-01 15:58:59 +08001287 self.init_hooks()
1288
Jon Salz822838b2013-03-25 17:32:33 +08001289 if self.test_list.options.clear_state_on_start:
1290 self.state_instance.clear_test_state()
1291
Jon Salz670ce062014-05-16 15:53:50 +08001292 # If the phase is invalid, this will raise a ValueError.
1293 phase.SetPersistentPhase(self.test_list.options.phase)
1294
Dean Liao85ca86f2014-11-03 12:28:08 +08001295 # For netboot firmware, mainfw_type should be 'netboot'.
Hung-Te Line594e5d2015-12-16 02:36:05 +08001296 if (self.dut.info.mainfw_type != 'nonchrome' and
1297 self.dut.info.firmware_version is None):
Ricky Liang45c73e72015-01-15 15:00:30 +08001298 self.state_instance.set_shared_data(
1299 'startup_error',
Vic Yang9bd4f772013-06-04 17:34:00 +08001300 'Netboot firmware detected\n'
1301 'Connect Ethernet and reboot to re-image.\n'
1302 u'侦测到网路开机固件\n'
1303 u'请连接乙太网并重启')
1304
Jon Salz0697cbf2012-07-04 15:14:04 +08001305 if not self.state_instance.has_shared_data('ui_lang'):
1306 self.state_instance.set_shared_data('ui_lang',
Ricky Liang45c73e72015-01-15 15:00:30 +08001307 self.test_list.options.ui_lang)
Jon Salz0697cbf2012-07-04 15:14:04 +08001308 self.state_instance.set_shared_data(
Ricky Liang45c73e72015-01-15 15:00:30 +08001309 'test_list_options',
1310 self.test_list.options.__dict__)
Jon Salz0697cbf2012-07-04 15:14:04 +08001311 self.state_instance.test_list = self.test_list
1312
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001313 self.check_log_rotation()
Jon Salz83ef34b2012-11-01 19:46:35 +08001314
Jon Salz23926422012-09-01 03:38:13 +08001315 if self.options.dummy_shopfloor:
Ricky Liang45c73e72015-01-15 15:00:30 +08001316 os.environ[shopfloor.SHOPFLOOR_SERVER_ENV_VAR_NAME] = (
1317 'http://%s:%d/' %
Joel Kitchingb85ed7f2014-10-08 18:24:39 +08001318 (net_utils.LOCALHOST, shopfloor.DEFAULT_SERVER_PORT))
Hung-Te Lin4e6357c2016-01-08 14:32:00 +08001319 self.dummy_shopfloor = process_utils.Spawn(
Wei-Han Chen2ebb92d2016-01-12 14:51:41 +08001320 [os.path.join(paths.FACTORY_PATH, 'bin', 'shopfloor_server'),
Jon Salz23926422012-09-01 03:38:13 +08001321 '--dummy'])
1322 elif self.test_list.options.shopfloor_server_url:
1323 shopfloor.set_server_url(self.test_list.options.shopfloor_server_url)
Jon Salz2bf2f6b2013-03-28 18:49:26 +08001324 shopfloor.set_enabled(True)
Jon Salz23926422012-09-01 03:38:13 +08001325
Jon Salz0697cbf2012-07-04 15:14:04 +08001326 self.init_states()
1327 self.start_event_server()
Wei-Ning Huang38b75f02015-02-25 18:25:14 +08001328 self.start_terminal_server()
Hung-Te Lincc41d2a2014-10-29 13:35:20 +08001329
Earl Oua3bca122016-10-21 16:00:30 +08001330 # Load and run Goofy plugins.
1331 self.plugin_controller = plugin_controller.PluginController(
1332 self.test_list.options.plugin_config_name, self)
1333 self.plugin_controller.StartAllPlugins()
1334
Joel Kitchingd28fbda2016-11-25 16:58:26 +08001335 # Set reference to the Instalog plugin.
1336 self.testlog.SetInstalogPlugin(
1337 self.plugin_controller.GetPluginInstance('instalog'))
1338
Jon Salz0697cbf2012-07-04 15:14:04 +08001339 # Note that we create a log watcher even if
1340 # sync_event_log_period_secs isn't set (no background
1341 # syncing), since we may use it to flush event logs as well.
1342 self.log_watcher = EventLogWatcher(
Ricky Liang45c73e72015-01-15 15:00:30 +08001343 self.test_list.options.sync_event_log_period_secs,
1344 event_log_db_file=None,
1345 handle_event_logs_callback=self.handle_event_logs)
Jon Salz0697cbf2012-07-04 15:14:04 +08001346 if self.test_list.options.sync_event_log_period_secs:
1347 self.log_watcher.StartWatchThread()
1348
1349 self.update_system_info()
1350
1351 os.environ['CROS_FACTORY'] = '1'
1352 os.environ['CROS_DISABLE_SITE_SYSINFO'] = '1'
1353
Jon Salze12c2b32013-06-25 16:24:34 +08001354 self.find_kcrashes()
1355
Shuo-Peng Liao268b40b2013-07-01 15:58:59 +08001356 # Should not move earlier.
1357 self.hooks.OnStartup()
1358
Ricky Liang36512a32014-07-25 11:47:04 +08001359 # Only after this point the Goofy backend is ready for UI connection.
1360 self.ready_for_ui_connection = True
1361
Ricky Liang650f6bf2012-09-28 13:22:54 +08001362 # Create download path for autotest beforehand or autotests run at
1363 # the same time might fail due to race condition.
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +08001364 if not sys_utils.InChroot():
Hung-Te Lin4e6357c2016-01-08 14:32:00 +08001365 file_utils.TryMakeDirs(os.path.join('/usr/local/autotest', 'tests',
1366 'download'))
Ricky Liang650f6bf2012-09-28 13:22:54 +08001367
Jon Salz0697cbf2012-07-04 15:14:04 +08001368 def state_change_callback(test, test_state):
1369 self.event_client.post_event(
Ricky Liang4bff3e32014-02-20 18:46:11 +08001370 Event(Event.Type.STATE_CHANGE, path=test.path, state=test_state))
Jon Salz0697cbf2012-07-04 15:14:04 +08001371 self.test_list.state_change_callback = state_change_callback
Jon Salz73e0fd02012-04-04 11:46:38 +08001372
Vic Yange2c76a82014-10-30 12:48:19 -07001373 self.autotest_prespawner = prespawner.AutotestPrespawner()
1374 self.autotest_prespawner.start()
1375
1376 self.pytest_prespawner = prespawner.PytestPrespawner()
1377 self.pytest_prespawner.start()
Jon Salza6711d72012-07-18 14:33:03 +08001378
Ricky Liang48e47f92014-02-26 19:31:51 +08001379 tests_after_shutdown = self.state_instance.get_shared_data(
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001380 TESTS_AFTER_SHUTDOWN, optional=True)
Jon Salz5c344f62012-07-13 14:31:16 +08001381 force_auto_run = (tests_after_shutdown == FORCE_AUTO_RUN)
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001382
Jon Salz5c344f62012-07-13 14:31:16 +08001383 if not force_auto_run and tests_after_shutdown is not None:
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001384 logging.info('Resuming tests after shutdown: %r', tests_after_shutdown)
1385 self.test_list_iterator = tests_after_shutdown
1386 self.test_list_iterator.set_test_list(self.test_list)
Peter Ammon1e1ec572014-06-26 17:56:32 -07001387 self.run_enqueue(self.run_next_test)
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001388 elif force_auto_run or self.test_list.options.auto_run_on_start:
1389 # If automation mode is enabled, allow suppress auto_run_on_start.
1390 if (self.options.automation_mode == 'NONE' or
1391 self.options.auto_run_on_start):
1392 status_filter = [TestState.UNTESTED]
1393 if self.test_list.options.retry_failed_on_start:
1394 status_filter.append(TestState.FAILED)
1395 self.run_enqueue(lambda: self.run_tests(self.test_list, status_filter))
1396 self.state_instance.set_shared_data(TESTS_AFTER_SHUTDOWN, None)
Ricky Liang4bff3e32014-02-20 18:46:11 +08001397 self.restore_active_run_state()
Hung-Te Linf2f78f72012-02-08 19:27:11 +08001398
Hung-Te Lin410f70a2015-12-15 14:53:42 +08001399 self.dut.hooks.OnTestStart()
Vic Yang08505c72015-01-06 17:01:53 -08001400
Dean Liao592e4d52013-01-10 20:06:39 +08001401 self.may_disable_cros_shortcut_keys()
1402
1403 def may_disable_cros_shortcut_keys(self):
1404 test_options = self.test_list.options
1405 if test_options.disable_cros_shortcut_keys:
1406 logging.info('Filter ChromeOS shortcut keys.')
1407 self.key_filter = KeyFilter(
1408 unmap_caps_lock=test_options.disable_caps_lock,
1409 caps_lock_keycode=test_options.caps_lock_keycode)
1410 self.key_filter.Start()
1411
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001412 def check_log_rotation(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001413 """Checks log rotation file presence/absence according to test_list option.
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001414
1415 Touch /var/lib/cleanup_logs_paused if test_list.options.disable_log_rotation
1416 is True, delete it otherwise. This must be done in idle loop because
1417 autotest client will touch /var/lib/cleanup_logs_paused each time it runs
1418 an autotest.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001419 """
Hung-Te Linf5f2d7f2016-01-08 17:12:46 +08001420 if sys_utils.InChroot():
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001421 return
1422 try:
1423 if self.test_list.options.disable_log_rotation:
1424 open(CLEANUP_LOGS_PAUSED, 'w').close()
1425 else:
1426 file_utils.TryUnlink(CLEANUP_LOGS_PAUSED)
1427 except: # pylint: disable=W0702
1428 # Oh well. Logs an error (but no trace)
1429 logging.info(
1430 'Unable to %s %s: %s',
1431 'touch' if self.test_list.options.disable_log_rotation else 'delete',
Hung-Te Linf707b242016-01-08 23:11:42 +08001432 CLEANUP_LOGS_PAUSED, debug_utils.FormatExceptionOnly())
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001433
Peter Ammon1e1ec572014-06-26 17:56:32 -07001434 def perform_periodic_tasks(self):
1435 """Override of base method to perform periodic work.
Vic Yang4953fc12012-07-26 16:19:53 +08001436
Peter Ammon1e1ec572014-06-26 17:56:32 -07001437 This method must not raise exceptions.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001438 """
Peter Ammon1e1ec572014-06-26 17:56:32 -07001439 super(Goofy, self).perform_periodic_tasks()
Jon Salzb22d1172012-08-06 10:38:57 +08001440
Earl Oua3bca122016-10-21 16:00:30 +08001441 self.check_plugins()
cychiang21886742012-07-05 15:16:32 +08001442 self.check_for_updates()
Cheng-Yi Chiang39d32ad2013-07-23 15:02:38 +08001443 self.check_log_rotation()
Jon Salz57717ca2012-04-04 16:47:25 +08001444
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +08001445 def handle_event_logs(self, chunks, periodic=False):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001446 """Callback for event watcher.
Jon Salz258a40c2012-04-19 12:34:01 +08001447
Jon Salz0697cbf2012-07-04 15:14:04 +08001448 Attempts to upload the event logs to the shopfloor server.
Vic Yang93027612013-05-06 02:42:49 +08001449
1450 Args:
Jon Salzd15bbcf2013-05-21 17:33:57 +08001451 chunks: A list of Chunk objects.
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +08001452 periodic: This event log handling is periodic. Error messages
1453 will only be shown for the first time.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001454 """
Vic Yang93027612013-05-06 02:42:49 +08001455 first_exception = None
1456 exception_count = 0
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +08001457 # Suppress error messages for periodic event syncing except for the
1458 # first time. If event syncing is not periodic, always show the error
1459 # messages.
1460 quiet = self._suppress_event_log_error_messages if periodic else False
Vic Yang93027612013-05-06 02:42:49 +08001461
Jon Salzd15bbcf2013-05-21 17:33:57 +08001462 for chunk in chunks:
Vic Yang93027612013-05-06 02:42:49 +08001463 try:
Jon Salzcddb6402013-05-23 12:56:42 +08001464 description = 'event logs (%s)' % str(chunk)
Vic Yang93027612013-05-06 02:42:49 +08001465 start_time = time.time()
1466 shopfloor_client = shopfloor.get_instance(
Ricky Liang45c73e72015-01-15 15:00:30 +08001467 detect=True,
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +08001468 timeout=self.test_list.options.shopfloor_timeout_secs,
1469 quiet=quiet)
Ricky Liang45c73e72015-01-15 15:00:30 +08001470 shopfloor_client.UploadEvent(chunk.log_name + '.' +
Jon Salzd15bbcf2013-05-21 17:33:57 +08001471 event_log.GetReimageId(),
1472 Binary(chunk.chunk))
Vic Yang93027612013-05-06 02:42:49 +08001473 logging.info(
Ricky Liang45c73e72015-01-15 15:00:30 +08001474 'Successfully synced %s in %.03f s',
1475 description, time.time() - start_time)
1476 except: # pylint: disable=W0702
Hung-Te Linf707b242016-01-08 23:11:42 +08001477 first_exception = (first_exception or
1478 (chunk.log_name + ': ' +
1479 debug_utils.FormatExceptionOnly()))
Vic Yang93027612013-05-06 02:42:49 +08001480 exception_count += 1
1481
1482 if exception_count:
1483 if exception_count == 1:
1484 msg = 'Log upload failed: %s' % first_exception
1485 else:
1486 msg = '%d log upload failed; first is: %s' % (
1487 exception_count, first_exception)
Cheng-Yi Chiangf5b21012015-03-17 15:37:14 +08001488 # For periodic event log syncing, only show the first error messages.
1489 if periodic:
1490 if not self._suppress_event_log_error_messages:
1491 self._suppress_event_log_error_messages = True
1492 logging.warning('Suppress periodic shopfloor error messages for '
1493 'event log syncing after the first one.')
1494 raise Exception(msg)
1495 # For event log syncing by request, show the error messages.
1496 else:
1497 raise Exception(msg)
Vic Yang93027612013-05-06 02:42:49 +08001498
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001499 def run_tests_with_status(self, statuses_to_run, root=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001500 """Runs all top-level tests with a particular status.
Jon Salz0405ab52012-03-16 15:26:52 +08001501
Jon Salz0697cbf2012-07-04 15:14:04 +08001502 All active tests, plus any tests to re-run, are reset.
Jon Salz57717ca2012-04-04 16:47:25 +08001503
Jon Salz0697cbf2012-07-04 15:14:04 +08001504 Args:
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001505 statuses_to_run: The particular status that caller wants to run.
Jon Salz0697cbf2012-07-04 15:14:04 +08001506 starting_at: If provided, only auto-runs tests beginning with
1507 this test.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001508 root: The root of tests to run. If not provided, it will be
1509 the root of all tests.
1510 """
Jon Salz0697cbf2012-07-04 15:14:04 +08001511 root = root or self.test_list
Jon Salz6dc031d2013-06-19 13:06:23 +08001512 self.abort_active_tests('Operator requested run/re-run of certain tests')
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001513 self.run_tests(root, status_filter=statuses_to_run)
Jon Salz0405ab52012-03-16 15:26:52 +08001514
Jon Salz0697cbf2012-07-04 15:14:04 +08001515 def restart_tests(self, root=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001516 """Restarts all tests."""
Jon Salz0697cbf2012-07-04 15:14:04 +08001517 root = root or self.test_list
Jon Salz0405ab52012-03-16 15:26:52 +08001518
Jon Salz6dc031d2013-06-19 13:06:23 +08001519 self.abort_active_tests('Operator requested restart of certain tests')
Jon Salz0697cbf2012-07-04 15:14:04 +08001520 for test in root.walk():
Ricky Liangfea4ac92014-08-21 11:55:59 +08001521 test.update_state(status=TestState.UNTESTED)
Jon Salz0697cbf2012-07-04 15:14:04 +08001522 self.run_tests(root)
Hung-Te Lin96632362012-03-20 21:14:18 +08001523
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001524 def auto_run(self, root=None):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001525 """"Auto-runs" tests that have not been run yet.
Hung-Te Lin96632362012-03-20 21:14:18 +08001526
Jon Salz0697cbf2012-07-04 15:14:04 +08001527 Args:
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001528 root: If provided, the root of tests to run. If not provided, the root
1529 will be test_list (root of all tests).
1530 """
Jon Salz0697cbf2012-07-04 15:14:04 +08001531 root = root or self.test_list
1532 self.run_tests_with_status([TestState.UNTESTED, TestState.ACTIVE],
Ricky Liang45c73e72015-01-15 15:00:30 +08001533 root=root)
Jon Salz968e90b2012-03-18 16:12:43 +08001534
Jon Salz0697cbf2012-07-04 15:14:04 +08001535 def handle_switch_test(self, event):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001536 """Switches to a particular test.
Jon Salz0405ab52012-03-16 15:26:52 +08001537
Ricky Liang6fe218c2013-12-27 15:17:17 +08001538 Args:
1539 event: The SWITCH_TEST event.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001540 """
Jon Salz0697cbf2012-07-04 15:14:04 +08001541 test = self.test_list.lookup_path(event.path)
1542 if not test:
1543 logging.error('Unknown test %r', event.key)
1544 return
Jon Salz73e0fd02012-04-04 11:46:38 +08001545
Jon Salz0697cbf2012-07-04 15:14:04 +08001546 invoc = self.invocations.get(test)
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001547 if invoc:
Jon Salz0697cbf2012-07-04 15:14:04 +08001548 # Already running: just bring to the front if it
1549 # has a UI.
1550 logging.info('Setting visible test to %s', test.path)
Jon Salz36fbbb52012-07-05 13:45:06 +08001551 self.set_visible_test(test)
Jon Salz0697cbf2012-07-04 15:14:04 +08001552 return
Jon Salz73e0fd02012-04-04 11:46:38 +08001553
Jon Salz6dc031d2013-06-19 13:06:23 +08001554 self.abort_active_tests('Operator requested abort (switch_test)')
Jon Salz0697cbf2012-07-04 15:14:04 +08001555 for t in test.walk():
1556 t.update_state(status=TestState.UNTESTED)
Jon Salz73e0fd02012-04-04 11:46:38 +08001557
Wei-Han Chenc17b4112016-11-22 14:56:51 +08001558 self.run_tests(test)
Jon Salz73e0fd02012-04-04 11:46:38 +08001559
Wei-Ning Huang38b75f02015-02-25 18:25:14 +08001560 def handle_key_filter_mode(self, event):
1561 if self.key_filter:
1562 if getattr(event, 'enabled'):
1563 self.key_filter.Start()
1564 else:
1565 self.key_filter.Stop()
1566
Jon Salz0697cbf2012-07-04 15:14:04 +08001567 def wait(self):
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001568 """Waits for all pending invocations.
Jon Salz0697cbf2012-07-04 15:14:04 +08001569
1570 Useful for testing.
Cheng-Yi Chiang1e3e2692013-12-24 18:02:36 +08001571 """
Jon Salz1acc8742012-07-17 17:45:55 +08001572 while self.invocations:
1573 for k, v in self.invocations.iteritems():
1574 logging.info('Waiting for %s to complete...', k)
1575 v.thread.join()
1576 self.reap_completed_tests()
Jon Salz0697cbf2012-07-04 15:14:04 +08001577
Claire Changd1961a22015-08-05 16:15:55 +08001578 def test_fail(self, test):
Hung-Te Lin410f70a2015-12-15 14:53:42 +08001579 self.dut.hooks.OnTestFailure(test)
Claire Changd1961a22015-08-05 16:15:55 +08001580 if self.link_manager:
1581 self.link_manager.UpdateStatus(False)
1582
Wei-Han Chenced08ef2016-11-08 09:40:02 +08001583
Hung-Te Linf2f78f72012-02-08 19:27:11 +08001584if __name__ == '__main__':
Peter Ammona3d298c2014-09-23 10:11:02 -07001585 Goofy.run_main_and_exit()