Add "iterations" option to run a test multiple times.
BUG=None
TEST=goofy_unittest.py, manual on device
Change-Id: I84c0fdd9989272412a68f815dd90d068ef2ba39e
Reviewed-on: https://gerrit.chromium.org/gerrit/27631
Commit-Ready: Jon Salz <jsalz@chromium.org>
Reviewed-by: Jon Salz <jsalz@chromium.org>
Tested-by: Jon Salz <jsalz@chromium.org>
diff --git a/py/goofy/goofy.py b/py/goofy/goofy.py
index 83e2ace..6bc1a14 100755
--- a/py/goofy/goofy.py
+++ b/py/goofy/goofy.py
@@ -609,12 +609,20 @@
Event.Type.PENDING_SHUTDOWN))
continue
- invoc = TestInvocation(self, test, on_completion=self.run_next_test)
- self.invocations[test] = invoc
- if self.visible_test is None and test.has_ui:
- self.set_visible_test(test)
- self.check_connection_manager()
- invoc.start()
+ self._run_test(test, test.iterations)
+
+ def _run_test(self, test, iterations_left=None):
+ invoc = TestInvocation(self, test, on_completion=self.run_next_test)
+ new_state = test.update_state(
+ status=TestState.ACTIVE, increment_count=1, error_msg='',
+ invocation=invoc.uuid, iterations_left=iterations_left)
+ invoc.count = new_state.count
+
+ self.invocations[test] = invoc
+ if self.visible_test is None and test.has_ui:
+ self.set_visible_test(test)
+ self.check_connection_manager()
+ invoc.start()
def check_connection_manager(self):
exclusive_tests = [
@@ -705,8 +713,13 @@
'''
for t, v in dict(self.invocations).iteritems():
if v.is_completed():
+ new_state = t.update_state(**v.update_state_on_completion)
del self.invocations[t]
+ if new_state.iterations_left and new_state.status == TestState.PASSED:
+ # Play it again, Sam!
+ self._run_test(t)
+
if (self.visible_test is None or
self.visible_test not in self.invocations):
self.set_visible_test(None)
@@ -733,7 +746,9 @@
factory.console.info('Killing active test %s...' % test.path)
invoc.abort_and_join()
factory.console.info('Killed %s' % test.path)
+ test.update_state(**invoc.update_state_on_completion)
del self.invocations[test]
+
if not abort:
test.update_state(status=TestState.UNTESTED)
self.reap_completed_tests()
@@ -1200,9 +1215,11 @@
Useful for testing.
'''
- for k, v in self.invocations.iteritems():
- logging.info('Waiting for %s to complete...', k)
- v.thread.join()
+ while self.invocations:
+ for k, v in self.invocations.iteritems():
+ logging.info('Waiting for %s to complete...', k)
+ v.thread.join()
+ self.reap_completed_tests()
def check_exceptions(self):
'''Raises an error if any exceptions have occurred in
diff --git a/py/goofy/goofy_unittest.py b/py/goofy/goofy_unittest.py
index ce1f13c..ca8e455 100755
--- a/py/goofy/goofy_unittest.py
+++ b/py/goofy/goofy_unittest.py
@@ -121,7 +121,7 @@
'''Hook invoked before init_goofy.'''
def check_one_test(self, test_id, name, passed, error_msg, trigger=None,
- does_not_start=False):
+ does_not_start=False, setup_mocks=True, expected_count=1):
'''Runs a single autotest, waiting for it to complete.
Args:
@@ -132,10 +132,12 @@
trigger: An optional callable that will be executed after mocks are
set up to trigger the autotest. If None, then the test is
expected to start itself.
- does_not_start: If True, checks that the test does not start
+ does_not_start: If True, checks that the test is not expected to start
(e.g., due to an unsatisfied require_run).
+ setup_mocks: If True, sets up mocks for the test runs.
+ expected_count: The expected run count.
'''
- if not does_not_start:
+ if setup_mocks and not does_not_start:
mock_autotest(self.env, name, passed, error_msg)
self.mocker.ReplayAll()
if trigger:
@@ -147,7 +149,7 @@
test_state = self.state.get_test_state(test_id)
self.assertEqual(TestState.PASSED if passed else TestState.FAILED,
test_state.status)
- self.assertEqual(0 if does_not_start else 1, test_state.count)
+ self.assertEqual(0 if does_not_start else expected_count, test_state.count)
self.assertEqual(error_msg, test_state.error_msg)
@@ -235,17 +237,17 @@
self.assertTrue(Event.Type.LOG in events_by_type,
repr(events_by_type))
- # Each test should have a transition to active and to its
- # final state
+ # Each test should have a transition to active, a transition to
+ # active + visible, and then to its final state
for path, final_status in (('a', TestState.PASSED),
- ('b', TestState.FAILED),
- ('c', TestState.FAILED)):
+ ('b', TestState.FAILED),
+ ('c', TestState.FAILED)):
statuses = [
event.state['status']
for event in events_by_type[Event.Type.STATE_CHANGE]
if event.path == path]
self.assertEqual(
- ['UNTESTED', 'ACTIVE', final_status],
+ ['ACTIVE', 'ACTIVE', final_status],
statuses)
@@ -403,6 +405,32 @@
failed_state.error_msg)
+class MultipleIterationsTest(GoofyTest):
+ '''Tests running a test multiple times.'''
+ test_list = '''
+ OperatorTest(id='a', autotest_name='a_A'),
+ OperatorTest(id='b', autotest_name='b_B', iterations=3),
+ OperatorTest(id='c', autotest_name='c_C', iterations=3),
+ OperatorTest(id='d', autotest_name='d_D'),
+ '''
+ def runTest(self):
+ self.check_one_test('a', 'a_A', True, '')
+
+ mock_autotest(self.env, 'b_B', True, '')
+ mock_autotest(self.env, 'b_B', True, '')
+ mock_autotest(self.env, 'b_B', True, '')
+ self.check_one_test('b', 'b_B', True, '', setup_mocks=False,
+ expected_count=3)
+
+ mock_autotest(self.env, 'c_C', True, '')
+ mock_autotest(self.env, 'c_C', False, 'I bent my wookie')
+ # iterations=3, but it should stop after the first failed iteration.
+ self.check_one_test('c', 'c_C', False, 'I bent my wookie',
+ setup_mocks=False, expected_count=2)
+
+ self.check_one_test('d', 'd_D', True, '')
+
+
class ConnectionManagerTest(GoofyTest):
options = '''
options.wlans = [WLAN('foo', 'bar', 'baz')]
diff --git a/py/goofy/invocation.py b/py/goofy/invocation.py
index afdf2d0..25adcdd 100755
--- a/py/goofy/invocation.py
+++ b/py/goofy/invocation.py
@@ -52,14 +52,22 @@
class TestInvocation(object):
'''
State for an active test.
+
+ Properties:
+ update_state_on_completion: State for Goofy to update on
+ completion; Goofy will call test.update_state(
+ **update_state_on_completion). So update_state_on_completion
+ will have at least status and error_msg properties to update
+ the test state.
'''
def __init__(self, goofy, test, on_completion=None):
'''Constructor.
- @param goofy: The controlling Goofy object.
- @param test: The FactoryTest object to test.
- @param on_completion: Callback to invoke in the goofy event queue
- on completion.
+ Args:
+ goofy: The controlling Goofy object.
+ test: The FactoryTest object to test.
+ on_completion: Callback to invoke in the goofy event queue
+ on completion.
'''
self.goofy = goofy
self.test = test
@@ -79,8 +87,9 @@
init_time=time.time(),
invocation=str(self.uuid))
self.count = None
-
self.log_path = os.path.join(self.output_dir, 'log')
+ self.update_state_on_completion = {}
+
self._lock = threading.Lock()
# The following properties are guarded by the lock.
self._aborted = False
@@ -355,11 +364,15 @@
if self._aborted:
return
- self.count = self.test.update_state(
- status=TestState.ACTIVE, increment_count=1, error_msg='',
- invocation=self.uuid).count
-
- factory.console.info('Running test %s' % self.test.path)
+ if self.test.iterations > 1:
+ iteration_string = ' [%s/%s]' % (
+ self.test.iterations -
+ self.test.get_state().iterations_left + 1,
+ self.test.iterations)
+ else:
+ iteration_string = ''
+ factory.console.info('Running test %s%s',
+ self.test.path, iteration_string)
log_args = dict(
path=self.test.path,
@@ -420,14 +433,16 @@
except:
logging.exception('Unable to log end_test event')
- factory.console.info('Test %s %s%s',
- self.test.path,
- status,
- ': %s' % error_msg if error_msg else '')
+ factory.console.info('Test %s%s %s%s',
+ self.test.path,
+ iteration_string,
+ status,
+ ': %s' % error_msg if error_msg else '')
- self.test.update_state(status=status, error_msg=error_msg,
- visible=False)
with self._lock:
+ self.update_state_on_completion = dict(
+ status=status, error_msg=error_msg,
+ visible=False, decrement_iterations_left=1)
self._completed = True
self.goofy.run_queue.put(self.goofy.reap_completed_tests)
diff --git a/py/test/factory.py b/py/test/factory.py
index a3fc3aa..a697c55 100644
--- a/py/test/factory.py
+++ b/py/test/factory.py
@@ -359,13 +359,16 @@
'''
The complete state of a test.
- @property status: The status of the test (one of ACTIVE, PASSED,
- FAILED, or UNTESTED).
- @property count: The number of times the test has been run.
- @property error_msg: The last error message that caused a test failure.
- @property shutdown_count: The next of times the test has caused a shutdown.
- @property visible: Whether the test is the currently visible test.
- @property invocation: The currently executing invocation.
+ Properties:
+ status: The status of the test (one of ACTIVE, PASSED,
+ FAILED, or UNTESTED).
+ count: The number of times the test has been run.
+ error_msg: The last error message that caused a test failure.
+ shutdown_count: The next of times the test has caused a shutdown.
+ visible: Whether the test is the currently visible test.
+ invocation: The currently executing invocation.
+ iterations_left: For an active test, the number of remaining
+ iterations after the current one.
'''
ACTIVE = 'ACTIVE'
PASSED = 'PASSED'
@@ -373,20 +376,22 @@
UNTESTED = 'UNTESTED'
def __init__(self, status=UNTESTED, count=0, visible=False, error_msg=None,
- shutdown_count=0, invocation=None):
+ shutdown_count=0, invocation=None, iterations_left=0):
self.status = status
self.count = count
self.visible = visible
self.error_msg = error_msg
self.shutdown_count = shutdown_count
self.invocation = invocation
+ self.iterations_left = iterations_left
def __repr__(self):
return std_repr(self)
def update(self, status=None, increment_count=0, error_msg=None,
shutdown_count=None, increment_shutdown_count=0, visible=None,
- invocation=None):
+ invocation=None,
+ decrement_iterations_left=0, iterations_left=None):
'''
Updates the state of a test.
@@ -399,6 +404,9 @@
@param visible: If non-None, whether the test should become visible.
@param invocation: The currently executing invocation, if any.
(Applies only if the status is ACTIVE.)
+ @param iterations_left: If non-None, the new iterations_left.
+ @param decrement_iterations_left: An amount by which to decrement
+ iterations_left.
Returns True if anything was changed.
'''
@@ -410,6 +418,8 @@
self.error_msg = error_msg
if shutdown_count is not None:
self.shutdown_count = shutdown_count
+ if iterations_left is not None:
+ self.iterations_left = iterations_left
if visible is not None:
self.visible = visible
@@ -420,6 +430,8 @@
self.count += increment_count
self.shutdown_count += increment_shutdown_count
+ self.iterations_left = max(
+ 0, self.iterations_left - decrement_iterations_left)
return self.__dict__ != old_dict
@@ -505,6 +517,7 @@
never_fails=None,
exclusive=None,
require_run=None,
+ iterations=1,
_root=None,
_default_id=None):
'''
@@ -540,6 +553,7 @@
SMT before FlushEventLogs must have already been run. The string
"all" may be used to refer to the root (i.e., all tests in the
whole test list before this one must already have been run).
+ @param iterations: Number of times to run the test.
@param _root: True only if this is the root node (for internal use
only).
'''
@@ -563,6 +577,10 @@
self.path = ''
self.parent = None
self.root = None
+ self.iterations = iterations
+ assert isinstance(self.iterations, int) and self.iterations > 0, (
+ 'In test %s, Iterations must be a positive integer, not %r' % (
+ self.path, self.iterations))
if _root:
self.id = None
@@ -879,7 +897,7 @@
REBOOT = 'reboot'
HALT = 'halt'
- def __init__(self, operation, iterations=1, delay_secs=5, **kw):
+ def __init__(self, operation, delay_secs=5, **kw):
super(ShutdownStep, self).__init__(**kw)
assert not self.autotest_name, (
'Reboot/halt steps may not have an autotest')
@@ -887,8 +905,6 @@
assert not self.backgroundable, (
'Reboot/halt steps may not be backgroundable')
- assert iterations > 0
- self.iterations = iterations
assert operation in [self.REBOOT, self.HALT]
self.operation = operation
assert delay_secs >= 0