Support e.g. --workers=2x to use two workers per core.
This is mostly useful for tests performing a lot of I/O and sleeping,
when you don't know on which architecture they end up running.
The syntax can also be used to reduce CPU load (e.g. --workers=0.5x).
Bug: webrtc:9717
Change-Id: I26b4552576b1dd56a69c2223da39f4bb1115bbf6
Reviewed-on: https://webrtc-review.googlesource.com/101643
Commit-Queue: Yves Gerey <yvesg@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24830}diff --git a/tools_webrtc/gtest_parallel_wrapper_test.py b/tools_webrtc/gtest_parallel_wrapper_test.py
index 364fc89..e5447e3 100755
--- a/tools_webrtc/gtest_parallel_wrapper_test.py
+++ b/tools_webrtc/gtest_parallel_wrapper_test.py
@@ -10,11 +10,13 @@
from contextlib import contextmanager
+import multiprocessing
import os
import tempfile
import unittest
-script = __import__('gtest-parallel-wrapper') # pylint: disable=invalid-name
+# pylint: disable=invalid-name
+gtest_parallel_wrapper = __import__('gtest-parallel-wrapper')
@contextmanager
@@ -24,105 +26,151 @@
os.rmdir(tmp_dir)
+class GtestParallelWrapperHelpersTest(unittest.TestCase):
+
+ def testGetWorkersAsIs(self):
+ # pylint: disable=protected-access
+ self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('12'), 12)
+
+ def testGetTwiceWorkers(self):
+ expected = 2 * multiprocessing.cpu_count()
+ # pylint: disable=protected-access
+ self.assertEqual(gtest_parallel_wrapper._ParseWorkersOption('2x'), expected)
+
+ def testGetHalfWorkers(self):
+ expected = max(multiprocessing.cpu_count() // 2, 1)
+ # pylint: disable=protected-access
+ self.assertEqual(
+ gtest_parallel_wrapper._ParseWorkersOption('0.5x'), expected)
+
+
class GtestParallelWrapperTest(unittest.TestCase):
+
@classmethod
def _Expected(cls, gtest_parallel_args):
return ['--shard_index=0', '--shard_count=1'] + gtest_parallel_args
def testOverwrite(self):
- result = script.ParseArgs(['--timeout=123', 'exec', '--timeout', '124'])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['--timeout=123', 'exec', '--timeout', '124'])
expected = self._Expected(['--timeout=124', 'exec'])
self.assertEqual(result.gtest_parallel_args, expected)
def testMixing(self):
- result = script.ParseArgs(
+ result = gtest_parallel_wrapper.ParseArgs(
['--timeout=123', '--param1', 'exec', '--param2', '--timeout', '124'])
expected = self._Expected(
['--timeout=124', 'exec', '--', '--param1', '--param2'])
self.assertEqual(result.gtest_parallel_args, expected)
def testMixingPositional(self):
- result = script.ParseArgs(['--timeout=123', 'exec', '--foo1', 'bar1',
- '--timeout', '124', '--foo2', 'bar2'])
- expected = self._Expected(['--timeout=124', 'exec', '--', '--foo1', 'bar1',
- '--foo2', 'bar2'])
+ result = gtest_parallel_wrapper.ParseArgs([
+ '--timeout=123', 'exec', '--foo1', 'bar1', '--timeout', '124', '--foo2',
+ 'bar2'
+ ])
+ expected = self._Expected(
+ ['--timeout=124', 'exec', '--', '--foo1', 'bar1', '--foo2', 'bar2'])
self.assertEqual(result.gtest_parallel_args, expected)
def testDoubleDash1(self):
- result = script.ParseArgs(
+ result = gtest_parallel_wrapper.ParseArgs(
['--timeout', '123', 'exec', '--', '--timeout', '124'])
expected = self._Expected(
['--timeout=123', 'exec', '--', '--timeout', '124'])
self.assertEqual(result.gtest_parallel_args, expected)
def testDoubleDash2(self):
- result = script.ParseArgs(['--timeout=123', '--', 'exec', '--timeout=124'])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['--timeout=123', '--', 'exec', '--timeout=124'])
expected = self._Expected(['--timeout=123', 'exec', '--', '--timeout=124'])
self.assertEqual(result.gtest_parallel_args, expected)
def testArtifacts(self):
with TemporaryDirectory() as tmp_dir:
output_dir = os.path.join(tmp_dir, 'foo')
- result = script.ParseArgs(['exec', '--store-test-artifacts',
- '--output_dir', output_dir])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['exec', '--store-test-artifacts', '--output_dir', output_dir])
exp_artifacts_dir = os.path.join(output_dir, 'test_artifacts')
- exp = self._Expected(['--output_dir=' + output_dir, 'exec', '--',
- '--test_artifacts_dir=' + exp_artifacts_dir])
+ exp = self._Expected([
+ '--output_dir=' + output_dir, 'exec', '--',
+ '--test_artifacts_dir=' + exp_artifacts_dir
+ ])
self.assertEqual(result.gtest_parallel_args, exp)
self.assertEqual(result.output_dir, output_dir)
self.assertEqual(result.test_artifacts_dir, exp_artifacts_dir)
def testNoDirsSpecified(self):
- result = script.ParseArgs(['exec'])
+ result = gtest_parallel_wrapper.ParseArgs(['exec'])
self.assertEqual(result.output_dir, None)
self.assertEqual(result.test_artifacts_dir, None)
def testOutputDirSpecified(self):
- result = script.ParseArgs(['exec', '--output_dir', '/tmp/foo'])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['exec', '--output_dir', '/tmp/foo'])
self.assertEqual(result.output_dir, '/tmp/foo')
self.assertEqual(result.test_artifacts_dir, None)
def testJsonTestResults(self):
- result = script.ParseArgs(['--isolated-script-test-output', '/tmp/foo',
- 'exec'])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['--isolated-script-test-output', '/tmp/foo', 'exec'])
expected = self._Expected(['--dump_json_test_results=/tmp/foo', 'exec'])
self.assertEqual(result.gtest_parallel_args, expected)
def testShortArg(self):
- result = script.ParseArgs(['-d', '/tmp/foo', 'exec'])
+ result = gtest_parallel_wrapper.ParseArgs(['-d', '/tmp/foo', 'exec'])
expected = self._Expected(['--output_dir=/tmp/foo', 'exec'])
self.assertEqual(result.gtest_parallel_args, expected)
self.assertEqual(result.output_dir, '/tmp/foo')
def testBoolArg(self):
- result = script.ParseArgs(['--gtest_also_run_disabled_tests', 'exec'])
+ result = gtest_parallel_wrapper.ParseArgs(
+ ['--gtest_also_run_disabled_tests', 'exec'])
expected = self._Expected(['--gtest_also_run_disabled_tests', 'exec'])
self.assertEqual(result.gtest_parallel_args, expected)
def testNoArgs(self):
- result = script.ParseArgs(['exec'])
+ result = gtest_parallel_wrapper.ParseArgs(['exec'])
expected = self._Expected(['exec'])
self.assertEqual(result.gtest_parallel_args, expected)
def testDocExample(self):
with TemporaryDirectory() as tmp_dir:
output_dir = os.path.join(tmp_dir, 'foo')
- result = script.ParseArgs([
+ result = gtest_parallel_wrapper.ParseArgs([
'some_test', '--some_flag=some_value', '--another_flag',
'--output_dir=' + output_dir, '--store-test-artifacts',
'--isolated-script-test-output=SOME_DIR',
- '--isolated-script-test-perf-output=SOME_OTHER_DIR',
- '--foo=bar', '--baz'])
+ '--isolated-script-test-perf-output=SOME_OTHER_DIR', '--foo=bar',
+ '--baz'
+ ])
expected_artifacts_dir = os.path.join(output_dir, 'test_artifacts')
expected = self._Expected([
'--output_dir=' + output_dir, '--dump_json_test_results=SOME_DIR',
- 'some_test', '--',
- '--test_artifacts_dir=' + expected_artifacts_dir,
+ 'some_test', '--', '--test_artifacts_dir=' + expected_artifacts_dir,
'--some_flag=some_value', '--another_flag',
- '--isolated-script-test-perf-output=SOME_OTHER_DIR',
- '--foo=bar', '--baz'])
+ '--isolated-script-test-perf-output=SOME_OTHER_DIR', '--foo=bar',
+ '--baz'
+ ])
self.assertEqual(result.gtest_parallel_args, expected)
+ def testStandardWorkers(self):
+ """Check integer value is passed as-is."""
+ result = gtest_parallel_wrapper.ParseArgs(['--workers', '17', 'exec'])
+ expected = self._Expected(['--workers=17', 'exec'])
+ self.assertEqual(result.gtest_parallel_args, expected)
+
+ def testTwoWorkersPerCpuCore(self):
+ result = gtest_parallel_wrapper.ParseArgs(['--workers', '2x', 'exec'])
+ workers = 2 * multiprocessing.cpu_count()
+ expected = self._Expected(['--workers=%s' % workers, 'exec'])
+ self.assertEqual(result.gtest_parallel_args, expected)
+
+ def testUseHalfTheCpuCores(self):
+ result = gtest_parallel_wrapper.ParseArgs(['--workers', '0.5x', 'exec'])
+ workers = max(multiprocessing.cpu_count() // 2, 1)
+ expected = self._Expected(['--workers=%s' % workers, 'exec'])
+ self.assertEqual(result.gtest_parallel_args, expected)
+
if __name__ == '__main__':
unittest.main()