tests: add build option to force skips as failures
This will be useful in CI, where we do not want to see any skips. If
something starts to skip, that's a mistake somewhere, and want to catch
it.
Signed-off-by: Pekka Paalanen <pekka.paalanen@collabora.com>
diff --git a/meson_options.txt b/meson_options.txt
index 239bd2d..ea225a9 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -211,6 +211,12 @@
description: 'Tests: output JUnit XML results'
)
option(
+ 'test-skip-is-failure',
+ type: 'boolean',
+ value: false,
+ description: 'Tests: consider skip to be a failure'
+)
+option(
'test-gl-renderer',
type: 'boolean',
value: true,
diff --git a/tests/meson.build b/tests/meson.build
index 90d776f..212bba0 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -276,6 +276,7 @@
test_config_h.set_quoted('WESTON_MODULE_MAP', env_modmap)
test_config_h.set_quoted('WESTON_DATA_DIR', join_paths(meson.current_source_dir(), '..', 'data'))
test_config_h.set_quoted('TESTSUITE_PLUGIN_PATH', exe_plugin_test.full_path())
+test_config_h.set10('WESTON_TEST_SKIP_IS_FAILURE', get_option('test-skip-is-failure'))
configure_file(output: 'test-config.h', configuration: test_config_h)
foreach t : tests
diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c
index d8becc3..0ab7bc1 100644
--- a/tests/weston-test-runner.c
+++ b/tests/weston-test-runner.c
@@ -35,6 +35,7 @@
#include <signal.h>
#include <getopt.h>
+#include "test-config.h"
#include "weston-test-runner.h"
#include "weston-testsuite-data.h"
#include "shared/string-helpers.h"
@@ -243,7 +244,11 @@
[RESULT_FAIL] = "fail",
[RESULT_HARD_ERROR] = "hard error",
[RESULT_OK] = "ok",
+#if WESTON_TEST_SKIP_IS_FAILURE
+ [RESULT_SKIP] = "skip error",
+#else
[RESULT_SKIP] = "skip",
+#endif
};
assert(ret >= 0 && ret < ARRAY_LENGTH(names));
@@ -284,6 +289,9 @@
case RESULT_SKIP:
suite_data->skipped++;
skip = " # SKIP";
+#if WESTON_TEST_SKIP_IS_FAILURE
+ fail = "not ";
+#endif
break;
}
@@ -327,10 +335,16 @@
const void *test_data,
int iteration)
{
+ const char *skip_error = "";
int iteration_nr = iteration + 1;
+#if WESTON_TEST_SKIP_IS_FAILURE
+ skip_error = "not ";
+#endif
+
suite_data->counter++;
- printf("ok %d %s %s/%d # SKIP fixture\n", suite_data->counter,
+ printf("%sok %d %s %s/%d # SKIP fixture\n",
+ skip_error, suite_data->counter,
suite_data->fixture_name, t->name, iteration_nr);
}
@@ -477,6 +491,11 @@
static enum test_result_code
counts_to_result(const struct wet_testsuite_data *data)
{
+#if WESTON_TEST_SKIP_IS_FAILURE
+ if (data->skipped > 0)
+ return RESULT_FAIL;
+#endif
+
/* RESULT_SKIP is reserved for fixture setup itself skipping everything */
if (data->total == data->passed + data->skipped)
return RESULT_OK;
@@ -644,7 +663,11 @@
if (ret == RESULT_SKIP) {
tap_skip_fixture(&harness->data);
+#if WESTON_TEST_SKIP_IS_FAILURE
+ ret = RESULT_FAIL;
+#else
continue;
+#endif
}
if (ret != RESULT_OK && result != RESULT_HARD_ERROR)