From a40a2f5c5077d2349d8442a2bdd9b8551ebcc04e Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Mon, 29 Aug 2022 12:43:36 -0600 Subject: [PATCH] ztest: make failed assumptions fail test binary Adding the new Kconfig (enabled by default) to make a failed assumption mark the final result as failed. This change has the following benefits which have been asked for by the Zephyr community: 1. A failed assumption does not go silent. In this example, the failed assumption will still mark the test as skipped, but the final result will be to mark the full test run as failed. This would allow blocking the CI when an assume fails. 2. Normal test skipping via the ztest_test_skip() is unaffected by this change. Those tests will be marked as skipped, but the binary will still pass. Signed-off-by: Yuval Peress --- subsys/testsuite/ztest/Kconfig | 9 +++++++++ .../ztest/include/zephyr/ztest_assert.h | 5 +++-- .../ztest/include/zephyr/ztest_test_new.h | 3 +++ subsys/testsuite/ztest/src/ztest_new.c | 16 ++++++++++++++++ tests/ztest/error_hook/src/main.c | 5 +++++ 5 files changed, 36 insertions(+), 2 deletions(-) diff --git a/subsys/testsuite/ztest/Kconfig b/subsys/testsuite/ztest/Kconfig index c7b23fd57a..3fbade151e 100644 --- a/subsys/testsuite/ztest/Kconfig +++ b/subsys/testsuite/ztest/Kconfig @@ -149,6 +149,15 @@ config ZTEST_VERBOSE_SUMMARY This option controls whether suite summary is shown verbosely or just in one line. +config ZTEST_FAIL_ON_ASSUME + bool "Fail the test run when an assumption fails" + default y + help + When enabled, the test binary will fail at the end if an assumption failed. This means + that while tests will still be marked as skipped on failed zassume calls, the final test + result will be shown as a failure in order to increase visibility. This precludes tests + that skipped with the ZTEST_EXPECT_SKIP annotation. + endif # ZTEST_NEW_API config TEST_LOGGING_FLUSH_AFTER_TEST diff --git a/subsys/testsuite/ztest/include/zephyr/ztest_assert.h b/subsys/testsuite/ztest/include/zephyr/ztest_assert.h index bfce8d6965..4f7c4ecd4e 100644 --- a/subsys/testsuite/ztest/include/zephyr/ztest_assert.h +++ b/subsys/testsuite/ztest/include/zephyr/ztest_assert.h @@ -27,6 +27,7 @@ extern "C" { const char *ztest_relative_filename(const char *file); void ztest_test_fail(void); void ztest_test_skip(void); +void ztest_skip_failed_assumption(void); #if CONFIG_ZTEST_ASSERT_VERBOSE == 0 static inline bool z_zassert_(bool cond, const char *file, int line) @@ -46,7 +47,7 @@ static inline bool z_zassume_(bool cond, const char *file, int line) { if (cond == false) { PRINT("\n Assumption failed at %s:%d\n", ztest_relative_filename(file), line); - ztest_test_skip(); + ztest_skip_failed_assumption(); return false; } @@ -93,7 +94,7 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil vprintk(msg, vargs); printk("\n"); va_end(vargs); - ztest_test_skip(); + ztest_skip_failed_assumption(); return false; } #if CONFIG_ZTEST_ASSERT_VERBOSE == 2 diff --git a/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h b/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h index 7777340651..b3e8c4d6ae 100644 --- a/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h +++ b/subsys/testsuite/ztest/include/zephyr/ztest_test_new.h @@ -331,6 +331,9 @@ void ztest_test_pass(void); */ void ztest_test_skip(void); + +void ztest_skip_failed_assumption(void); + #define Z_TEST(suite, fn, t_options, use_fixture) \ struct ztest_unit_test_stats z_ztest_unit_test_stats_##suite##_##fn; \ static void _##suite##_##fn##_wrapper(void *data); \ diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c index 525fad1201..ecede8060c 100644 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ b/subsys/testsuite/ztest/src/ztest_new.c @@ -291,6 +291,15 @@ static inline const char *get_friendly_phase_name(enum ztest_phase phase) } } +static bool current_test_failed_assumption; +void ztest_skip_failed_assumption(void) +{ + if (IS_ENABLED(CONFIG_ZTEST_FAIL_ON_ASSUME)) { + current_test_failed_assumption = true; + } + ztest_test_skip(); +} + #ifndef KERNEL /* Static code analysis tool can raise a violation that the standard header @@ -399,6 +408,9 @@ out: ret = get_final_test_result(test, ret); Z_TC_END_RESULT(ret, test->name); + if (ret == TC_SKIP && current_test_failed_assumption) { + test_status = 1; + } return ret; } @@ -579,6 +591,9 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test ret = get_final_test_result(test, ret); Z_TC_END_RESULT(ret, test->name); + if (ret == TC_SKIP && current_test_failed_assumption) { + test_status = 1; + } return ret; } @@ -661,6 +676,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) #endif TC_SUITE_START(suite->name); + current_test_failed_assumption = false; test_result = ZTEST_RESULT_PENDING; phase = TEST_PHASE_SETUP; #ifndef KERNEL diff --git a/tests/ztest/error_hook/src/main.c b/tests/ztest/error_hook/src/main.c index 7c5ae0a35b..8805037f7f 100644 --- a/tests/ztest/error_hook/src/main.c +++ b/tests/ztest/error_hook/src/main.c @@ -372,12 +372,14 @@ static void *fail_assume_in_setup_setup(void) ZTEST_SUITE(fail_assume_in_setup, NULL, fail_assume_in_setup_setup, NULL, NULL, NULL); +ZTEST_EXPECT_SKIP(fail_assume_in_setup, test_to_skip0); ZTEST(fail_assume_in_setup, test_to_skip0) { /* This test should never be run */ ztest_test_fail(); } +ZTEST_EXPECT_SKIP(fail_assume_in_setup, test_to_skip1); ZTEST(fail_assume_in_setup, test_to_skip1) { /* This test should never be run */ @@ -392,12 +394,14 @@ static void fail_assume_in_before_before(void *unused) ZTEST_SUITE(fail_assume_in_before, NULL, NULL, fail_assume_in_before_before, NULL, NULL); +ZTEST_EXPECT_SKIP(fail_assume_in_before, test_to_skip0); ZTEST(fail_assume_in_before, test_to_skip0) { /* This test should never be run */ ztest_test_fail(); } +ZTEST_EXPECT_SKIP(fail_assume_in_before, test_to_skip1); ZTEST(fail_assume_in_before, test_to_skip1) { /* This test should never be run */ @@ -406,6 +410,7 @@ ZTEST(fail_assume_in_before, test_to_skip1) ZTEST_SUITE(fail_assume_in_test, NULL, NULL, NULL, NULL, NULL); +ZTEST_EXPECT_SKIP(fail_assume_in_test, test_to_skip); ZTEST(fail_assume_in_test, test_to_skip) { zassume_true(false);