patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

ztest: make failed assumptions fail test binary

Browse source 
Adding the new Kconfig (enabled by default) to make a failed assumption
mark the final result as failed. This change has the following benefits
which have been asked for by the Zephyr community:
1. A failed assumption does not go silent. In this example, the failed
   assumption will still mark the test as skipped, but the final result
   will be to mark the full test run as failed. This would allow
   blocking the CI when an assume fails.
2. Normal test skipping via the ztest_test_skip() is unaffected by this
   change. Those tests will be marked as skipped, but the binary will
   still pass.

Signed-off-by: Yuval Peress <peress@google.com>
This commit is contained in:
Yuval Peress 2022-08-29 12:43:36 -06:00 committed by Anas Nashif
parent 41e6758b24
commit a40a2f5c50
5 changed files with 36 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
subsys/testsuite/ztest/Kconfig
View file

@ -149,6 +149,15 @@ config ZTEST_VERBOSE_SUMMARY
This option controls whether suite summary is shown verbosely or
just in one line.
config ZTEST_FAIL_ON_ASSUME
bool "Fail the test run when an assumption fails"
default y
help
When enabled, the test binary will fail at the end if an assumption failed. This means
that while tests will still be marked as skipped on failed zassume calls, the final test
result will be shown as a failure in order to increase visibility. This precludes tests
that skipped with the ZTEST_EXPECT_SKIP annotation.
endif # ZTEST_NEW_API
config TEST_LOGGING_FLUSH_AFTER_TEST

5
subsys/testsuite/ztest/include/zephyr/ztest_assert.h
View file

@ -27,6 +27,7 @@ extern "C" {
const char *ztest_relative_filename(const char *file);
void ztest_test_fail(void);
void ztest_test_skip(void);
void ztest_skip_failed_assumption(void);
#if CONFIG_ZTEST_ASSERT_VERBOSE == 0
static inline bool z_zassert_(bool cond, const char *file, int line)
@ -46,7 +47,7 @@ static inline bool z_zassume_(bool cond, const char *file, int line)
{
if (cond == false) {
PRINT("\n Assumption failed at %s:%d\n", ztest_relative_filename(file), line);
ztest_test_skip();
ztest_skip_failed_assumption();
return false;
}
@ -93,7 +94,7 @@ static inline bool z_zassume(bool cond, const char *default_msg, const char *fil
vprintk(msg, vargs);
printk("\n");
va_end(vargs);
ztest_test_skip();
ztest_skip_failed_assumption();
return false;
}
#if CONFIG_ZTEST_ASSERT_VERBOSE == 2

3
subsys/testsuite/ztest/include/zephyr/ztest_test_new.h
View file

@ -331,6 +331,9 @@ void ztest_test_pass(void);
*/
void ztest_test_skip(void);
void ztest_skip_failed_assumption(void);
#define Z_TEST(suite, fn, t_options, use_fixture) \
struct ztest_unit_test_stats z_ztest_unit_test_stats_##suite##_##fn; \
static void _##suite##_##fn##_wrapper(void *data); \

16
subsys/testsuite/ztest/src/ztest_new.c
View file

@ -291,6 +291,15 @@ static inline const char *get_friendly_phase_name(enum ztest_phase phase)
}
}
static bool current_test_failed_assumption;
void ztest_skip_failed_assumption(void)
{
if (IS_ENABLED(CONFIG_ZTEST_FAIL_ON_ASSUME)) {
current_test_failed_assumption = true;
}
ztest_test_skip();
}
#ifndef KERNEL
/* Static code analysis tool can raise a violation that the standard header
@ -399,6 +408,9 @@ out:
ret = get_final_test_result(test, ret);
Z_TC_END_RESULT(ret, test->name);
if (ret == TC_SKIP && current_test_failed_assumption) {
test_status = 1;
}
return ret;
}
@ -579,6 +591,9 @@ static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test
ret = get_final_test_result(test, ret);
Z_TC_END_RESULT(ret, test->name);
if (ret == TC_SKIP && current_test_failed_assumption) {
test_status = 1;
}
return ret;
}
@ -661,6 +676,7 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
#endif
TC_SUITE_START(suite->name);
current_test_failed_assumption = false;
test_result = ZTEST_RESULT_PENDING;
phase = TEST_PHASE_SETUP;
#ifndef KERNEL

5
tests/ztest/error_hook/src/main.c
View file

@ -372,12 +372,14 @@ static void *fail_assume_in_setup_setup(void)
ZTEST_SUITE(fail_assume_in_setup, NULL, fail_assume_in_setup_setup, NULL, NULL, NULL);
ZTEST_EXPECT_SKIP(fail_assume_in_setup, test_to_skip0);
ZTEST(fail_assume_in_setup, test_to_skip0)
{
/* This test should never be run */
ztest_test_fail();
}
ZTEST_EXPECT_SKIP(fail_assume_in_setup, test_to_skip1);
ZTEST(fail_assume_in_setup, test_to_skip1)
{
/* This test should never be run */
@ -392,12 +394,14 @@ static void fail_assume_in_before_before(void *unused)
ZTEST_SUITE(fail_assume_in_before, NULL, NULL, fail_assume_in_before_before, NULL, NULL);
ZTEST_EXPECT_SKIP(fail_assume_in_before, test_to_skip0);
ZTEST(fail_assume_in_before, test_to_skip0)
{
/* This test should never be run */
ztest_test_fail();
}
ZTEST_EXPECT_SKIP(fail_assume_in_before, test_to_skip1);
ZTEST(fail_assume_in_before, test_to_skip1)
{
/* This test should never be run */
@ -406,6 +410,7 @@ ZTEST(fail_assume_in_before, test_to_skip1)
ZTEST_SUITE(fail_assume_in_test, NULL, NULL, NULL, NULL, NULL);
ZTEST_EXPECT_SKIP(fail_assume_in_test, test_to_skip);
ZTEST(fail_assume_in_test, test_to_skip)
{
zassume_true(false);