patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

ztest: Add CLI arguments to filter test/suites ran

Browse source 
Added test command line arguments to filter
which tests are executed. Filtered tests should follow
suiteA::test1,suiteB::test2 format.

Signed-off-by: Al Semjonovs <asemjonovs@google.com>
This commit is contained in:
Al Semjonovs 2022-03-30 13:38:00 -06:00 committed by Anas Nashif
parent f78a081066
commit 0411aa666c
6 changed files with 351 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
subsys/testsuite/ztest/CMakeLists.txt
View file

@ -16,3 +16,10 @@ zephyr_library_sources( src/ztest_error_hook.c)
zephyr_library_sources_ifdef(CONFIG_ZTEST_NEW_API src/ztest_rules.c)
zephyr_library_sources_ifdef(CONFIG_ZTEST_MOCKING src/ztest_mock.c)
zephyr_library_sources_ifdef(CONFIG_ZTRESS src/ztress.c)
if(CONFIG_ARCH_POSIX)
zephyr_library_sources_ifdef(CONFIG_ZTEST_NEW_API src/ztest_posix.c)
else()
zephyr_library_sources_ifdef(CONFIG_ZTEST_NEW_API src/ztest_defaults.c)
endif()

27
subsys/testsuite/ztest/include/ztest_test_new.h
View file

@ -124,6 +124,13 @@ extern struct ztest_suite_node _ztest_suite_node_list_end[];
.predicate = PREDICATE, \
.stats = &UTIL_CAT(z_ztest_test_node_stats_, SUITE_NAME), \
}
/**
* Default entry point for running or listing registered unit tests.
*
* @param state The current state of the machine as it relates to the test executable.
*/
void ztest_run_all(const void *state);
/**
* Run the registered unit tests which return true from their pragma function.
*
@ -156,6 +163,16 @@ void ztest_verify_all_test_suites_ran(void);
*/
int z_ztest_run_test_suite(const char *name);
/**
* @brief Returns next test within suite.
*
* @param suite Name of suite to get next test from.
* @param prev Previous unit test acquired from suite, use NULL to return first
* unit test.
* @return struct ztest_unit_test*
*/
struct ztest_unit_test *z_ztest_get_next_test(const char *suite, struct ztest_unit_test *prev);
/**
* @defgroup ztest_test Ztest testing macros
* @ingroup ztest
@ -356,6 +373,16 @@ extern struct k_mem_partition ztest_mem_partition;
*/
#define ztest_run_test_suite(suite) z_ztest_run_test_suite(STRINGIFY(suite))
/**
* @brief Structure for architecture specific APIs
*
*/
struct ztest_arch_api {
void (*run_all)(const void *state);
bool (*should_suite_run)(const void *state, struct ztest_suite_node *suite);
bool (*should_test_run)(const char *suite, const char *test);
};
/**
* @}
*/

11
subsys/testsuite/ztest/src/ztest.c
View file

@ -43,17 +43,8 @@ static ZTEST_BMEM int test_status;
* @param file Filename to check
* @returns Shortened filename, or @file if it could not be shortened
*/
const char *ztest_relative_filename(const char *file)
const char *__weak ztest_relative_filename(const char *file)
{
#ifdef CONFIG_ARCH_POSIX
const char *cwd;
char buf[200];
cwd = getcwd(buf, sizeof(buf));
if (cwd && strlen(file) > strlen(cwd) &&
!strncmp(file, cwd, strlen(cwd)))
return file + strlen(cwd) + 1; /* move past the trailing '/' */
#endif
return file;
}

75
subsys/testsuite/ztest/src/ztest_defaults.c Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright (c) 2022 Google LLC
*
* SPDX-License-Identifier: Apache-2.0
*/
#include "ztest_test_new.h"
/**
* @brief Try to shorten a filename by removing the current directory
*
* This helps to reduce the very long filenames in assertion failures. It
* removes the current directory from the filename and returns the rest.
* This makes assertions a lot more readable, and sometimes they fit on one
* line.
*
* @param file Filename to check
* @returns Shortened filename, or @file if it could not be shortened
*/
const char *ztest_relative_filename(const char *file)
{
return file;
}
/**
* Default entry point for running registered unit tests.
*
* @param state The current state of the machine as it relates to the test executable.
*/
void z_ztest_run_all(const void *state)
{
ztest_run_test_suites(state);
ztest_verify_all_test_suites_ran();
}
/**
* @brief Determines if the test suite should run based on test cases listed
* in the command line argument.
*
* @param state The current state of the machine as it relates to the test
* executable.
* @param suite Pointer to ztest_suite_node
* @return true
* @return false
*/
bool z_ztest_should_suite_run(const void *state, struct ztest_suite_node *suite)
{
bool run_suite = true;
if (suite->predicate != NULL) {
run_suite = suite->predicate(state);
}
return run_suite;
}
/**
* @brief Determines if the test case should run based on test cases listed
* in the command line argument. Run all tests for non-posix builds
*
* @param suite - name of test suite
* @param test - name of unit test
* @return true
* @return false
*/
bool z_ztest_should_test_run(const char *suite, const char *test)
{
return true;
}
ZTEST_DMEM const struct ztest_arch_api ztest_api = {
.run_all = z_ztest_run_all,
.should_suite_run = z_ztest_should_suite_run,
.should_test_run = z_ztest_should_test_run
};

58
subsys/testsuite/ztest/src/ztest_new.c
View file

@ -16,10 +16,6 @@
static struct k_thread ztest_thread;
#endif
#ifdef CONFIG_ARCH_POSIX
#include <unistd.h>
#endif
#ifdef CONFIG_ZTEST_SHUFFLE
#include <zephyr/random/rand32.h>
#include <stdlib.h>
@ -55,29 +51,7 @@ ZTEST_DMEM enum ztest_phase phase = TEST_PHASE_FRAMEWORK;
static ZTEST_BMEM int test_status;
/**
* @brief Try to shorten a filename by removing the current directory
*
* This helps to reduce the very long filenames in assertion failures. It
* removes the current directory from the filename and returns the rest.
* This makes assertions a lot more readable, and sometimes they fit on one
* line.
*
* @param file Filename to check
* @returns Shortened filename, or @file if it could not be shortened
*/
const char *ztest_relative_filename(const char *file)
{
#ifdef CONFIG_ARCH_POSIX
const char *cwd;
char buf[200];
cwd = getcwd(buf, sizeof(buf));
if (cwd && strlen(file) > strlen(cwd) && !strncmp(file, cwd, strlen(cwd)))
return file + strlen(cwd) + 1; /* move past the trailing '/' */
#endif
return file;
}
extern ZTEST_DMEM const struct ztest_arch_api ztest_api;
static int cleanup_test(struct ztest_unit_test *test)
{
@ -488,7 +462,7 @@ static struct ztest_suite_node *ztest_find_test_suite(const char *name)
return NULL;
}
struct ztest_unit_test *ztest_get_next_test(const char *suite, struct ztest_unit_test *prev)
struct ztest_unit_test *z_ztest_get_next_test(const char *suite, struct ztest_unit_test *prev)
{
struct ztest_unit_test *test = (prev == NULL) ? _ztest_unit_test_list_start : prev + 1;
@ -556,15 +530,19 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
if (strcmp(suite->name, test->test_suite_name) != 0) {
continue;
}
fail += run_test(suite, test, data);
if (ztest_api.should_test_run(suite->name, test->name)) {
fail += run_test(suite, test, data);
}
if (fail && FAIL_FAST) {
break;
}
}
#else
while (((test = ztest_get_next_test(suite->name, test)) != NULL)) {
fail += run_test(suite, test, data);
while (((test = z_ztest_get_next_test(suite->name, test)) != NULL)) {
if (ztest_api.should_test_run(suite->name, test->name)) {
fail += run_test(suite, test, data);
}
if (fail && FAIL_FAST) {
break;
@ -605,18 +583,10 @@ K_APPMEM_PARTITION_DEFINE(ztest_mem_partition);
static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *state)
{
struct ztest_suite_stats *stats = ptr->stats;
bool should_run = true;
int count = 0;
if (ptr->predicate != NULL) {
should_run = ptr->predicate(state);
} else {
/* If predicate is NULL, only run this test once. */
should_run = stats->run_count == 0;
}
for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
if (should_run) {
if (ztest_api.should_suite_run(state, ptr)) {
int fail = z_ztest_run_test_suite_ptr(ptr);
count++;
@ -680,10 +650,14 @@ void ztest_verify_all_test_suites_ran(void)
}
}
void ztest_run_all(const void *state)
{
ztest_api.run_all(state);
}
void __weak test_main(void)
{
ztest_run_test_suites(NULL);
ztest_verify_all_test_suites_ran();
ztest_run_all(NULL);
}
#ifndef KERNEL

225
subsys/testsuite/ztest/src/ztest_posix.c Normal file
View file

@ -0,0 +1,225 @@
/*
* Copyright (c) 2022 Google LLC
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include "cmdline.h" /* native_posix command line options header */
#include "soc.h"
#include "tc_util.h"
#include "ztest_test_new.h"
static const char *test_args;
static bool list_tests;
static void add_test_filter_option(void)
{
static struct args_struct_t test_filter_s[] = {
/*
* Fields:
* manual, mandatory, switch,
* option_name, var_name ,type,
* destination, callback,
* description
*/
{ false, false, true, "list", NULL, 'b', (void *)&list_tests, NULL,
"List all suite and test cases" },
{ false, false, false, "test", "suite::test", 's', (void *)&test_args, NULL,
"Name of tests to run. Comma separated list formatted as "
"\'suiteA::test1,suiteA::test2,suiteB::*\'. An * can be used "
"as a wildcard to run all tests within a suite." },
ARG_TABLE_ENDMARKER
};
native_add_command_line_opts(test_filter_s);
}
NATIVE_TASK(add_test_filter_option, PRE_BOOT_1, 10);
/**
* @brief Try to shorten a filename by removing the current directory
*
* This helps to reduce the very long filenames in assertion failures. It
* removes the current directory from the filename and returns the rest.
* This makes assertions a lot more readable, and sometimes they fit on one
* line.
*
* Overrides implementation in ztest_new.c
*
* @param file Filename to check
* @returns Shortened filename, or @file if it could not be shortened
*/
const char *ztest_relative_filename(const char *file)
{
const char *cwd;
char buf[200];
cwd = getcwd(buf, sizeof(buf));
if (cwd && strlen(file) > strlen(cwd) && !strncmp(file, cwd, strlen(cwd))) {
return file + strlen(cwd) + 1; /* move past the trailing '/' */
}
return file;
}
/**
* @brief Helper function to set list_tests
*
* @param value - Sets list_tests to value
*/
void ztest_set_list_test(bool value)
{
list_tests = value;
}
/**
* @brief Helper function to get command line argument for listing tests
*
* @return true
* @return false
*/
bool z_ztest_get_list_test(void)
{
return list_tests;
}
/**
* @brief Helper function to get command line test arguments
*
* @return const char*
*/
const char *ztest_get_test_args(void)
{
return test_args;
}
/**
* @brief Lists registered unit tests in this binary, one per line
*
* @return int Number of tests in binary
*/
int z_ztest_list_tests(void)
{
struct ztest_suite_node *ptr;
struct ztest_unit_test *test = NULL;
int test_count = 0;
for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
test = NULL;
while ((test = z_ztest_get_next_test(ptr->name, test)) != NULL) {
TC_PRINT("%s::%s\n", test->test_suite_name, test->name);
test_count++;
}
}
/* List tests only once */
ztest_set_list_test(false);
return test_count;
}
/**
* Default entry point for running or listing registered unit tests.
*
* @param state The current state of the machine as it relates to the test executable.
*/
void z_ztest_run_all(const void *state)
{
if (z_ztest_get_list_test()) {
z_ztest_list_tests();
} else {
ztest_run_test_suites(state);
ztest_verify_all_test_suites_ran();
}
}
/**
* @brief Checks if the test_args contains the suite/test name.
*
* @param suite_name
* @param test_name
* @return true
* @return false
*/
static bool z_ztest_testargs_contains(const char *suite_name, const char *test_name)
{
bool found = false;
char *test_args_local = strdup(test_args);
char *suite_test_pair;
char *last_suite_test_pair;
char *suite_arg;
char *test_arg;
char *last_arg;
suite_test_pair = strtok_r(test_args_local, ",", &last_suite_test_pair);
while (suite_test_pair && !found) {
suite_arg = strtok_r(suite_test_pair, ":", &last_arg);
test_arg = strtok_r(NULL, ":", &last_arg);
found = !strcmp(suite_arg, suite_name);
if (test_name) {
found &= !strcmp(test_arg, "*") ||
!strcmp(test_arg, test_name);
}
suite_test_pair = strtok_r(NULL, ",", &last_suite_test_pair);
}
free(test_args_local);
return found;
}
/**
* @brief Determines if the test case should run based on test cases listed
* in the command line argument.
*
* Overrides implementation in ztest_new.c
*
* @param suite - name of test suite
* @param test - name of unit test
* @return true
* @return false
*/
bool z_ztest_should_test_run(const char *suite, const char *test)
{
bool run_test = false;
run_test = (test_args == NULL ||
z_ztest_testargs_contains(suite, test));
return run_test;
}
/**
* @brief Determines if the test suite should run based on test cases listed
* in the command line argument.
*
* Overrides implementation in ztest_new.c
*
* @param state The current state of the machine as it relates to the test
* executable.
* @param suite Pointer to ztest_suite_node
* @return true
* @return false
*/
bool z_ztest_should_suite_run(const void *state, struct ztest_suite_node *suite)
{
bool run_suite = true;
if (test_args != NULL && !z_ztest_testargs_contains(suite->name, NULL)) {
run_suite = false;
suite->stats->run_count++;
} else if (suite->predicate != NULL) {
run_suite = suite->predicate(state);
}
return run_suite;
}
ZTEST_DMEM const struct ztest_arch_api ztest_api = {
.run_all = z_ztest_run_all,
.should_suite_run = z_ztest_should_suite_run,
.should_test_run = z_ztest_should_test_run
};