ztest: Add register functionality
Add new functionality to ztest to improve test modularity. The two primary new entry points are: * ztest_register_test_suite * ztest_run_registered_test_suites When registering a new test suite, users provide the name as well as an optional predicate used to filter the tests for each run. Using NULL as the predicate ensures that the test is run exactly once (after which it is automatically filtered from future runs). Calls to ztest_run_registered_test_suites take a state pointer as an argument. This allows the the pragma functions to decide whether the test should be run. The biggest benefit of this system (other than the ability to filter tests and maintain a larger test state) is the ability to better modularize the test source code. Instead of all the various tests having to coordinate and the main function having to know which tests to run, each source file manages registering its own test suite and handling the conditions for running the suite. Signed-off-by: Yuval Peress <peress@chromium.org>
This commit is contained in:
Yuval Peress
Anas Nashif
parent
87c1f9a6e4
commit
dee79d2b66
|
|
@ -140,3 +140,7 @@
|
|||
#ifdef CONFIG_USERSPACE
|
||||
_static_kernel_objects_end = .;
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_ZTEST)
|
||||
ITERABLE_SECTION_RAM(ztest_suite_node, 4)
|
||||
#endif /* CONFIG_ZTEST */
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
#
|
||||
# Copyright (c) 2018 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import contextlib
|
||||
import string
|
||||
|
|
@ -34,6 +33,7 @@ import platform
|
|||
import yaml
|
||||
import json
|
||||
from multiprocessing import Lock, Process, Value
|
||||
from typing import List
|
||||
|
||||
try:
|
||||
# Use the C LibYAML parser if available, rather than the Python parser.
|
||||
|
|
@ -1576,6 +1576,41 @@ class DisablePyTestCollectionMixin(object):
|
|||
__test__ = False
|
||||
|
||||
|
||||
class ScanPathResult:
|
||||
"""Result of the TestCase.scan_path function call.
|
||||
|
||||
Attributes:
|
||||
matches A list of test cases
|
||||
warnings A string containing one or more
|
||||
warnings to display
|
||||
has_registered_test_suites Whether or not the path contained any
|
||||
calls to the ztest_register_test_suite
|
||||
macro.
|
||||
has_run_registered_test_suites Whether or not the path contained at
|
||||
least one call to
|
||||
ztest_run_registered_test_suites.
|
||||
"""
|
||||
def __init__(self,
|
||||
matches: List[str] = None,
|
||||
warnings: str = None,
|
||||
has_registered_test_suites: bool = False,
|
||||
has_run_registered_test_suites: bool = False):
|
||||
self.matches = matches
|
||||
self.warnings = warnings
|
||||
self.has_registered_test_suites = has_registered_test_suites
|
||||
self.has_run_registered_test_suites = has_run_registered_test_suites
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, ScanPathResult):
|
||||
return False
|
||||
return (sorted(self.matches) == sorted(other.matches) and
|
||||
self.warnings == other.warnings and
|
||||
(self.has_registered_test_suites ==
|
||||
other.has_registered_test_suites) and
|
||||
(self.has_run_registered_test_suites ==
|
||||
other.has_run_registered_test_suites))
|
||||
|
||||
|
||||
class TestCase(DisablePyTestCollectionMixin):
|
||||
"""Class representing a test application
|
||||
"""
|
||||
|
|
@ -1662,29 +1697,42 @@ Tests should reference the category and subsystem with a dot as a separator.
|
|||
# line--as we only search starting the end of this match
|
||||
br"^\s*ztest_test_suite\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
|
||||
re.MULTILINE)
|
||||
registered_suite_regex = re.compile(
|
||||
br"^\s*ztest_register_test_suite"
|
||||
br"\(\s*(?P<suite_name>[a-zA-Z0-9_]+)\s*,",
|
||||
re.MULTILINE)
|
||||
stc_regex = re.compile(
|
||||
br"^\s*" # empy space at the beginning is ok
|
||||
br"""^\s* # empy space at the beginning is ok
|
||||
# catch the case where it is declared in the same sentence, e.g:
|
||||
#
|
||||
# ztest_test_suite(mutex_complex, ztest_user_unit_test(TESTNAME));
|
||||
br"(?:ztest_test_suite\([a-zA-Z0-9_]+,\s*)?"
|
||||
# ztest_register_test_suite(n, p, ztest_user_unit_test(TESTNAME),
|
||||
(?:ztest_
|
||||
(?:test_suite\(|register_test_suite\([a-zA-Z0-9_]+\s*,\s*)
|
||||
[a-zA-Z0-9_]+\s*,\s*
|
||||
)?
|
||||
# Catch ztest[_user]_unit_test-[_setup_teardown](TESTNAME)
|
||||
br"ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?"
|
||||
ztest_(?:1cpu_)?(?:user_)?unit_test(?:_setup_teardown)?
|
||||
# Consume the argument that becomes the extra testcse
|
||||
br"\(\s*"
|
||||
br"(?P<stc_name>[a-zA-Z0-9_]+)"
|
||||
\(\s*(?P<stc_name>[a-zA-Z0-9_]+)
|
||||
# _setup_teardown() variant has two extra arguments that we ignore
|
||||
br"(?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?"
|
||||
br"\s*\)",
|
||||
(?:\s*,\s*[a-zA-Z0-9_]+\s*,\s*[a-zA-Z0-9_]+)?
|
||||
\s*\)""",
|
||||
# We don't check how it finishes; we don't care
|
||||
re.MULTILINE)
|
||||
re.MULTILINE | re.VERBOSE)
|
||||
suite_run_regex = re.compile(
|
||||
br"^\s*ztest_run_test_suite\((?P<suite_name>[a-zA-Z0-9_]+)\)",
|
||||
re.MULTILINE)
|
||||
registered_suite_run_regex = re.compile(
|
||||
br"^\s*ztest_run_registered_test_suites\("
|
||||
br"(\*+|&)?(?P<state_identifier>[a-zA-Z0-9_]+)\)",
|
||||
re.MULTILINE)
|
||||
achtung_regex = re.compile(
|
||||
br"(#ifdef|#endif)",
|
||||
re.MULTILINE)
|
||||
warnings = None
|
||||
has_registered_test_suites = False
|
||||
has_run_registered_test_suites = False
|
||||
|
||||
with open(inf_name) as inf:
|
||||
if os.name == 'nt':
|
||||
|
|
@ -1695,52 +1743,94 @@ Tests should reference the category and subsystem with a dot as a separator.
|
|||
|
||||
with contextlib.closing(mmap.mmap(**mmap_args)) as main_c:
|
||||
suite_regex_match = suite_regex.search(main_c)
|
||||
if not suite_regex_match:
|
||||
registered_suite_regex_match = registered_suite_regex.search(
|
||||
main_c)
|
||||
|
||||
if registered_suite_regex_match:
|
||||
has_registered_test_suites = True
|
||||
if registered_suite_run_regex.search(main_c):
|
||||
has_run_registered_test_suites = True
|
||||
|
||||
if not suite_regex_match and not has_registered_test_suites:
|
||||
# can't find ztest_test_suite, maybe a client, because
|
||||
# it includes ztest.h
|
||||
return None, None
|
||||
return ScanPathResult(
|
||||
matches=None,
|
||||
warnings=None,
|
||||
has_registered_test_suites=has_registered_test_suites,
|
||||
has_run_registered_test_suites=has_run_registered_test_suites)
|
||||
|
||||
suite_run_match = suite_run_regex.search(main_c)
|
||||
if not suite_run_match:
|
||||
if suite_regex_match and not suite_run_match:
|
||||
raise ValueError("can't find ztest_run_test_suite")
|
||||
|
||||
if suite_regex_match:
|
||||
search_start = suite_regex_match.end()
|
||||
else:
|
||||
search_start = registered_suite_regex_match.end()
|
||||
|
||||
if suite_run_match:
|
||||
search_end = suite_run_match.start()
|
||||
else:
|
||||
search_end = re.compile(br"\);", re.MULTILINE) \
|
||||
.search(main_c, search_start) \
|
||||
.end()
|
||||
achtung_matches = re.findall(
|
||||
achtung_regex,
|
||||
main_c[suite_regex_match.end():suite_run_match.start()])
|
||||
main_c[search_start:search_end])
|
||||
if achtung_matches:
|
||||
warnings = "found invalid %s in ztest_test_suite()" \
|
||||
% ", ".join(sorted({match.decode() for match in achtung_matches},reverse = True))
|
||||
_matches = re.findall(
|
||||
stc_regex,
|
||||
main_c[suite_regex_match.end():suite_run_match.start()])
|
||||
main_c[search_start:search_end])
|
||||
for match in _matches:
|
||||
if not match.decode().startswith("test_"):
|
||||
warnings = "Found a test that does not start with test_"
|
||||
matches = [match.decode().replace("test_", "", 1) for match in _matches]
|
||||
return matches, warnings
|
||||
return ScanPathResult(
|
||||
matches=matches,
|
||||
warnings=warnings,
|
||||
has_registered_test_suites=has_registered_test_suites,
|
||||
has_run_registered_test_suites=has_run_registered_test_suites)
|
||||
|
||||
def scan_path(self, path):
|
||||
subcases = []
|
||||
has_registered_test_suites = False
|
||||
has_run_registered_test_suites = False
|
||||
for filename in glob.glob(os.path.join(path, "src", "*.c*")):
|
||||
try:
|
||||
_subcases, warnings = self.scan_file(filename)
|
||||
if warnings:
|
||||
logger.error("%s: %s" % (filename, warnings))
|
||||
raise TwisterRuntimeError("%s: %s" % (filename, warnings))
|
||||
if _subcases:
|
||||
subcases += _subcases
|
||||
result: ScanPathResult = self.scan_file(filename)
|
||||
if result.warnings:
|
||||
logger.error("%s: %s" % (filename, result.warnings))
|
||||
raise TwisterRuntimeError(
|
||||
"%s: %s" % (filename, result.warnings))
|
||||
if result.matches:
|
||||
subcases += result.matches
|
||||
if result.has_registered_test_suites:
|
||||
has_registered_test_suites = True
|
||||
if result.has_run_registered_test_suites:
|
||||
has_run_registered_test_suites = True
|
||||
except ValueError as e:
|
||||
logger.error("%s: can't find: %s" % (filename, e))
|
||||
|
||||
for filename in glob.glob(os.path.join(path, "*.c")):
|
||||
try:
|
||||
_subcases, warnings = self.scan_file(filename)
|
||||
if warnings:
|
||||
logger.error("%s: %s" % (filename, warnings))
|
||||
if _subcases:
|
||||
subcases += _subcases
|
||||
result: ScanPathResult = self.scan_file(filename)
|
||||
if result.warnings:
|
||||
logger.error("%s: %s" % (filename, result.warnings))
|
||||
if result.matches:
|
||||
subcases += result.matches
|
||||
except ValueError as e:
|
||||
logger.error("%s: can't find: %s" % (filename, e))
|
||||
|
||||
if has_registered_test_suites and not has_run_registered_test_suites:
|
||||
warning = \
|
||||
"Found call to 'ztest_register_test_suite()' but no "\
|
||||
"call to 'ztest_run_registered_test_suites()'"
|
||||
logger.error(warning)
|
||||
raise TwisterRuntimeError(warning)
|
||||
|
||||
return subcases
|
||||
|
||||
def parse_subcases(self, test_path):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
tests:
|
||||
test_d.check_1:
|
||||
tags: test_d
|
||||
build_only: True
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
ztest_register_test_suite(feature4, NULL,
|
||||
ztest_unit_test(test_unit_1a),
|
||||
ztest_unit_test(test_unit_1b));
|
||||
|
|
@ -13,7 +13,8 @@ import pytest
|
|||
|
||||
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
||||
from twisterlib import TestInstance, BuildError, TestCase, TwisterException
|
||||
from twisterlib import (TestInstance, BuildError, TestCase, TwisterException,
|
||||
ScanPathResult)
|
||||
|
||||
|
||||
TESTDATA_1 = [
|
||||
|
|
@ -112,21 +113,45 @@ def test_get_unique_exception(testcase_root, workdir, name, exception):
|
|||
unique = TestCase(testcase_root, workdir, name)
|
||||
assert unique == exception
|
||||
|
||||
|
||||
TESTDATA_5 = [
|
||||
("testcases/tests/test_ztest.c", None, ['a', 'c', 'unit_a', 'newline', 'test_test_aa', 'user', 'last']),
|
||||
("testcases/tests/test_a/test_ztest_error.c", "Found a test that does not start with test_", ['1a', '1c', '2a', '2b']),
|
||||
("testcases/tests/test_a/test_ztest_error_1.c", "found invalid #ifdef, #endif in ztest_test_suite()", ['unit_1a', 'unit_1b', 'Unit_1c']),
|
||||
("testcases/tests/test_ztest.c",
|
||||
ScanPathResult(
|
||||
warnings=None,
|
||||
matches=['a', 'c', 'unit_a',
|
||||
'newline',
|
||||
'test_test_aa',
|
||||
'user', 'last'],
|
||||
has_registered_test_suites=False,
|
||||
has_run_registered_test_suites=False)),
|
||||
("testcases/tests/test_a/test_ztest_error.c",
|
||||
ScanPathResult(
|
||||
warnings="Found a test that does not start with test_",
|
||||
matches=['1a', '1c', '2a', '2b'],
|
||||
has_registered_test_suites=False,
|
||||
has_run_registered_test_suites=False)),
|
||||
("testcases/tests/test_a/test_ztest_error_1.c",
|
||||
ScanPathResult(
|
||||
warnings="found invalid #ifdef, #endif in ztest_test_suite()",
|
||||
matches=['unit_1a', 'unit_1b', 'Unit_1c'],
|
||||
has_registered_test_suites=False,
|
||||
has_run_registered_test_suites=False)),
|
||||
("testcases/tests/test_d/test_ztest_error_register_test_suite.c",
|
||||
ScanPathResult(
|
||||
warnings=None, matches=['unit_1a', 'unit_1b'],
|
||||
has_registered_test_suites=True,
|
||||
has_run_registered_test_suites=False)),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("test_file, expected_warnings, expected_subcases", TESTDATA_5)
|
||||
def test_scan_file(test_data, test_file, expected_warnings, expected_subcases):
|
||||
@pytest.mark.parametrize("test_file, expected", TESTDATA_5)
|
||||
def test_scan_file(test_data, test_file, expected: ScanPathResult):
|
||||
'''Testing scan_file method with different ztest files for warnings and results'''
|
||||
|
||||
testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".", "test_a.check_1")
|
||||
testcase = TestCase("/scripts/tests/twister/test_data/testcases/tests", ".",
|
||||
"test_a.check_1")
|
||||
|
||||
results, warnings = testcase.scan_file(os.path.join(test_data, test_file))
|
||||
assert sorted(results) == sorted(expected_subcases)
|
||||
assert warnings == expected_warnings
|
||||
result: ScanPathResult = testcase.scan_file(os.path.join(test_data, test_file))
|
||||
assert result == expected
|
||||
|
||||
|
||||
TESTDATA_6 = [
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ def test_testsuite_add_testcases(class_testsuite):
|
|||
'test_c.check_2',
|
||||
'test_a.check_1',
|
||||
'test_a.check_2',
|
||||
'test_d.check_1',
|
||||
'sample_test.app']
|
||||
testcase_list = []
|
||||
for key in sorted(class_testsuite.testcases.keys()):
|
||||
|
|
@ -58,9 +59,18 @@ def test_add_configurations(test_data, class_testsuite, board_root_dir):
|
|||
def test_get_all_testcases(class_testsuite, all_testcases_dict):
|
||||
""" Testing get_all_testcases function of TestSuite class in Twister """
|
||||
class_testsuite.testcases = all_testcases_dict
|
||||
expected_tests = ['sample_test.app', 'test_a.check_1.1a', 'test_a.check_1.1c',
|
||||
'test_a.check_1.2a', 'test_a.check_1.2b', 'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a', 'test_a.check_1.unit_1b', 'test_a.check_2.1a', 'test_a.check_2.1c', 'test_a.check_2.2a', 'test_a.check_2.2b', 'test_a.check_2.Unit_1c', 'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b', 'test_b.check_1', 'test_b.check_2', 'test_c.check_1', 'test_c.check_2']
|
||||
assert len(class_testsuite.get_all_tests()) == 19
|
||||
expected_tests = ['sample_test.app', 'test_a.check_1.1a',
|
||||
'test_a.check_1.1c',
|
||||
'test_a.check_1.2a', 'test_a.check_1.2b',
|
||||
'test_a.check_1.Unit_1c', 'test_a.check_1.unit_1a',
|
||||
'test_a.check_1.unit_1b', 'test_a.check_2.1a',
|
||||
'test_a.check_2.1c', 'test_a.check_2.2a',
|
||||
'test_a.check_2.2b', 'test_a.check_2.Unit_1c',
|
||||
'test_a.check_2.unit_1a', 'test_a.check_2.unit_1b',
|
||||
'test_b.check_1', 'test_b.check_2', 'test_c.check_1',
|
||||
'test_c.check_2', 'test_d.check_1.unit_1a',
|
||||
'test_d.check_1.unit_1b']
|
||||
assert len(class_testsuite.get_all_tests()) == len(expected_tests)
|
||||
assert sorted(class_testsuite.get_all_tests()) == sorted(expected_tests)
|
||||
|
||||
def test_get_platforms(class_testsuite, platforms_list):
|
||||
|
|
|
|||
16
subsys/testsuite/include/ztest.ld
Normal file
16
subsys/testsuite/include/ztest.ld
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
SECTIONS
|
||||
{
|
||||
.data.ztest_suite_node_area : ALIGN(4)
|
||||
{
|
||||
_ztest_suite_node_list_start = .;
|
||||
KEEP(*(SORT_BY_NAME(._ztest_suite_node.static.*)))
|
||||
_ztest_suite_node_list_end = .;
|
||||
}
|
||||
}
|
||||
INSERT AFTER .data;
|
||||
|
|
@ -62,6 +62,10 @@ target_compile_options(testbinary PRIVATE
|
|||
$<$<COMPILE_LANGUAGE:ASM>:${EXTRA_AFLAGS_AS_LIST}>
|
||||
)
|
||||
|
||||
target_link_options(testbinary PRIVATE
|
||||
-T "${ZEPHYR_BASE}/subsys/testsuite/include/ztest.ld"
|
||||
)
|
||||
|
||||
target_link_libraries(testbinary PRIVATE
|
||||
${EXTRA_LDFLAGS_AS_LIST}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
#define ZEPHYR_TESTSUITE_ZTEST_TEST_H_
|
||||
|
||||
#include <app_memory/app_memdomain.h>
|
||||
#include <init.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -27,7 +29,80 @@ struct unit_test {
|
|||
uint32_t thread_options;
|
||||
};
|
||||
|
||||
void z_ztest_run_test_suite(const char *name, struct unit_test *suite);
|
||||
/**
|
||||
* Stats about a ztest suite
|
||||
*/
|
||||
struct ztest_suite_stats {
|
||||
/** The number of times that the suite ran */
|
||||
uint32_t run_count;
|
||||
/** The number of times that the suite was skipped */
|
||||
uint32_t skip_count;
|
||||
/** The number of times that the suite failed */
|
||||
uint32_t fail_count;
|
||||
};
|
||||
|
||||
/**
|
||||
* A single node of test suite. Each node should be added to a single linker section which will
|
||||
* allow ztest_run_registered_test_suites() to iterate over the various nodes.
|
||||
*/
|
||||
struct ztest_suite_node {
|
||||
/** The name of the test suite. */
|
||||
const char *name;
|
||||
/** Pointer to the test suite. */
|
||||
struct unit_test *suite;
|
||||
/**
|
||||
* An optional predicate function to determine if the test should run. If NULL, then the
|
||||
* test will only run once on the first attempt.
|
||||
*
|
||||
* @param state The current state of the test application.
|
||||
* @return True if the suite should be run; false to skip.
|
||||
*/
|
||||
bool (*predicate)(const void *state);
|
||||
/** Stats */
|
||||
struct ztest_suite_stats stats;
|
||||
};
|
||||
|
||||
extern struct ztest_suite_node _ztest_suite_node_list_start[];
|
||||
extern struct ztest_suite_node _ztest_suite_node_list_end[];
|
||||
|
||||
/**
|
||||
* Create and register a ztest suite. Using this macro creates a new test suite (using
|
||||
* ztest_test_suite). It then creates a struct ztest_suite_node in a specific linker section.
|
||||
*
|
||||
* Tests can then be run by calling ztest_run_registered_test_suites(const void *state) by passing
|
||||
* in the current state. See the documentation for ztest_run_registered_test_suites for more info.
|
||||
*
|
||||
* @param SUITE_NAME The name of the suite (see ztest_test_suite for more info)
|
||||
* @param PREDICATE A function to test against the state and determine if the test should run.
|
||||
* @param args Varargs placeholder for the remaining arguments passed for the unit tests.
|
||||
*/
|
||||
#define ztest_register_test_suite(SUITE_NAME, PREDICATE, args...) \
|
||||
ztest_test_suite(SUITE_NAME, ##args); \
|
||||
static STRUCT_SECTION_ITERABLE(ztest_suite_node, z_ztest_test_node_##SUITE_NAME) = { \
|
||||
.name = #SUITE_NAME, \
|
||||
.suite = _##SUITE_NAME, \
|
||||
.predicate = PREDICATE, \
|
||||
};
|
||||
|
||||
/**
|
||||
* Run the registered unit tests which return true from their pragma function.
|
||||
*
|
||||
* @param state The current state of the machine as it relates to the test executable.
|
||||
* @return The number of tests that ran.
|
||||
*/
|
||||
int ztest_run_registered_test_suites(const void *state);
|
||||
|
||||
/**
|
||||
* @brief Run a test suite.
|
||||
*
|
||||
* Internal implementation. Do not call directly. This will run the full test suite along with some
|
||||
* checks for fast failures and initialization.
|
||||
*
|
||||
* @param name The name of the suite to run.
|
||||
* @param suite Pointer to the first unit test.
|
||||
* @return Negative value if the test suite never ran; otherwise, return the number of failures.
|
||||
*/
|
||||
int z_ztest_run_test_suite(const char *name, struct unit_test *suite);
|
||||
|
||||
/**
|
||||
* @defgroup ztest_test Ztest testing macros
|
||||
|
|
|
|||
|
|
@ -396,12 +396,12 @@ static int run_test(struct unit_test *test)
|
|||
|
||||
#endif /* !KERNEL */
|
||||
|
||||
void z_ztest_run_test_suite(const char *name, struct unit_test *suite)
|
||||
int z_ztest_run_test_suite(const char *name, struct unit_test *suite)
|
||||
{
|
||||
int fail = 0;
|
||||
|
||||
if (test_status < 0) {
|
||||
return;
|
||||
return test_status;
|
||||
}
|
||||
|
||||
init_testing();
|
||||
|
|
@ -418,6 +418,8 @@ void z_ztest_run_test_suite(const char *name, struct unit_test *suite)
|
|||
TC_SUITE_END(name, (fail > 0 ? TC_FAIL : TC_PASS));
|
||||
|
||||
test_status = (test_status || fail) ? 1 : 0;
|
||||
|
||||
return fail;
|
||||
}
|
||||
|
||||
void end_report(void)
|
||||
|
|
@ -433,6 +435,36 @@ void end_report(void)
|
|||
K_APPMEM_PARTITION_DEFINE(ztest_mem_partition);
|
||||
#endif
|
||||
|
||||
int ztest_run_registered_test_suites(const void *state)
|
||||
{
|
||||
struct ztest_suite_node *ptr;
|
||||
int count = 0;
|
||||
|
||||
for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
|
||||
struct ztest_suite_stats *stats = &ptr->stats;
|
||||
bool should_run = true;
|
||||
|
||||
if (ptr->predicate != NULL) {
|
||||
should_run = ptr->predicate(state);
|
||||
} else {
|
||||
/* If pragma is NULL, only run this test once. */
|
||||
should_run = stats->run_count == 0;
|
||||
}
|
||||
|
||||
if (should_run) {
|
||||
int fail = z_ztest_run_test_suite(ptr->name, ptr->suite);
|
||||
|
||||
count++;
|
||||
stats->run_count++;
|
||||
stats->fail_count += (fail != 0) ? 1 : 0;
|
||||
} else {
|
||||
stats->skip_count++;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
#ifndef KERNEL
|
||||
int main(void)
|
||||
{
|
||||
|
|
|
|||
10
tests/ztest/register/CMakeLists.txt
Normal file
10
tests/ztest/register/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
# Copyright 2021 Google LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
cmake_minimum_required(VERSION 3.20.0)
|
||||
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
|
||||
project(register)
|
||||
|
||||
FILE(GLOB app_sources src/*.c)
|
||||
target_include_directories(app PRIVATE include)
|
||||
target_sources(app PRIVATE ${app_sources})
|
||||
23
tests/ztest/register/include/common.h
Normal file
23
tests/ztest/register/include/common.h
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#ifndef TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_
|
||||
#define TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
enum phase {
|
||||
PHASE_VERIFY,
|
||||
PHASE_NULL_PREDICATE_0,
|
||||
PHASE_NULL_PREDICATE_1,
|
||||
PHASE_STEPS_0,
|
||||
PHASE_STEPS_1,
|
||||
};
|
||||
struct global_test_state {
|
||||
enum phase phase;
|
||||
};
|
||||
|
||||
#endif /* TESTS_ZTEST_REGISTER_INCLUDE_COMMON_H_ */
|
||||
1
tests/ztest/register/prj.conf
Normal file
1
tests/ztest/register/prj.conf
Normal file
|
|
@ -0,0 +1 @@
|
|||
CONFIG_ZTEST=y
|
||||
198
tests/ztest/register/src/main.c
Normal file
198
tests/ztest/register/src/main.c
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <string.h>
|
||||
#include <ztest.h>
|
||||
#include "common.h"
|
||||
|
||||
#define num_registered_suites (_ztest_suite_node_list_end - _ztest_suite_node_list_start)
|
||||
|
||||
/** The current state of the test application. */
|
||||
static struct global_test_state global_state;
|
||||
|
||||
/**
|
||||
* Copies of the test entry's snapshot, used to test assertions. There's no good way to get the
|
||||
* number of registered test suites at compile time so this is set to an arbitrary large size that
|
||||
* should be enough even if the number of tests grows. There's a runtime check for this in
|
||||
* test_verify_execution.
|
||||
*/
|
||||
static struct ztest_suite_stats stats_snapshot[128];
|
||||
|
||||
/** The results of a single execution */
|
||||
struct execution_results {
|
||||
/** The test phase that was run */
|
||||
enum phase test_phase;
|
||||
/** The number of tests that ran */
|
||||
int test_run_count;
|
||||
} execution_results;
|
||||
|
||||
/**
|
||||
* Helper function used to find a test entry by name.
|
||||
*
|
||||
* @param name The name of the test entry.
|
||||
* @return Pointer to the struct unit_test_node or NULL if not found.
|
||||
*/
|
||||
static struct ztest_suite_node *find_test_node(const char *name)
|
||||
{
|
||||
struct ztest_suite_node *ptr;
|
||||
|
||||
for (ptr = _ztest_suite_node_list_start; ptr != _ztest_suite_node_list_end; ++ptr) {
|
||||
if (strcmp(ptr->name, name) == 0) {
|
||||
return ptr;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Find a snapshot in the stats_snapshot array
|
||||
*
|
||||
* Lookup a test case by name and find the matching ztest_suite_stats.
|
||||
*
|
||||
* @param name The name of the test entry.
|
||||
* @return Pointer to the stats snapshot.
|
||||
*/
|
||||
static struct ztest_suite_stats *find_snapshot(const char *name)
|
||||
{
|
||||
int index = find_test_node(name) - _ztest_suite_node_list_start;
|
||||
|
||||
return stats_snapshot + index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the global state between phases. This function can be thought of similarly to making a
|
||||
* change affecting the state of the application being tested.
|
||||
*
|
||||
* @param phase The new phase of the application.
|
||||
*/
|
||||
static void reset_state(enum phase phase)
|
||||
{
|
||||
execution_results.test_phase = phase;
|
||||
execution_results.test_run_count = 0;
|
||||
global_state.phase = phase;
|
||||
|
||||
for (int i = 0; i < num_registered_suites; ++i) {
|
||||
stats_snapshot[i] = _ztest_suite_node_list_start[i].stats;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a snapshot of the tests' stats. This function should be called after each run in order
|
||||
* to assert on only the changes in the stats.
|
||||
*/
|
||||
static void take_stats_snapshot(void)
|
||||
{
|
||||
for (int i = 0; i < num_registered_suites; ++i) {
|
||||
struct ztest_suite_stats *snapshot = stats_snapshot + i;
|
||||
struct ztest_suite_stats *current = &_ztest_suite_node_list_start[i].stats;
|
||||
|
||||
snapshot->run_count = current->run_count - snapshot->run_count;
|
||||
snapshot->skip_count = current->skip_count - snapshot->skip_count;
|
||||
snapshot->fail_count = current->fail_count - snapshot->fail_count;
|
||||
}
|
||||
}
|
||||
|
||||
static void test_verify_execution(void)
|
||||
{
|
||||
const struct ztest_suite_stats *stats;
|
||||
|
||||
zassert_true(ARRAY_SIZE(stats_snapshot) >= num_registered_suites,
|
||||
"Not enough stats snapshots, please allocate more.");
|
||||
switch (execution_results.test_phase) {
|
||||
case PHASE_NULL_PREDICATE_0:
|
||||
/* Verify that only remove_first_node suite was run and removed. */
|
||||
stats = find_snapshot("run_null_predicate_once");
|
||||
zassert_equal(1, execution_results.test_run_count, NULL);
|
||||
zassert_equal(1, stats->run_count, NULL);
|
||||
zassert_equal(0, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
break;
|
||||
case PHASE_NULL_PREDICATE_1:
|
||||
/* Verify that only remove_first_two_nodes_* were run. */
|
||||
zassert_equal(0, execution_results.test_run_count, NULL);
|
||||
stats = find_snapshot("run_null_predicate_once");
|
||||
zassert_equal(0, stats->run_count, NULL);
|
||||
zassert_equal(1, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
break;
|
||||
case PHASE_STEPS_0:
|
||||
/* Verify that steps_0 and steps_all suites were run. */
|
||||
zassert_equal(2, execution_results.test_run_count, NULL);
|
||||
stats = find_snapshot("test_step_0");
|
||||
zassert_equal(1, stats->run_count, NULL);
|
||||
zassert_equal(0, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
stats = find_snapshot("test_step_1");
|
||||
zassert_equal(0, stats->run_count, NULL);
|
||||
zassert_equal(1, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
stats = find_snapshot("test_step_all");
|
||||
zassert_equal(1, stats->run_count, NULL);
|
||||
zassert_equal(0, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
break;
|
||||
case PHASE_STEPS_1:
|
||||
/* Verify that steps_1 and steps_all suites were run. */
|
||||
zassert_equal(2, execution_results.test_run_count, NULL);
|
||||
stats = find_snapshot("test_step_0");
|
||||
zassert_equal(0, stats->run_count, NULL);
|
||||
zassert_equal(1, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
stats = find_snapshot("test_step_1");
|
||||
zassert_equal(1, stats->run_count, NULL);
|
||||
zassert_equal(0, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
stats = find_snapshot("test_step_all");
|
||||
zassert_equal(1, stats->run_count, NULL);
|
||||
zassert_equal(0, stats->skip_count, NULL);
|
||||
zassert_equal(0, stats->fail_count, NULL);
|
||||
break;
|
||||
default:
|
||||
ztest_test_fail();
|
||||
}
|
||||
}
|
||||
|
||||
static bool verify_predicate(const void *state)
|
||||
{
|
||||
const struct global_test_state *s = state;
|
||||
|
||||
return s->phase == PHASE_VERIFY;
|
||||
}
|
||||
|
||||
ztest_register_test_suite(verify, verify_predicate,
|
||||
ztest_unit_test(test_verify_execution));
|
||||
|
||||
void test_main(void)
|
||||
{
|
||||
/* Make sure that when predicate is set to NULL, the test is run. */
|
||||
reset_state(PHASE_NULL_PREDICATE_0);
|
||||
execution_results.test_run_count = ztest_run_registered_test_suites(&global_state);
|
||||
take_stats_snapshot();
|
||||
global_state.phase = PHASE_VERIFY;
|
||||
ztest_run_registered_test_suites(&global_state);
|
||||
|
||||
/* Try running the tests again, nothing should run. */
|
||||
reset_state(PHASE_NULL_PREDICATE_1);
|
||||
execution_results.test_run_count = ztest_run_registered_test_suites(&global_state);
|
||||
take_stats_snapshot();
|
||||
global_state.phase = PHASE_VERIFY;
|
||||
ztest_run_registered_test_suites(&global_state);
|
||||
|
||||
/* Run filter tests for step 0. */
|
||||
reset_state(PHASE_STEPS_0);
|
||||
execution_results.test_run_count = ztest_run_registered_test_suites(&global_state);
|
||||
global_state.phase = PHASE_VERIFY;
|
||||
take_stats_snapshot();
|
||||
ztest_run_registered_test_suites(&global_state);
|
||||
|
||||
/* Run filter tests for step 1. */
|
||||
reset_state(PHASE_STEPS_1);
|
||||
execution_results.test_run_count = ztest_run_registered_test_suites(&global_state);
|
||||
global_state.phase = PHASE_VERIFY;
|
||||
take_stats_snapshot();
|
||||
ztest_run_registered_test_suites(&global_state);
|
||||
}
|
||||
14
tests/ztest/register/src/test_null_predicate.c
Normal file
14
tests/ztest/register/src/test_null_predicate.c
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <ztest.h>
|
||||
#include "common.h"
|
||||
|
||||
static void test_stub(void)
|
||||
{
|
||||
}
|
||||
|
||||
ztest_register_test_suite(run_null_predicate_once, NULL, ztest_unit_test(test_stub));
|
||||
21
tests/ztest/register/src/test_step_0.c
Normal file
21
tests/ztest/register/src/test_step_0.c
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <ztest.h>
|
||||
#include "common.h"
|
||||
|
||||
static void test(void)
|
||||
{
|
||||
}
|
||||
|
||||
static bool predicate(const void *state)
|
||||
{
|
||||
const struct global_test_state *s = state;
|
||||
|
||||
return s->phase == PHASE_STEPS_0;
|
||||
}
|
||||
|
||||
ztest_register_test_suite(test_step_0, predicate, ztest_unit_test(test));
|
||||
21
tests/ztest/register/src/test_step_1.c
Normal file
21
tests/ztest/register/src/test_step_1.c
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <ztest.h>
|
||||
#include "common.h"
|
||||
|
||||
static void test(void)
|
||||
{
|
||||
}
|
||||
|
||||
static bool predicate(const void *state)
|
||||
{
|
||||
const struct global_test_state *s = state;
|
||||
|
||||
return s->phase == PHASE_STEPS_1;
|
||||
}
|
||||
|
||||
ztest_register_test_suite(test_step_1, predicate, ztest_unit_test(test));
|
||||
21
tests/ztest/register/src/test_step_all.c
Normal file
21
tests/ztest/register/src/test_step_all.c
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright 2021 Google LLC
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <ztest.h>
|
||||
#include "common.h"
|
||||
|
||||
static void test(void)
|
||||
{
|
||||
}
|
||||
|
||||
static bool predicate(const void *state)
|
||||
{
|
||||
const struct global_test_state *s = state;
|
||||
|
||||
return s->phase == PHASE_STEPS_0 || s->phase == PHASE_STEPS_1;
|
||||
}
|
||||
|
||||
ztest_register_test_suite(test_step_all, predicate, ztest_unit_test(test));
|
||||
5
tests/ztest/register/testcase.yaml
Normal file
5
tests/ztest/register/testcase.yaml
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
tests:
|
||||
testing.ztest.register:
|
||||
tags: test_framework
|
||||
integration_platforms:
|
||||
- native_posix
|
||||
Loading…
Reference in a new issue