patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

scripts: tests: Blackbox test expension - disable

Browse source 
Add test related to disable-suite-name-check flag

This flag disabling extended test suite name verification at
the beginning of Ztest test. This option could be useful for tests
or platforms, which from some reasons cannot print early logs.

Add test related disable-warnings-as-errors

Do not treat warning conditions as errors.

Signed-off-by: Artur Wilczak <arturx.wilczak@intel.com>
This commit is contained in:
Artur Wilczak 2024-02-23 14:14:12 +01:00 committed by Fabio Baltieri
parent d076962f73
commit 6d8132c445
5 changed files with 179 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/CMakeLists.txt Normal file
View file

@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
cmake_minimum_required(VERSION 3.20.0)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(integration)
FILE(GLOB app_sources src/*.c)
target_sources(app PRIVATE ${app_sources})

7
scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/prj.conf Normal file
View file

@ -0,0 +1,7 @@
CONFIG_ZTEST=y
CONFIG_TEST_LOGGING_DEFAULTS=y
CONFIG_LOG=y
CONFIG_LOG_OUTPUT=y
CONFIG_LOG_BACKEND_UART=y
CONFIG_LOG_MODE_IMMEDIATE=y
CONFIG_MAIN_STACK_SIZE=4096

26
scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/src/main.c Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright (c) 2024 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/ztest.h>
#include <zephyr/logging/log.h>
#define LOG_MODULE_NAME log_test
LOG_MODULE_REGISTER(LOG_MODULE_NAME, LOG_LEVEL_INF);
ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
/**
* @brief Test Asserts
*
* This test verifies various assert macros provided by ztest.
*
*/
ZTEST(a1_1_tests, test_assert)
{
TC_PRINT("Create log message before rise warning\n");
LOG_WRN("log warning to custom warning");
#warning ("Custom warning");
}

10
scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/test_data.yaml Normal file
View file

@ -0,0 +1,10 @@
common:
timeout: 10
tests:
always_warning.dummy:
platform_allow:
- native_posix
- qemu_x86
- qemu_x86_64
integration_platforms:
- native_posix

128
scripts/tests/twister_blackbox/test_disable.py Normal file
View file

@ -0,0 +1,128 @@
#!/usr/bin/env python3
# Copyright (c) 2024 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
"""
Blackbox tests for twister's command line functions related to disable features.
"""
import importlib
import pytest
import mock
import os
import sys
import re
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
from twisterlib.testplan import TestPlan
@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
class TestDisable:
TESTDATA_1 = [
(
os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
['qemu_x86'],
'--disable-suite-name-check',
[r"Expected suite names:\[['\w+'\[,\s]*\]", r"Detected suite names:\[['\w+'\[,\s]*\]"],
True
),
(
os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
['qemu_x86'],
'-v',
[r"Expected suite names:\[['(\w+)'[, ]*]+", r"Detected suite names:\[['(\w+)'[, ]*]+"],
False
),
]
TESTDATA_2 = [
(
os.path.join(TEST_DATA, 'tests', 'always_warning'),
['qemu_x86'],
'--disable-warnings-as-errors',
'0'
),
(
os.path.join(TEST_DATA, 'tests', 'always_warning'),
['qemu_x86'],
'-v',
'1'
),
]
@classmethod
def setup_class(cls):
apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
cls.twister_module = importlib.util.module_from_spec(cls.spec)
@classmethod
def teardown_class(cls):
pass
@pytest.mark.parametrize(
'test_path, test_platforms, flag, expected, expected_none',
TESTDATA_1,
ids=[
'disable-suite-name-check',
'suite-name-check'
],
)
def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platforms, flag, expected, expected_none):
args = ['-i', '--outdir', out_path, '-T', test_path] + \
[flag] + \
['-vv'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == '0'
if expected_none:
assert re.search(expected[0], err) is None, f"Not expected string in log: {expected[0]}"
assert re.search(expected[1], err) is None, f"Not expected: {expected[1]}"
else:
assert re.search(expected[0], err) is not None, f"Expected string in log: {expected[0]}"
assert re.search(expected[1], err) is not None, f"Expected string in log: {expected[1]}"
@pytest.mark.parametrize(
'test_path, test_platforms, flag, expected_exit_code',
TESTDATA_2,
ids=[
'disable-warnings-as-errors',
'warnings-as-errors'
],
)
def test_disable_warnings_as_errors(self, capfd, out_path, test_path, test_platforms, flag, expected_exit_code):
args = ['-i', '--outdir', out_path, '-T', test_path] + \
[flag] + \
['-vv'] + \
['--build-only'] + \
[val for pair in zip(
['-p'] * len(test_platforms), test_platforms
) for val in pair]
with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
pytest.raises(SystemExit) as sys_exit:
self.loader.exec_module(self.twister_module)
out, err = capfd.readouterr()
sys.stdout.write(out)
sys.stderr.write(err)
assert str(sys_exit.value) == expected_exit_code, \
f"Twister return not expected ({expected_exit_code}) exit code: ({sys_exit.value})"