scripts: tests: Blackbox test expension - disable

Add test related to disable-suite-name-check flag

This flag disabling extended test suite name verification at
the beginning of Ztest test. This option could be useful for tests
or platforms, which from some reasons cannot print early logs.

Add test related disable-warnings-as-errors

Do not treat warning conditions as errors.

Signed-off-by: Artur Wilczak <arturx.wilczak@intel.com>

scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(integration)
+
+FILE(GLOB app_sources src/*.c)
+target_sources(app PRIVATE ${app_sources})

scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/prj.conf

@@ -0,0 +1,7 @@
+CONFIG_ZTEST=y
+CONFIG_TEST_LOGGING_DEFAULTS=y
+CONFIG_LOG=y
+CONFIG_LOG_OUTPUT=y
+CONFIG_LOG_BACKEND_UART=y
+CONFIG_LOG_MODE_IMMEDIATE=y
+CONFIG_MAIN_STACK_SIZE=4096

scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/src/main.c

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+#include <zephyr/logging/log.h>
+
+#define LOG_MODULE_NAME log_test
+LOG_MODULE_REGISTER(LOG_MODULE_NAME, LOG_LEVEL_INF);
+
+ZTEST_SUITE(a1_1_tests, NULL, NULL, NULL, NULL, NULL);
+
+/**
+ * @brief Test Asserts
+ *
+ * This test verifies various assert macros provided by ztest.
+ *
+ */
+ZTEST(a1_1_tests, test_assert)
+{
+		TC_PRINT("Create log message before rise warning\n");
+		LOG_WRN("log warning to custom warning");
+		#warning ("Custom warning");
+}

scripts/tests/twister_blackbox/test_data/tests/always_warning/dummy/test_data.yaml

@@ -0,0 +1,10 @@
+common:
+  timeout: 10
+tests:
+  always_warning.dummy:
+    platform_allow:
+      - native_posix
+      - qemu_x86
+      - qemu_x86_64
+    integration_platforms:
+      - native_posix

scripts/tests/twister_blackbox/test_disable.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+"""
+Blackbox tests for twister's command line functions related to disable features.
+"""
+
+import importlib
+import pytest
+import mock
+import os
+import sys
+import re
+
+from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock
+from twisterlib.testplan import TestPlan
+
+
+@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock)
+class TestDisable:
+    TESTDATA_1 = [
+        (
+            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
+            ['qemu_x86'],
+            '--disable-suite-name-check',
+            [r"Expected suite names:\[['\w+'\[,\s]*\]", r"Detected suite names:\[['\w+'\[,\s]*\]"],
+            True
+        ),
+        (
+            os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'),
+            ['qemu_x86'],
+            '-v',
+            [r"Expected suite names:\[['(\w+)'[, ]*]+", r"Detected suite names:\[['(\w+)'[, ]*]+"],
+            False
+        ),
+    ]
+    TESTDATA_2 = [
+        (
+            os.path.join(TEST_DATA, 'tests', 'always_warning'),
+            ['qemu_x86'],
+            '--disable-warnings-as-errors',
+            '0'
+        ),
+        (
+            os.path.join(TEST_DATA, 'tests', 'always_warning'),
+            ['qemu_x86'],
+            '-v',
+            '1'
+        ),
+    ]
+
+
+    @classmethod
+    def setup_class(cls):
+        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
+        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
+        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
+        cls.twister_module = importlib.util.module_from_spec(cls.spec)
+
+
+    @classmethod
+    def teardown_class(cls):
+        pass
+
+
+    @pytest.mark.parametrize(
+        'test_path, test_platforms, flag, expected, expected_none',
+        TESTDATA_1,
+        ids=[
+            'disable-suite-name-check',
+            'suite-name-check'
+        ],
+    )
+
+    def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platforms, flag, expected, expected_none):
+        args = ['-i', '--outdir', out_path, '-T', test_path] + \
+               [flag] + \
+               ['-vv'] + \
+               [val for pair in zip(
+                   ['-p'] * len(test_platforms), test_platforms
+               ) for val in pair]
+
+        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
+            pytest.raises(SystemExit) as sys_exit:
+            self.loader.exec_module(self.twister_module)
+
+        out, err = capfd.readouterr()
+        sys.stdout.write(out)
+        sys.stderr.write(err)
+
+        assert str(sys_exit.value) == '0'
+        if expected_none:
+            assert re.search(expected[0], err) is None, f"Not expected string in log: {expected[0]}"
+            assert re.search(expected[1], err) is None, f"Not expected: {expected[1]}"
+        else:
+            assert re.search(expected[0], err) is not None, f"Expected string in log: {expected[0]}"
+            assert re.search(expected[1], err) is not None, f"Expected string in log: {expected[1]}"
+
+
+    @pytest.mark.parametrize(
+        'test_path, test_platforms, flag, expected_exit_code',
+        TESTDATA_2,
+        ids=[
+            'disable-warnings-as-errors',
+            'warnings-as-errors'
+        ],
+    )
+
+    def test_disable_warnings_as_errors(self, capfd, out_path, test_path, test_platforms, flag, expected_exit_code):
+        args = ['-i', '--outdir', out_path, '-T', test_path] + \
+               [flag] + \
+               ['-vv'] + \
+               ['--build-only'] + \
+               [val for pair in zip(
+                   ['-p'] * len(test_platforms), test_platforms
+               ) for val in pair]
+
+        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
+            pytest.raises(SystemExit) as sys_exit:
+            self.loader.exec_module(self.twister_module)
+
+        out, err = capfd.readouterr()
+        sys.stdout.write(out)
+        sys.stderr.write(err)
+
+        assert str(sys_exit.value) == expected_exit_code, \
+            f"Twister return not expected ({expected_exit_code}) exit code: ({sys_exit.value})"