diff --git a/scripts/tests/twister_blackbox/test_platform.py b/scripts/tests/twister_blackbox/test_platform.py index 8303aebb3b..9dd05aa7ff 100644 --- a/scripts/tests/twister_blackbox/test_platform.py +++ b/scripts/tests/twister_blackbox/test_platform.py @@ -18,7 +18,49 @@ from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock from twisterlib.testplan import TestPlan +@mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) class TestPlatform: + TESTDATA_1 = [ + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), + ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], + { + 'selected_test_scenarios': 3, + 'selected_test_instances': 9, + 'skipped_configurations': 3, + 'skipped_by_static_filter': 3, + 'skipped_at_runtime': 0, + 'passed_configurations': 6, + 'failed_configurations': 0, + 'errored_configurations': 0, + 'executed_test_cases': 10, + 'skipped_test_cases': 5, + 'platform_count': 3, + 'executed_on_platform': 4, + 'only_built': 2 + } + ), + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), + ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], + { + 'selected_test_scenarios': 1, + 'selected_test_instances': 3, + 'skipped_configurations': 3, + 'skipped_by_static_filter': 3, + 'skipped_at_runtime': 0, + 'passed_configurations': 0, + 'failed_configurations': 0, + 'errored_configurations': 0, + 'executed_test_cases': 0, + 'skipped_test_cases': 3, + 'platform_count': 3, + 'executed_on_platform': 0, + 'only_built': 0 + } + ), + ] + @classmethod def setup_class(cls): apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister') @@ -35,7 +77,6 @@ class TestPlatform: [(True, '0'), (False, '2')], ids=['dummy in additional board root', 'no additional board root, crash'] ) - @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_board_root(self, out_path, board_root, expected_returncode): test_platforms = ['qemu_x86', 'dummy_board/dummy_soc'] board_root_path = os.path.join(TEST_DATA, 'boards') @@ -60,7 +101,6 @@ class TestPlatform: assert str(sys_exit.value) == expected_returncode - @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_force_platform(self, out_path): test_platforms = ['qemu_x86', 'frdm_k64f'] path = os.path.join(TEST_DATA, 'tests', 'dummy') @@ -86,7 +126,6 @@ class TestPlatform: assert len(filtered_j) == 12 - @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) def test_platform(self, out_path): path = os.path.join(TEST_DATA, 'tests', 'dummy') args = ['-i', '--outdir', out_path, '-T', path, '-y'] + \ @@ -107,3 +146,187 @@ class TestPlatform: assert str(sys_exit.value) == '0' assert all([platform == 'qemu_x86' for platform, _, _ in filtered_j]) + + @pytest.mark.parametrize( + 'test_path, test_platforms', + [ + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), + ['qemu_x86'], + ), + ], + ids=[ + 'any_platform', + ], + ) + @pytest.mark.parametrize( + 'flag', + ['-l', '--all'] + ) + def test_any_platform(self, capfd, out_path, test_path, test_platforms, flag): + args = ['--outdir', out_path, '-T', test_path, '-y'] + \ + [flag] + \ + [val for pair in zip( + ['-p'] * len(test_platforms), test_platforms + ) for val in pair] + + with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ + pytest.raises(SystemExit) as sys_exit: + self.loader.exec_module(self.twister_module) + + out, err = capfd.readouterr() + sys.stdout.write(out) + sys.stderr.write(err) + + assert str(sys_exit.value) == '0' + + @pytest.mark.parametrize( + 'test_path, test_platforms, expected', + [ + ( + os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), + ['qemu_x86', 'qemu_x86_64'], + { + 'passed_configurations': 3, + 'selected_test_instances': 6, + 'executed_on_platform': 2, + 'only_built': 1, + } + ), + ], + ids=[ + 'exclude_platform', + ], + ) + def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expected): + args = ['--outdir', out_path, '-T', test_path] + \ + ['--exclude-platform', "qemu_x86"] + \ + [val for pair in zip( + ['-p'] * len(test_platforms), test_platforms + ) for val in pair] + + with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ + pytest.raises(SystemExit) as sys_exit: + self.loader.exec_module(self.twister_module) + + out, err = capfd.readouterr() + sys.stdout.write(out) + sys.stderr.write(err) + + pass_regex = r'^INFO - (?P[0-9]+) of' \ + r' (?P[0-9]+) test configurations passed' + + built_regex = r'^INFO - (?P[0-9]+)' \ + r' test configurations executed on platforms, (?P[0-9]+)' \ + r' test configurations were only built.$' + + pass_search = re.search(pass_regex, err, re.MULTILINE) + + assert pass_search + assert int(pass_search.group('passed_configurations')) == \ + expected['passed_configurations'] + assert int(pass_search.group('test_instances')) == \ + expected['selected_test_instances'] + + built_search = re.search(built_regex, err, re.MULTILINE) + + assert built_search + assert int(built_search.group('executed_on_platform')) == \ + expected['executed_on_platform'] + assert int(built_search.group('only_built')) == \ + expected['only_built'] + + assert str(sys_exit.value) == '0' + + @pytest.mark.parametrize( + 'test_path, test_platforms, expected', + TESTDATA_1, + ids=[ + 'emulation_only tests/dummy/agnostic', + 'emulation_only tests/dummy/device', + ] + ) + def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected): + args = ['-i', '--outdir', out_path, '-T', test_path] + \ + ['--emulation-only'] + \ + [val for pair in zip( + ['-p'] * len(test_platforms), test_platforms + ) for val in pair] + + with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ + pytest.raises(SystemExit) as sys_exit: + self.loader.exec_module(self.twister_module) + + select_regex = r'^INFO - (?P[0-9]+) test scenarios' \ + r' \((?P[0-9]+) test instances\) selected,' \ + r' (?P[0-9]+) configurations skipped' \ + r' \((?P[0-9]+) by static filter,' \ + r' (?P[0-9]+) at runtime\)\.$' + + pass_regex = r'^INFO - (?P[0-9]+) of' \ + r' (?P[0-9]+) test configurations passed' \ + r' \([0-9]+\.[0-9]+%\), (?P[0-9]+) failed,' \ + r' (?P[0-9]+) errored,' \ + r' (?P[0-9]+) skipped with' \ + r' [0-9]+ warnings in [0-9]+\.[0-9]+ seconds$' + + case_regex = r'^INFO - In total (?P[0-9]+)' \ + r' test cases were executed, (?P[0-9]+) skipped' \ + r' on (?P[0-9]+) out of total [0-9]+ platforms' \ + r' \([0-9]+\.[0-9]+%\)$' + + built_regex = r'^INFO - (?P[0-9]+)' \ + r' test configurations executed on platforms, (?P[0-9]+)' \ + r' test configurations were only built.$' + + out, err = capfd.readouterr() + sys.stdout.write(out) + sys.stderr.write(err) + + select_search = re.search(select_regex, err, re.MULTILINE) + + assert select_search + assert int(select_search.group('test_scenarios')) == \ + expected['selected_test_scenarios'] + assert int(select_search.group('test_instances')) == \ + expected['selected_test_instances'] + assert int(select_search.group('skipped_configurations')) == \ + expected['skipped_configurations'] + assert int(select_search.group('skipped_by_static_filter')) == \ + expected['skipped_by_static_filter'] + assert int(select_search.group('skipped_at_runtime')) == \ + expected['skipped_at_runtime'] + + pass_search = re.search(pass_regex, err, re.MULTILINE) + + assert pass_search + assert int(pass_search.group('passed_configurations')) == \ + expected['passed_configurations'] + assert int(pass_search.group('test_instances')) == \ + expected['selected_test_instances'] + assert int(pass_search.group('failed_configurations')) == \ + expected['failed_configurations'] + assert int(pass_search.group('errored_configurations')) == \ + expected['errored_configurations'] + assert int(pass_search.group('skipped_configurations')) == \ + expected['skipped_configurations'] + + case_search = re.search(case_regex, err, re.MULTILINE) + + assert case_search + assert int(case_search.group('executed_test_cases')) == \ + expected['executed_test_cases'] + assert int(case_search.group('skipped_test_cases')) == \ + expected['skipped_test_cases'] + assert int(case_search.group('platform_count')) == \ + expected['platform_count'] + + built_search = re.search(built_regex, err, re.MULTILINE) + + assert built_search + assert int(built_search.group('executed_on_platform')) == \ + expected['executed_on_platform'] + assert int(built_search.group('only_built')) == \ + expected['only_built'] + + assert str(sys_exit.value) == '0' diff --git a/scripts/tests/twister_blackbox/test_runner.py b/scripts/tests/twister_blackbox/test_runner.py index 80025ae7b9..97e36d8b8a 100644 --- a/scripts/tests/twister_blackbox/test_runner.py +++ b/scripts/tests/twister_blackbox/test_runner.py @@ -5,6 +5,7 @@ """ Blackbox tests for twister's command line functions """ +# pylint: disable=duplicate-code import importlib import mock @@ -21,46 +22,6 @@ from twisterlib.testplan import TestPlan @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', testsuite_filename_mock) class TestRunner: TESTDATA_1 = [ - ( - os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), - ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], - { - 'selected_test_scenarios': 3, - 'selected_test_instances': 9, - 'skipped_configurations': 3, - 'skipped_by_static_filter': 3, - 'skipped_at_runtime': 0, - 'passed_configurations': 6, - 'failed_configurations': 0, - 'errored_configurations': 0, - 'executed_test_cases': 10, - 'skipped_test_cases': 5, - 'platform_count': 3, - 'executed_on_platform': 4, - 'only_built': 2 - } - ), - ( - os.path.join(TEST_DATA, 'tests', 'dummy', 'device'), - ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], - { - 'selected_test_scenarios': 1, - 'selected_test_instances': 3, - 'skipped_configurations': 3, - 'skipped_by_static_filter': 3, - 'skipped_at_runtime': 0, - 'passed_configurations': 0, - 'failed_configurations': 0, - 'errored_configurations': 0, - 'executed_test_cases': 0, - 'skipped_test_cases': 3, - 'platform_count': 3, - 'executed_on_platform': 0, - 'only_built': 0 - } - ), - ] - TESTDATA_2 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], @@ -78,7 +39,7 @@ class TestRunner: } ), ] - TESTDATA_3 = [ + TESTDATA_2 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86', 'qemu_x86_64', 'frdm_k64f'], @@ -99,13 +60,13 @@ class TestRunner: } ) ] - TESTDATA_4 = [ + TESTDATA_3 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86'], ), ] - TESTDATA_5 = [ + TESTDATA_4 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86', 'qemu_x86_64'], @@ -117,26 +78,14 @@ class TestRunner: } ), ] - TESTDATA_6 = [ + TESTDATA_5 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), ['qemu_x86'], os.path.join(TEST_DATA, "pre_script.sh") ), ] - TESTDATA_7 = [ - ( - os.path.join(TEST_DATA, 'tests', 'dummy', 'agnostic'), - ['qemu_x86', 'qemu_x86_64'], - { - 'passed_configurations': 3, - 'selected_test_instances': 6, - 'executed_on_platform': 2, - 'only_built': 1, - } - ), - ] - TESTDATA_8 = [ + TESTDATA_6 = [ ( os.path.join(TEST_DATA, 'tests', 'always_fail', 'dummy'), ['qemu_x86_64'], @@ -148,7 +97,7 @@ class TestRunner: '2', ), ] - TESTDATA_9 = [ + TESTDATA_7 = [ ( os.path.join(TEST_DATA, 'tests', 'always_fail', 'dummy'), ['qemu_x86'], @@ -160,7 +109,7 @@ class TestRunner: '30', ), ] - TESTDATA_10 = [ + TESTDATA_8 = [ ( os.path.join(TEST_DATA, 'tests', 'always_timeout', 'dummy'), ['qemu_x86'], @@ -172,7 +121,7 @@ class TestRunner: '0.5', ), ] - TESTDATA_11 = [ + TESTDATA_9 = [ ( os.path.join(TEST_DATA, 'tests', 'dummy'), ['qemu_x86'], @@ -198,7 +147,7 @@ class TestRunner: [r'3 of 4 test configurations passed \(100.00%\), 0 failed, 0 errored, 1 skipped'] ), ] - TESTDATA_12 = [ + TESTDATA_10 = [ ( os.path.join(TEST_DATA, 'tests', 'one_fail_one_pass'), ['qemu_x86'], @@ -211,7 +160,7 @@ class TestRunner: } ) ] - TESTDATA_13 = [ + TESTDATA_11 = [ ( os.path.join(TEST_DATA, 'tests', 'always_build_error'), ['qemu_x86_64'], @@ -238,98 +187,6 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, expected', TESTDATA_1, - ids=[ - 'emulation_only tests/dummy/agnostic', - 'emulation_only tests/dummy/device', - ] - ) - def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expected): - args = ['-i', '--outdir', out_path, '-T', test_path, '--emulation-only'] + \ - [val for pair in zip( - ['-p'] * len(test_platforms), test_platforms - ) for val in pair] - - with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ - pytest.raises(SystemExit) as sys_exit: - self.loader.exec_module(self.twister_module) - - select_regex = r'^INFO - (?P[0-9]+) test scenarios' \ - r' \((?P[0-9]+) test instances\) selected,' \ - r' (?P[0-9]+) configurations skipped' \ - r' \((?P[0-9]+) by static filter,' \ - r' (?P[0-9]+) at runtime\)\.$' - - pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' \ - r' \([0-9]+\.[0-9]+%\), (?P[0-9]+) failed,' \ - r' (?P[0-9]+) errored,' \ - r' (?P[0-9]+) skipped with' \ - r' [0-9]+ warnings in [0-9]+\.[0-9]+ seconds$' - - case_regex = r'^INFO - In total (?P[0-9]+)' \ - r' test cases were executed, (?P[0-9]+) skipped' \ - r' on (?P[0-9]+) out of total [0-9]+ platforms' \ - r' \([0-9]+\.[0-9]+%\)$' - - built_regex = r'^INFO - (?P[0-9]+)' \ - r' test configurations executed on platforms, (?P[0-9]+)' \ - r' test configurations were only built.$' - - out, err = capfd.readouterr() - sys.stdout.write(out) - sys.stderr.write(err) - - select_search = re.search(select_regex, err, re.MULTILINE) - - assert select_search - assert int(select_search.group('test_scenarios')) == \ - expected['selected_test_scenarios'] - assert int(select_search.group('test_instances')) == \ - expected['selected_test_instances'] - assert int(select_search.group('skipped_configurations')) == \ - expected['skipped_configurations'] - assert int(select_search.group('skipped_by_static_filter')) == \ - expected['skipped_by_static_filter'] - assert int(select_search.group('skipped_at_runtime')) == \ - expected['skipped_at_runtime'] - - pass_search = re.search(pass_regex, err, re.MULTILINE) - - assert pass_search - assert int(pass_search.group('passed_configurations')) == \ - expected['passed_configurations'] - assert int(pass_search.group('test_instances')) == \ - expected['selected_test_instances'] - assert int(pass_search.group('failed_configurations')) == \ - expected['failed_configurations'] - assert int(pass_search.group('errored_configurations')) == \ - expected['errored_configurations'] - assert int(pass_search.group('skipped_configurations')) == \ - expected['skipped_configurations'] - - case_search = re.search(case_regex, err, re.MULTILINE) - - assert case_search - assert int(case_search.group('executed_test_cases')) == \ - expected['executed_test_cases'] - assert int(case_search.group('skipped_test_cases')) == \ - expected['skipped_test_cases'] - assert int(case_search.group('platform_count')) == \ - expected['platform_count'] - - built_search = re.search(built_regex, err, re.MULTILINE) - - assert built_search - assert int(built_search.group('executed_on_platform')) == \ - expected['executed_on_platform'] - assert int(built_search.group('only_built')) == \ - expected['only_built'] - - assert str(sys_exit.value) == '0' - - @pytest.mark.parametrize( - 'test_path, test_platforms, expected', - TESTDATA_2, ids=[ 'build_only tests/dummy/agnostic', 'build_only tests/dummy/device', @@ -369,7 +226,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, expected', - TESTDATA_3, + TESTDATA_2, ids=[ 'test_only' ], @@ -424,11 +281,6 @@ class TestRunner: sys.stdout.write(out) sys.stderr.write(err) - path = os.path.join(out_path, 'twister.log') - with open(path, "r") as log_file: - text_content = log_file.read() - print(text_content) - select_search = re.search(select_regex, err, re.MULTILINE) assert select_search @@ -479,7 +331,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms', - TESTDATA_4, + TESTDATA_3, ids=[ 'dry_run', ], @@ -504,36 +356,9 @@ class TestRunner: assert str(sys_exit.value) == '0' - @pytest.mark.parametrize( - 'test_path, test_platforms', - TESTDATA_4, - ids=[ - 'any_platform', - ], - ) - @pytest.mark.parametrize( - 'flag', - ['-l', '--all'] - ) - def test_any_platform(self, capfd, out_path, test_path, test_platforms, flag): - args = ['--outdir', out_path, '-T', test_path, flag] + \ - [val for pair in zip( - ['-p'] * len(test_platforms), test_platforms - ) for val in pair] - - with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ - pytest.raises(SystemExit) as sys_exit: - self.loader.exec_module(self.twister_module) - - out, err = capfd.readouterr() - sys.stdout.write(out) - sys.stderr.write(err) - - assert str(sys_exit.value) == '0' - @pytest.mark.parametrize( 'test_path, test_platforms, expected', - TESTDATA_5, + TESTDATA_4, ids=[ 'cmake_only', ], @@ -579,7 +404,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, file_name', - TESTDATA_6, + TESTDATA_5, ids=[ 'pre_script', ], @@ -600,55 +425,9 @@ class TestRunner: assert str(sys_exit.value) == '0' - @pytest.mark.parametrize( - 'test_path, test_platforms, expected', - TESTDATA_7, - ids=[ - 'exclude_platform', - ], - ) - def test_exclude_platform(self, capfd, out_path, test_path, test_platforms, expected): - args = ['--outdir', out_path, '-T', test_path, '--exclude-platform', "qemu_x86"] + \ - [val for pair in zip( - ['-p'] * len(test_platforms), test_platforms - ) for val in pair] - - with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \ - pytest.raises(SystemExit) as sys_exit: - self.loader.exec_module(self.twister_module) - - out, err = capfd.readouterr() - sys.stdout.write(out) - sys.stderr.write(err) - - pass_regex = r'^INFO - (?P[0-9]+) of' \ - r' (?P[0-9]+) test configurations passed' - - built_regex = r'^INFO - (?P[0-9]+)' \ - r' test configurations executed on platforms, (?P[0-9]+)' \ - r' test configurations were only built.$' - - pass_search = re.search(pass_regex, err, re.MULTILINE) - - assert pass_search - assert int(pass_search.group('passed_configurations')) == \ - expected['passed_configurations'] - assert int(pass_search.group('test_instances')) == \ - expected['selected_test_instances'] - - built_search = re.search(built_regex, err, re.MULTILINE) - - assert built_search - assert int(built_search.group('executed_on_platform')) == \ - expected['executed_on_platform'] - assert int(built_search.group('only_built')) == \ - expected['only_built'] - - assert str(sys_exit.value) == '0' - @pytest.mark.parametrize( 'test_path, test_platforms', - TESTDATA_4, + TESTDATA_3, ids=[ 'device_flash_timeout', ], @@ -671,7 +450,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, iterations', - TESTDATA_8, + TESTDATA_6, ids=[ 'retry 2', 'retry 3' @@ -707,7 +486,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, interval', - TESTDATA_9, + TESTDATA_7, ids=[ 'retry interval 15', 'retry interval 30' @@ -731,7 +510,6 @@ class TestRunner: end_time = time.time() elapsed_time = end_time - start_time - print(f"Time elapsed: {elapsed_time:.2f} seconds") if elapsed_time < int(interval): assert 'interval was too short' @@ -739,7 +517,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, timeout', - TESTDATA_10, + TESTDATA_8, ids=[ 'timeout-multiplier 2 - 20s', 'timeout-multiplier 0.5 - 5s' @@ -770,7 +548,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, tags, expected', - TESTDATA_11, + TESTDATA_9, ids=[ 'tags device', 'tags subgruped', @@ -795,14 +573,13 @@ class TestRunner: sys.stderr.write(err) for line in expected: - print(line) assert re.search(line, err) assert str(sys_exit.value) == '0' @pytest.mark.parametrize( 'test_path, test_platforms, expected', - TESTDATA_12, + TESTDATA_10, ids=[ 'only_failed' ], @@ -865,7 +642,7 @@ class TestRunner: @pytest.mark.parametrize( 'test_path, test_platforms, iterations', - TESTDATA_13, + TESTDATA_11, ids=[ 'retry 2', 'retry 3'