ztest: shell: add shell support

- Support for listing both testcases and testsuites
- Support for running single suites or single test cases
- Support shuffling tests and repeating execution based on command line
  arguments.

For example, build with

west build -p  -b qemu_cortex_m3  tests/kernel/sleep   -t run -- \
-DCONFIG_ZTEST_SHUFFLE=y -DCONFIG_ZTEST_SHELL=y

Following commands are available:

uart:~$ ztest
ztest - Ztest commands
Subcommands:
  run-all          :Run all tests
  shuffle          :Shuffle tests
  list-testsuites  :List all test suites
  list-testcases   :List all test cases
  run-testsuite    :Run test suite
  run-testcase     :Run testcase

shuffle accepts two arguments --suite_iter and --case_iter which allows
repeated exercution of testcases or suites.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

doc/develop/test/ztest.rst

@@ -197,15 +197,15 @@ function can be written as follows:
 
         /* Only suites that use a predicate checking for phase == PWR_PHASE_0 will run. */
         state.phase = PWR_PHASE_0;
-        ztest_run_all(&state);
+        ztest_run_all(&state, false, 1, 1);
 
         /* Only suites that use a predicate checking for phase == PWR_PHASE_1 will run. */
         state.phase = PWR_PHASE_1;
-        ztest_run_all(&state);
+        ztest_run_all(&state, false, 1, 1);
 
         /* Only suites that use a predicate checking for phase == PWR_PHASE_2 will run. */
         state.phase = PWR_PHASE_2;
-        ztest_run_all(&state);
+        ztest_run_all(&state, false, 1, 1);
 
         /* Check that all the suites in this binary ran at least once. */
         ztest_verify_all_test_suites_ran();

subsys/testsuite/ztest/CMakeLists.txt

@@ -26,6 +26,8 @@ zephyr_library_sources_ifdef(CONFIG_ZTRESS         src/ztress.c)
 
 if(CONFIG_ARCH_POSIX)
   zephyr_library_sources(src/ztest_posix.c)
+elseif(CONFIG_ZTEST_SHELL)
+  zephyr_library_sources(src/ztest_shell.c)
 else()
   zephyr_library_sources(src/ztest_defaults.c)
 endif()

subsys/testsuite/ztest/Kconfig

@@ -23,6 +23,15 @@ config ZTEST_TEST_DELAY_MS
 	  Add a delay between between tests to manage output on the console on
 	  systems that can't handle the rapid output rate.
 
+config ZTEST_SHELL
+	bool "Ztest with shell support"
+	select SHELL
+	select SHELL_THREAD_PRIORITY_OVERRIDE
+	select GETOPT_LONG
+	select SHELL_GETOPT
+	help
+	  Enable shell to manage test execution and selection.
+
 config ZTEST_CPU_HOLD_TIME_MS
 	int "Time in milliseconds to hold other CPUs for 1cpu type tests"
 	default 3000

subsys/testsuite/ztest/include/zephyr/ztest_test.h

@@ -232,8 +232,11 @@ extern struct ztest_suite_node _ztest_suite_node_list_end[];
  * Default entry point for running or listing registered unit tests.
  *
  * @param state The current state of the machine as it relates to the test executable.
+ * @param shuffle Shuffle tests
+ * @param suite_iter Test suite repetitions.
+ * @param case_iter Test case repetitions.
  */
-void ztest_run_all(const void *state);
+void ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter);
 
 /**
  * The result of the current running test. It's possible that the setup function sets the result
@@ -265,18 +268,25 @@ enum ztest_phase {
  * Run the registered unit tests which return true from their predicate function.
  *
  * @param state The current state of the machine as it relates to the test executable.
+ * @param shuffle Shuffle tests
+ * @param suite_iter Test suite repetitions.
+ * @param case_iter Test case repetitions.
  * @return The number of tests that ran.
  */
 
 #ifdef ZTEST_UNITTEST
-int z_impl_ztest_run_test_suites(const void *state);
-static inline int ztest_run_test_suites(const void *state)
+int z_impl_ztest_run_test_suites(const void *state, bool shuffle,
+				int suite_iter, int case_iter);
+
+static inline int ztest_run_test_suites(const void *state, bool shuffle,
+				int suite_iter, int case_iter)
 {
-	return z_impl_ztest_run_test_suites(state);
+	return z_impl_ztest_run_test_suites(state, shuffle, suite_iter, case_iter);
 }
 
 #else
-__syscall int ztest_run_test_suites(const void *state);
+__syscall int ztest_run_test_suites(const void *state, bool shuffle,
+				int suite_iter, int case_iter);
 #endif
 
 #ifdef ZTEST_UNITTEST
@@ -315,9 +325,12 @@ void ztest_verify_all_test_suites_ran(void);
  * checks for fast failures and initialization.
  *
  * @param name The name of the suite to run.
+ * @param shuffle Shuffle tests
+ * @param suite_iter Test suite repetitions.
+ * @param case_iter Test case repetitions.
  * @return Negative value if the test suite never ran; otherwise, return the number of failures.
  */
-int z_ztest_run_test_suite(const char *name);
+int z_ztest_run_test_suite(const char *name, bool shuffle, int suite_iter, int case_iter);
 
 /**
  * @brief Returns next test within suite.
@@ -534,14 +547,15 @@ void ztest_simple_1cpu_after(void *data);
  *
  * @param suite Test suite to run.
  */
-#define ztest_run_test_suite(suite) z_ztest_run_test_suite(STRINGIFY(suite))
+#define ztest_run_test_suite(suite) z_ztest_run_test_suite(STRINGIFY(suite), \
+	int suite_iter, int case_iter)
 
 /**
  * @brief Structure for architecture specific APIs
  *
  */
 struct ztest_arch_api {
-	void (*run_all)(const void *state);
+	void (*run_all)(const void *state, bool shuffle, int suite_iter, int case_iter);
 	bool (*should_suite_run)(const void *state, struct ztest_suite_node *suite);
 	bool (*should_test_run)(const char *suite, const char *test);
 };

subsys/testsuite/ztest/src/ztest.c

@@ -18,11 +18,15 @@ static struct k_thread ztest_thread;
 #endif
 static bool failed_expectation;
 
+#ifdef CONFIG_ZTEST_SHELL
+#include <zephyr/shell/shell.h>
+#endif
+
 #ifdef CONFIG_ZTEST_SHUFFLE
 #include <stdlib.h>
 #include <time.h>
-
 #include <zephyr/random/random.h>
+
 #define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
 #define NUM_ITER_PER_TEST  CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT
 #else
@@ -651,30 +655,33 @@ struct ztest_unit_test *z_ztest_get_next_test(const char *suite, struct ztest_un
 	return NULL;
 }
 
-#ifdef CONFIG_ZTEST_SHUFFLE
-static void z_ztest_shuffle(void *dest[], intptr_t start, size_t num_items, size_t element_size)
+#if CONFIG_ZTEST_SHUFFLE
+static void z_ztest_shuffle(bool shuffle, void *dest[], intptr_t start,
+			    size_t num_items, size_t element_size)
 {
-	void *tmp;
-
 	/* Initialize dest array */
 	for (size_t i = 0; i < num_items; ++i) {
 		dest[i] = (void *)(start + (i * element_size));
 	}
+	void *tmp;
 
 	/* Shuffle dest array */
-	for (size_t i = num_items - 1; i > 0; i--) {
-		int j = sys_rand32_get() % (i + 1);
+	if (shuffle) {
+		for (size_t i = num_items - 1; i > 0; i--) {
+			int j = sys_rand32_get() % (i + 1);
 
-		if (i != j) {
-			tmp = dest[j];
-			dest[j] = dest[i];
-			dest[i] = tmp;
+			if (i != j) {
+				tmp = dest[j];
+				dest[j] = dest[i];
+				dest[i] = tmp;
+			}
 		}
 	}
 }
-#endif /* CONFIG_ZTEST_SHUFFLE */
+#endif
 
-static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
+static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite,
+				      bool shuffle, int suite_iter, int case_iter)
 {
 	struct ztest_unit_test *test = NULL;
 	void *data = NULL;
@@ -714,14 +721,13 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 		data = suite->setup();
 	}
 
-	for (int i = 0; i < NUM_ITER_PER_TEST; i++) {
-		fail = 0;
-
+	for (int i = 0; i < case_iter; i++) {
 #ifdef CONFIG_ZTEST_SHUFFLE
 		struct ztest_unit_test *tests_to_run[ZTEST_TEST_COUNT];
 
 		memset(tests_to_run, 0, ZTEST_TEST_COUNT * sizeof(struct ztest_unit_test *));
-		z_ztest_shuffle((void **)tests_to_run, (intptr_t)_ztest_unit_test_list_start,
+		z_ztest_shuffle(shuffle, (void **)tests_to_run,
+				(intptr_t)_ztest_unit_test_list_start,
 				ZTEST_TEST_COUNT, sizeof(struct ztest_unit_test));
 		for (size_t j = 0; j < ZTEST_TEST_COUNT; ++j) {
 			test = tests_to_run[j];
@@ -771,7 +777,6 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 			}
 		}
 #endif
-
 		if (test_status == ZTEST_STATUS_OK && fail != 0) {
 			test_status = ZTEST_STATUS_HAS_FAILURE;
 		}
@@ -786,9 +791,10 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite)
 	return fail;
 }
 
-int z_ztest_run_test_suite(const char *name)
+int z_ztest_run_test_suite(const char *name, bool shuffle, int suite_iter, int case_iter)
 {
-	return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name));
+	return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name),
+					shuffle, suite_iter, case_iter);
 }
 
 #ifdef CONFIG_USERSPACE
@@ -942,14 +948,15 @@ static void __ztest_show_suite_summary(void)
 	flush_log();
 }
 
-static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *state)
+static int __ztest_run_test_suite(struct ztest_suite_node *ptr,
+			const void *state, bool shuffle, int suite_iter, int case_iter)
 {
 	struct ztest_suite_stats *stats = ptr->stats;
 	int count = 0;
 
-	for (int i = 0; i < NUM_ITER_PER_SUITE; i++) {
+	for (int i = 0; i < suite_iter; i++) {
 		if (ztest_api.should_suite_run(state, ptr)) {
-			int fail = z_ztest_run_test_suite_ptr(ptr);
+			int fail = z_ztest_run_test_suite_ptr(ptr, shuffle, suite_iter, case_iter);
 
 			count++;
 			stats->run_count++;
@@ -962,7 +969,7 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat
 	return count;
 }
 
-int z_impl_ztest_run_test_suites(const void *state)
+int z_impl_ztest_run_test_suites(const void *state, bool shuffle, int suite_iter, int case_iter)
 {
 	int count = 0;
 
@@ -974,31 +981,32 @@ int z_impl_ztest_run_test_suites(const void *state)
 	struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT];
 
 	memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *));
-	z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start,
+	z_ztest_shuffle(shuffle, (void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start,
 			ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node));
 	for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
 		__ztest_init_unit_test_result_for_suite(suites_to_run[i]);
 	}
 	for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) {
-		count += __ztest_run_test_suite(suites_to_run[i], state);
+		count += __ztest_run_test_suite(suites_to_run[i],
+				state, shuffle, suite_iter, case_iter);
 		/* Stop running tests if we have a critical error or if we have a failure and
 		 * FAIL_FAST was set
 		 */
 		if (test_status == ZTEST_STATUS_CRITICAL_ERROR ||
-		    (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
+				(test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
 			break;
 		}
 	}
 #else
 	for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
-	     ptr < _ztest_suite_node_list_end; ++ptr) {
+			ptr < _ztest_suite_node_list_end; ++ptr) {
 		__ztest_init_unit_test_result_for_suite(ptr);
-		count += __ztest_run_test_suite(ptr, state);
+		count += __ztest_run_test_suite(ptr, state, shuffle, suite_iter, case_iter);
 		/* Stop running tests if we have a critical error or if we have a failure and
 		 * FAIL_FAST was set
 		 */
 		if (test_status == ZTEST_STATUS_CRITICAL_ERROR ||
-		    (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
+				(test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
 			break;
 		}
 	}
@@ -1070,12 +1078,18 @@ void ztest_verify_all_test_suites_ran(void)
 	}
 }
 
-void ztest_run_all(const void *state) { ztest_api.run_all(state); }
+void ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter)
+{
+	ztest_api.run_all(state, shuffle, suite_iter, case_iter);
+}
 
 void __weak test_main(void)
 {
-	ztest_run_all(NULL);
-
+#if CONFIG_ZTEST_SHUFFLE
+	ztest_run_all(NULL, true, NUM_ITER_PER_SUITE, NUM_ITER_PER_TEST);
+#else
+	ztest_run_all(NULL, false, NUM_ITER_PER_SUITE, NUM_ITER_PER_TEST);
+#endif
 	ztest_verify_all_test_suites_ran();
 }
 
@@ -1100,6 +1114,179 @@ int main(void)
 	return test_status;
 }
 #else
+
+/* Shell */
+
+#ifdef CONFIG_ZTEST_SHELL
+static int cmd_list_suites(const struct shell *sh, size_t argc, char **argv)
+{
+	struct ztest_suite_node *suite;
+
+	for (suite = _ztest_suite_node_list_start; suite < _ztest_suite_node_list_end; ++suite) {
+		shell_print(sh, "%s", suite->name);
+	}
+	return 0;
+}
+
+static int cmd_list_cases(const struct shell *sh, size_t argc, char **argv)
+{
+	struct ztest_suite_node *ptr;
+	struct ztest_unit_test *test = NULL;
+	int test_count = 0;
+
+	for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) {
+		test = NULL;
+		while ((test = z_ztest_get_next_test(ptr->name, test)) != NULL) {
+			shell_print(sh, "%s::%s", test->test_suite_name, test->name);
+			test_count++;
+		}
+	}
+	return 0;
+}
+extern void ztest_set_test_args(char *argv);
+extern void ztest_reset_test_args(void);
+
+static int cmd_runall(const struct shell *sh, size_t argc, char **argv)
+{
+	ztest_reset_test_args();
+	ztest_run_all(NULL, false, 1, 1);
+	end_report();
+	return 0;
+}
+
+#ifdef CONFIG_ZTEST_SHUFFLE
+static int cmd_shuffle(const struct shell *sh, size_t argc, char **argv)
+{
+
+	struct getopt_state *state;
+	int opt;
+	static struct option long_options[] = {{"suite_iter", required_argument, 0, 's'},
+		{"case_iter", required_argument, 0, 'c'},
+		{0, 0, 0, 0}};
+	int opt_index = 0;
+	int val;
+	int opt_num = 0;
+
+	int suite_iter = 1;
+	int case_iter = 1;
+
+	while ((opt = getopt_long(argc, argv, "s:c:", long_options, &opt_index)) != -1) {
+		state = getopt_state_get();
+		switch (opt) {
+		case 's':
+			val = atoi(state->optarg);
+			if (val < 1) {
+				shell_fprintf(sh, SHELL_ERROR,
+					"Invalid number of suite interations\n");
+				return -ENOEXEC;
+			}
+			suite_iter = val;
+			opt_num++;
+			break;
+		case 'c':
+			val = atoi(state->optarg);
+			if (val < 1) {
+				shell_fprintf(sh, SHELL_ERROR,
+					"Invalid number of case interations\n");
+				return -ENOEXEC;
+			}
+			case_iter = val;
+			opt_num++;
+			break;
+		default:
+			shell_fprintf(sh, SHELL_ERROR,
+				"Invalid option or option usage: %s\n", argv[opt_index + 1]);
+			return -ENOEXEC;
+		}
+	}
+	ztest_reset_test_args();
+	ztest_run_all(NULL, true, suite_iter, case_iter);
+	end_report();
+	return 0;
+}
+#endif
+
+static int cmd_run_suite(const struct shell *sh, size_t argc, char **argv)
+{
+	int count = 0;
+	bool shuffle = false;
+
+	ztest_set_test_args(argv[1]);
+
+	for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start;
+			ptr < _ztest_suite_node_list_end; ++ptr) {
+		__ztest_init_unit_test_result_for_suite(ptr);
+		count += __ztest_run_test_suite(ptr, NULL, shuffle, 1, 1);
+		if (test_status == ZTEST_STATUS_CRITICAL_ERROR ||
+				(test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) {
+			break;
+		}
+	}
+	return 0;
+}
+
+static void testsuite_list_get(size_t idx, struct shell_static_entry *entry);
+
+SHELL_DYNAMIC_CMD_CREATE(testsuite_names, testsuite_list_get);
+
+static size_t testsuite_get_all_static(struct ztest_suite_node const **suites)
+{
+	*suites = _ztest_suite_node_list_start;
+	return _ztest_suite_node_list_end - _ztest_suite_node_list_start;
+}
+
+static const struct ztest_suite_node *suite_lookup(size_t idx, const char *prefix)
+{
+	size_t match_idx = 0;
+	const struct ztest_suite_node *suite;
+	size_t len = testsuite_get_all_static(&suite);
+	const struct ztest_suite_node *suite_end = suite + len;
+
+	while (suite < suite_end) {
+		if ((suite->name != NULL) && (strlen(suite->name) != 0) &&
+				((prefix == NULL) ||
+				(strncmp(prefix, suite->name, strlen(prefix)) == 0))) {
+			if (match_idx == idx) {
+				return suite;
+			}
+			++match_idx;
+		}
+		++suite;
+	}
+
+	return NULL;
+}
+
+static void testsuite_list_get(size_t idx, struct shell_static_entry *entry)
+{
+	const struct ztest_suite_node *suite = suite_lookup(idx, "");
+
+	entry->syntax = (suite != NULL) ? suite->name : NULL;
+	entry->handler = NULL;
+	entry->help = NULL;
+	entry->subcmd = NULL;
+}
+
+	SHELL_STATIC_SUBCMD_SET_CREATE(
+		sub_ztest_cmds,
+		SHELL_CMD_ARG(run-all, NULL, "Run all tests", cmd_runall, 0, 0),
+#ifdef CONFIG_ZTEST_SHUFFLE
+		SHELL_COND_CMD_ARG(CONFIG_ZTEST_SHUFFLE, shuffle, NULL,
+			"Shuffle tests", cmd_shuffle, 0, 2),
+#endif
+		SHELL_CMD_ARG(list-testsuites, NULL,
+			"List all test suites", cmd_list_suites, 0, 0),
+		SHELL_CMD_ARG(list-testcases, NULL,
+			"List all test cases", cmd_list_cases, 0, 0),
+		SHELL_CMD_ARG(run-testsuite, &testsuite_names,
+			"Run test suite", cmd_run_suite, 2, 0),
+		SHELL_CMD_ARG(run-testcase, NULL, "Run testcase", cmd_run_suite, 2, 0),
+		SHELL_SUBCMD_SET_END /* Array terminated. */
+	);
+
+SHELL_CMD_REGISTER(ztest, &sub_ztest_cmds, "Ztest commands", NULL);
+#endif /* CONFIG_ZTEST_SHELL */
+
 int main(void)
 {
 #ifdef CONFIG_USERSPACE
@@ -1116,6 +1303,7 @@ int main(void)
 #endif /* CONFIG_USERSPACE */
 
 	z_init_mock();
+#ifndef CONFIG_ZTEST_SHELL
 	test_main();
 	end_report();
 	flush_log();
@@ -1152,7 +1340,8 @@ int main(void)
 		; /* Spin */
 	}
 	irq_unlock(key);
-#endif
+#endif /* CONFIG_ZTEST_NO_YIELD */
+#endif /* CONFIG_ZTEST_SHELL */
 	return 0;
 }
 #endif

subsys/testsuite/ztest/src/ztest_defaults.c

@@ -27,9 +27,9 @@ const char *ztest_relative_filename(const char *file)
  *
  * @param state The current state of the machine as it relates to the test executable.
  */
-void z_ztest_run_all(const void *state)
+void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter)
 {
-	ztest_run_test_suites(state);
+	ztest_run_test_suites(state, shuffle, suite_iter, case_iter);
 }
 
 /**

subsys/testsuite/ztest/src/ztest_posix.c

@@ -95,6 +95,8 @@ const char *ztest_get_test_args(void)
 	return test_args;
 }
 
+
+
 /**
  * @brief Lists registered unit tests in this binary, one per line
  *
@@ -126,12 +128,12 @@ int z_ztest_list_tests(void)
  *
  * @param state The current state of the machine as it relates to the test executable.
  */
-void z_ztest_run_all(const void *state)
+void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter)
 {
 	if (z_ztest_get_list_test()) {
 		z_ztest_list_tests();
 	} else {
-		ztest_run_test_suites(state);
+		ztest_run_test_suites(state, shuffle, suite_iter, case_iter);
 	}
 }
 

subsys/testsuite/ztest/src/ztest_shell.c

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2022 Google LLC
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/ztest.h>
+
+
+static const char *test_args;
+
+/**
+ * @brief Try to shorten a filename by removing the current directory
+ *
+ * This helps to reduce the very long filenames in assertion failures. It
+ * removes the current directory from the filename and returns the rest.
+ * This makes assertions a lot more readable, and sometimes they fit on one
+ * line.
+ *
+ * @param file Filename to check
+ * @returns Shortened filename, or @file if it could not be shortened
+ */
+const char *ztest_relative_filename(const char *file)
+{
+	return file;
+}
+
+/**
+ * Default entry point for running registered unit tests.
+ *
+ * @param state The current state of the machine as it relates to the test executable.
+ */
+void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter)
+{
+	ztest_run_test_suites(state, shuffle, suite_iter, case_iter);
+}
+
+void ztest_reset_test_args(void)
+{
+	test_args = NULL;
+}
+
+void ztest_set_test_args(char *args)
+{
+	test_args = strdup(args);
+}
+
+/**
+ * @brief Helper function to get command line test arguments
+ *
+ * @return const char*
+ */
+char *ztest_get_test_args(void)
+{
+	if (test_args != NULL) {
+		return strdup(test_args);
+	} else {
+		return NULL;
+	}
+}
+
+
+/**
+ * @brief Checks if the test_args contains the suite/test name.
+ *
+ * @param suite_name
+ * @param test_name
+ * @return true
+ * @return false
+ */
+static bool z_ztest_testargs_contains(const char *suite_name, const char *test_name)
+{
+	bool found = false;
+	char *test_args_local = ztest_get_test_args();
+	char *suite_test_pair;
+	char *last_suite_test_pair;
+	char *suite_arg;
+	char *test_arg;
+	char *last_arg;
+
+	if (test_args_local == NULL) {
+		return true;
+	}
+	suite_test_pair = strtok_r(test_args_local, ",", &last_suite_test_pair);
+
+	while (suite_test_pair && !found) {
+		suite_arg = strtok_r(suite_test_pair, ":", &last_arg);
+		test_arg = strtok_r(NULL, ":", &last_arg);
+
+		found = !strcmp(suite_arg, suite_name);
+		if (test_name && test_arg) {
+			found &= !strcmp(test_arg, "*") ||
+				 !strcmp(test_arg, test_name);
+		}
+
+		suite_test_pair = strtok_r(NULL, ",", &last_suite_test_pair);
+	}
+	return found;
+}
+
+/**
+ * @brief Determines if the test suite should run based on test cases listed
+ *	  in the command line argument.
+ *
+ * Overrides implementation in ztest.c
+ *
+ * @param state The current state of the machine as it relates to the test
+ *		executable.
+ * @param suite Pointer to ztest_suite_node
+ * @return true
+ * @return false
+ */
+bool z_ztest_should_suite_run(const void *state, struct ztest_suite_node *suite)
+{
+	char *test_args_local = ztest_get_test_args();
+	bool run_suite = true;
+
+	if (test_args_local != NULL && !z_ztest_testargs_contains(suite->name, NULL)) {
+		run_suite = false;
+		suite->stats->run_count++;
+	} else if (suite->predicate != NULL) {
+		run_suite = suite->predicate(state);
+	}
+
+	return run_suite;
+}
+
+
+
+/**
+ * @brief Determines if the test case should run based on test cases listed
+ *	  in the command line argument. Run all tests for non-posix builds
+ *
+ * @param suite - name of test suite
+ * @param test  - name of unit test
+ * @return true
+ * @return false
+ */
+bool z_ztest_should_test_run(const char *suite, const char *test)
+{
+	bool run_test = false;
+
+	run_test = z_ztest_testargs_contains(suite, test);
+
+	return run_test;
+}
+
+ZTEST_DMEM const struct ztest_arch_api ztest_api = {
+	.run_all = z_ztest_run_all,
+	.should_suite_run = z_ztest_should_suite_run,
+	.should_test_run = z_ztest_should_test_run
+};

tests/bluetooth/host/keys/bt_keys_get_addr/src/main.c

@@ -181,11 +181,11 @@ void test_main(void)
 {
 	/* Only startup suite will run. */
 	all_startup_checks_executed = false;
-	ztest_run_all(NULL);
+	ztest_run_all(NULL, false, 1, 1);
 
 	/* All other suites, except startup suite, will run. */
 	all_startup_checks_executed = true;
-	ztest_run_all(NULL);
+	ztest_run_all(NULL, false, 1, 1);
 
 	/* Check that all the suites in this binary ran at least once. */
 	ztest_verify_all_test_suites_ran();

tests/drivers/eeprom/api/src/main.c

@@ -178,7 +178,7 @@ static void run_tests_on_eeprom(const struct device *dev)
 
 	printk("Running tests on device \"%s\"\n", eeprom->name);
 	k_object_access_grant(eeprom, k_current_get());
-	ztest_run_all(NULL);
+	ztest_run_all(NULL, false, 1, 1);
 }
 
 void test_main(void)

tests/kernel/threads/thread_apis/src/main.c

@@ -43,7 +43,7 @@ static ZTEST_DMEM int tp = 10;
  */
 ZTEST(threads_lifecycle, test_systhreads_main)
 {
-	zassert_true(main_prio == CONFIG_MAIN_THREAD_PRIORITY);
+	zassert_true(main_prio == CONFIG_MAIN_THREAD_PRIORITY, "%d", CONFIG_MAIN_THREAD_PRIORITY);
 }
 
 /**
@@ -185,7 +185,7 @@ ZTEST_USER(threads_lifecycle, test_thread_name_user_get_set)
 	zassert_equal(ret, -EINVAL, "not a thread object");
 	ret = k_thread_name_copy(&z_main_thread, thread_name,
 				     sizeof(thread_name));
-	zassert_equal(ret, 0, "couldn't get main thread name");
+	zassert_equal(ret, 0, "couldn't get main thread name: %s (%d)", thread_name, ret);
 	LOG_DBG("Main thread name is '%s'", thread_name);
 
 	/* Set and get child thread's name */

tests/subsys/mgmt/mcumgr/cb_notifications/src/main.c

@@ -248,7 +248,7 @@ static void cleanup_test(void *p)
 void test_main(void)
 {
 	while (test_state.test_set < CB_NOTIFICATION_TEST_SET_COUNT) {
-		ztest_run_all(&test_state);
+		ztest_run_all(&test_state, false, 1, 1);
 		++test_state.test_set;
 	}
 

tests/subsys/mgmt/mcumgr/os_mgmt_datetime/src/main.c

@@ -1458,7 +1458,7 @@ static void cleanup_test(void *p)
 void test_main(void)
 {
 	while (test_state.test_set < OS_MGMT_DATETIME_TEST_SET_COUNT) {
-		ztest_run_all(&test_state);
+		ztest_run_all(&test_state, false, 1, 1);
 		++test_state.test_set;
 	}
 

tests/subsys/mgmt/mcumgr/os_mgmt_info/src/main.c

@@ -1499,7 +1499,7 @@ static void cleanup_test(void *p)
 void test_main(void)
 {
 	while (test_state.test_set < OS_MGMT_TEST_SET_COUNT) {
-		ztest_run_all(&test_state);
+		ztest_run_all(&test_state, false, 1, 1);
 		++test_state.test_set;
 	}
 

tests/ztest/error_hook/src/main.c

@@ -424,7 +424,7 @@ ZTEST(fail_assume_in_test, test_to_skip)
 
 void test_main(void)
 {
-	ztest_run_test_suites(NULL);
+	ztest_run_test_suites(NULL, false, 1, 1);
 	/* Can't run ztest_verify_all_test_suites_ran() since some tests are
 	 * skipped by design.
 	 */