ztest: Add documentation

Origin: Original

Change-Id: I0927c25fbbba5d4863f199d058d311c10d52d784
Signed-off-by: Jaakko Hannikainen <jaakko.hannikainen@intel.com>
Signed-off-by: Anas Nashif <anas.nashif@intel.com>

doc/api/api.rst

@@ -26,4 +26,5 @@ The use of the Zephyr APIs is the same for all SoCs and boards.
    system_log.rst
    power_management_api
    file_system
+   testing
 

doc/api/mocking.c

@@ -0,0 +1,35 @@
+#include <ztest.h>
+
+static void expect_two_parameters(int a, int b)
+{
+	ztest_check_expected_value(a);
+	ztest_check_expected_value(b);
+}
+
+static void parameter_tests(void)
+{
+	ztest_expect_value(expect_two_parameters, a, 2);
+	ztest_expect_value(expect_two_parameters, b, 3);
+	expect_two_parameters(2, 3);
+}
+
+static int returns_int(void)
+{
+	return ztest_get_return_value();
+}
+
+static void return_value_tests(void)
+{
+	ztest_returns_value(returns_int, 5);
+	assert_equal(returns_int(), 5, NULL);
+}
+
+void test_main(void)
+{
+	ztest_test_suite(mock_framework_tests,
+		ztest_unit_test(parameter_test),
+		ztest_unit_test(return_value_test)
+	);
+
+	ztest_run_test_suite(mock_framework_tests);
+}

doc/api/testing.rst

@@ -0,0 +1,147 @@
+.. _testing:
+
+Zephyr testing framework
+########################
+
+Ztest provides a simple testing framework intended to be used during
+development.
+It provides basic assertion macros and a generic test structure.
+
+The framework can be used in two ways, either as a generic framework for
+integration testing, or for unit testing specific modules.
+
+Quick start - Integration testing
+*********************************
+
+A simple working base is located at `samples/testing/integration`. Just copy the
+files to `tests/` and edit them for your needs. The test will then be
+automatically built and run by the sanitycheck script. If you are testing the `bar`
+component of `foo`, you should copy the sample folder to tests/foo/bar. It can
+then be tested with `./scripts/sanitycheck -s tests/foo/bar/test`.
+
+The sample contains the following files:
+
+Makefile
+
+.. literalinclude:: ../../samples/testing/integration/Makefile
+   :language: Make
+   :linenos:
+
+testcase.ini
+
+.. literalinclude:: ../../samples/testing/integration/testcase.ini
+   :language: ini
+   :linenos:
+
+prj.conf
+
+.. literalinclude:: ../../samples/testing/integration/prj.conf
+   :language: text
+   :linenos:
+
+src/Makefile
+
+.. literalinclude:: ../../samples/testing/integration/src/Makefile
+   :language: Make
+   :linenos:
+
+src/main.c
+
+.. literalinclude:: ../../samples/testing/integration/src/main.c
+   :language: c
+   :linenos:
+
+.. contents::
+   :depth: 1
+   :local:
+   :backlinks: top
+
+Quick start - Unit testing
+**************************
+
+Ztest can be used for unit testing. This means that rather than including the
+entire Zephyr OS for testing a single function, you can focus the testing
+efforts into the specific module in question. This will speed up testing since
+only the module will have to be compiled in, and the tested functions will be
+called directly.
+
+Since you won't be including basic kernel data structures that most code
+depends on, you have to provide function stubs in the test. Ztest provides
+some helpers for mocking functions, as demonstrated below.
+
+In a unit test, mock objects can simulate the behavior of complex real objects
+and are used to decide whether a test failed or passed by verifying whether an
+interaction with an object occurred, and if required, to assert the order of
+that interaction.
+
+The `samples/testing/unit` folder contains an example for testing
+the net-buf api of Zephyr.
+
+Makefile
+
+.. literalinclude:: ../../samples/testing/unit/Makefile
+   :language: Make
+   :linenos:
+
+testcase.ini
+
+.. literalinclude:: ../../samples/testing/unit/testcase.ini
+   :language: ini
+   :linenos:
+
+main.c
+
+.. literalinclude:: ../../samples/testing/unit/main.c
+   :language: c
+   :linenos:
+
+API reference
+*************
+
+Running tests
+-------------
+
+.. doxygengroup:: ztest_test
+   :project: Zephyr
+   :content-only:
+
+Assertions
+----------
+
+These macros will instantly fail the test if the related assertion fails.
+When an assertion fails, it will print the current file, line and function,
+alongside a reason for the failure and an optional message. If the config
+option `CONFIG_ZTEST_ASSERT_VERBOSE=0`, the assertions will only print the
+file and line numbers, reducing the binary size of the test.
+
+Example output for a failed macro from
+`assert_equal(buf->ref, 2, "Invalid refcount")`:
+
+.. code-block:: none
+
+    Assertion failed at main.c:62: test_get_single_buffer: Invalid refcount (buf->ref not equal to 2)
+    Aborted at unit test function
+
+.. doxygengroup:: ztest_assert
+   :project: Zephyr
+   :content-only:
+
+Mocking
+-------
+
+These functions allow abstracting callbacks and related functions and
+controlling them from specific tests. You can enable the mocking framework by
+setting `CONFIG_ZTEST_MOCKING=y` in the configuration file of the test.
+The amount of concurrent return values and expected parameters is
+limited by `ZTEST_PARAMETER_COUNT`.
+
+Here is an example for configuring the function `expect_two_parameters` to
+expect the values `a=2` and `b=3`, and telling `returns_int` to return `5`:
+
+.. literalinclude:: mocking.c
+   :language: c
+   :linenos:
+
+.. doxygengroup:: ztest_mock
+   :project: Zephyr
+   :content-only:

doc/doxygen.config

@@ -105,8 +105,9 @@ INPUT                  = \
                         include/arch/arm/ include/arch/arm/cortex_m \
                         include/arch/nios2/ \
                         lib/libc/minimal/include/ \
-			ext/lib/crypto/tinycrypt/include/ \
-			lib/iot/zoap/zoap.h
+                        ext/lib/crypto/tinycrypt/include/ \
+                        lib/iot/zoap/zoap.h \
+                        tests/ztest/include/
 INPUT_ENCODING         = UTF-8
 FILE_PATTERNS          = "*.c" "*.h" "*.S"
 RECURSIVE              = YES

samples/testing/integration/Makefile

@@ -0,0 +1,5 @@
+BOARD ?= qemu_x86
+KERNEL_TYPE ?= nano
+CONF_FILE ?= prj.conf
+
+include $(ZEPHYR_BASE)/Makefile.inc

samples/testing/integration/prj.conf

@@ -0,0 +1 @@
+CONFIG_ZTEST=y

samples/testing/integration/src/Makefile

@@ -0,0 +1,3 @@
+obj-y = main.o
+
+include $(ZEPHYR_BASE)/tests/Makefile.test

samples/testing/integration/src/main.c

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <ztest.h>
+
+static void assert_tests(void)
+{
+	assert_true(1, "1 was false");
+	assert_false(0, "0 was true");
+	assert_is_null(NULL, "NULL was not NULL");
+	assert_not_null("foo", "\"foo\" was NULL");
+	assert_equal(1, 1, "1 was not equal to 1");
+	assert_equal_ptr(NULL, NULL, "NULL was not equal to NULL");
+}
+
+void test_main(void)
+{
+	ztest_test_suite(framework_tests,
+		ztest_unit_test(assert_tests)
+	);
+
+	ztest_run_test_suite(framework_tests);
+}

samples/testing/integration/testcase.ini

@@ -0,0 +1,2 @@
+[test]
+tags = my_tags

samples/testing/unit/Makefile

@@ -0,0 +1 @@
+include $(ZEPHYR_BASE)/tests/unit/Makefile.unittest

samples/testing/unit/main.c

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <ztest.h>
+
+#include <net/buf.c>
+
+void nano_fifo_init(struct nano_fifo *fifo) {}
+void nano_fifo_put_list(struct nano_fifo *fifo, void *head, void *tail) {}
+
+nano_context_type_t sys_execution_context_type_get(void)
+{
+	return NANO_CTX_FIBER;
+}
+
+void *nano_fifo_get(struct nano_fifo *fifo, int32_t timeout)
+{
+	return ztest_get_return_value_ptr();
+}
+
+void nano_fifo_put(struct nano_fifo *fifo, void *data)
+{
+	ztest_check_expected_value(data);
+}
+
+#define BUF_COUNT 1
+#define BUF_SIZE 74
+
+static struct nano_fifo bufs_fifo;
+static NET_BUF_POOL(bufs_pool, BUF_COUNT, BUF_SIZE, &bufs_fifo,
+		NULL, sizeof(int));
+
+static void init_pool(void)
+{
+	ztest_expect_value(nano_fifo_put, data, &bufs_pool);
+	net_buf_pool_init(bufs_pool);
+}
+
+static void test_get_single_buffer(void)
+{
+	struct net_buf *buf;
+
+	init_pool();
+
+	ztest_returns_value(nano_fifo_get, bufs_pool);
+	buf = net_buf_get_timeout(&bufs_fifo, 0, TICKS_NONE);
+
+	assert_equal_ptr(buf, &bufs_pool[0], "Returned buffer not from pool");
+	assert_equal(buf->ref, 1, "Invalid refcount");
+	assert_equal(buf->len, 0, "Invalid length");
+	assert_equal(buf->flags, 0, "Invalid flags");
+	assert_equal_ptr(buf->frags, NULL, "Frags not NULL");
+}
+
+void test_main(void)
+{
+	ztest_test_suite(net_buf_test,
+		ztest_unit_test(test_get_single_buffer)
+	);
+
+	ztest_run_test_suite(net_buf_test);
+}

samples/testing/unit/testcase.ini

@@ -0,0 +1,3 @@
+[test]
+type = unit
+tags = buf

tests/ztest/include/ztest.h

@@ -14,13 +14,24 @@
  * limitations under the License.
  */
 
+/**
+ * @file
+ *
+ * @brief Zephyr testing suite
+ */
+
 #ifndef __ZTEST_H__
 #define __ZTEST_H__
 
+/**
+ * @defgroup ztest Zephyr testing suite
+ */
+
 #ifndef KERNEL
 #define CONFIG_STDOUT_CONSOLE 1
 #define CONFIG_ZTEST_ASSERT_VERBOSE 1
 #define CONFIG_ZTEST_MOCKING
+#define CONFIG_MICROKERNEL 1
 /* FIXME: Properly integrate with Zephyr's arch specific code */
 #define CONFIG_X86 1
 #define PRINT printf

tests/ztest/include/ztest_assert.h

@@ -15,6 +15,12 @@
  * limitations under the License.
  */
 
+/**
+ * @file
+ *
+ * @brief Zephyr testing framework assertion macros
+ */
+
 #ifndef __ZTEST_ASSERT_H__
 #define __ZTEST_ASSERT_H__
 
@@ -60,20 +66,89 @@ static inline void _assert(int cond, const char *msg, const char *default_msg,
 
 #endif /* CONFIG_ZTEST_ASSERT_VERBOSE */
 
+/**
+ * @defgroup ztest_assert Ztest assertion macros
+ * @ingroup ztest
+ *
+ * This module provides assertions when using Ztest.
+ *
+ * @{
+ */
+
+/**
+ * @brief Fail the test, if @a cond is false
+ *
+ * You probably don't need to call this macro directly. You should
+ * instead use assert_{condition} macros below.
+ *
+ * @param cond Condition to check
+ * @param msg Optional, can be NULL. Message to print if @a cond is false.
+ * @param default_msg Message to print if @a cond is false
+ */
+
 #define assert(cond, msg, default_msg) \
 	_assert(cond, msg, msg ? ("(" default_msg ")") : (default_msg), \
 		__FILE__, __LINE__, __func__)
 
+/**
+ * @brief Assert that this function call won't be reached
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_unreachable(msg) assert(0, msg, "Reached unreachable code")
 
+/**
+ * @brief Assert that @a cond is true
+ * @param cond Condition to check
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_true(cond, msg) assert(cond, msg, #cond " is false")
+
+/**
+ * @brief Assert that @a cond is false
+ * @param cond Condition to check
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_false(cond, msg) assert(!(cond), msg, #cond " is true")
 
+/**
+ * @brief Assert that @a ptr is NULL
+ * @param ptr Pointer to compare
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_is_null(ptr, msg) assert((ptr) == NULL, msg, #ptr " is not NULL")
+
+/**
+ * @brief Assert that @a ptr is not NULL
+ * @param ptr Pointer to compare
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_not_null(ptr, msg) assert((ptr) != NULL, msg, #ptr " is NULL")
 
+/**
+ * @brief Assert that @a a equals @a b
+ *
+ * @a a and @a b won't be converted and will be compared directly.
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_equal(a, b, msg) assert((a) == (b), msg, #a " not equal to " #b)
+
+/**
+ * @brief Assert that @a a equals @a b
+ *
+ * @a a and @a b will be converted to `void *` before comparing.
+ *
+ * @param a Value to compare
+ * @param b Value to compare
+ * @param msg Optional message to print if the assertion fails
+ */
 #define assert_equal_ptr(a, b, msg) \
 	assert((void *)(a) == (void *)(b), msg, #a " not equal to  " #b)
 
+/**
+ * @}
+ */
+
 #endif /* __ZTEST_ASSERT_H__ */

tests/ztest/include/ztest_mock.h

@@ -14,26 +14,89 @@
  * limitations under the License.
  */
 
+/**
+ * @file
+ *
+ * @brief Ztest mocking support
+ */
+
 #ifndef __ZTEST_MOCK_H__
 #define __ZTEST_MOCK_H__
 
+/**
+ * @defgroup ztest_mock Ztest mocking support
+ * @ingroup ztest
+ *
+ * This module provides simple mocking functions for unit testing. These
+ * need CONFIG_ZTEST_MOCKING=y.
+ *
+ * @{
+ */
+
+/**
+ * @brief Tell function @a func to expect the value @a value for @a param
+ *
+ * When using ztest_check_expected_value(), tell that the value of @a param
+ * should be @a value. The value will internally be stored as an `uintptr_t`.
+ *
+ * @param func Function in question
+ * @param param Parameter for which the value should be set
+ * @param value Value for @a param
+ */
 #define ztest_expect_value(func, param, value) \
 	_ztest_expect_value(STRINGIFY(func), STRINGIFY(param), \
 			    (uintptr_t)(value))
 
+/**
+ * @brief If @a param doesn't match the value set by ztest_expect_value(),
+ * fail the test
+ *
+ * This will first check that does @a param have a value to be expected, and
+ * then checks whether the value of the parameter is equal to the expected
+ * value. If either of these checks fail, the current test will fail. This
+ * must be called from the called function.
+ *
+ * @param param Parameter to check
+ */
 #define ztest_check_expected_value(param) \
 	_ztest_check_expected_value(__func__, STRINGIFY(param), \
 				    (uintptr_t)(param))
 
+/**
+ * @brief Tell @a func that it should return @a value
+ *
+ * @param func Function that should return @a value
+ * @param value Value to return from @a func
+ */
 #define ztest_returns_value(func, value) \
 	_ztest_returns_value(STRINGIFY(func), (uintptr_t)(value))
 
+/**
+ * @brief Get the return value for current function
+ *
+ * The return value must have been set previously with ztest_returns_value().
+ * If no return value exists, the current test will fail.
+ *
+ * @returns The value the current function should return
+ */
 #define ztest_get_return_value() \
 	_ztest_get_return_value(__func__)
 
+/**
+ * @brief Get the return value as a pointer for current function
+ *
+ * The return value must have been set previously with ztest_returns_value().
+ * If no return value exists, the current test will fail.
+ *
+ * @returns The value the current function should return as a `void *`
+ */
 #define ztest_get_return_value_ptr() \
 	((void *)_ztest_get_return_value(__func__))
 
+/**
+ * @}
+ */
+
 #ifdef CONFIG_ZTEST_MOCKING
 
 #include <stdint.h>

tests/ztest/include/ztest_test.h

@@ -14,6 +14,12 @@
  * limitations under the License.
  */
 
+/**
+ * @file
+ *
+ * @brief Zephyr testing framework _test.
+ */
+
 #ifndef __ZTEST_TEST_H__
 #define __ZTEST_TEST_H__
 
@@ -26,24 +32,91 @@ struct unit_test {
 
 void _ztest_run_test_suite(const char *name, struct unit_test *suite);
 
+/**
+ * @defgroup ztest_test Ztest testing macros
+ * @ingroup ztest
+ *
+ * This module eases the testing process by providing helpful macros and other
+ * testing structures.
+ *
+ * @{
+ */
+
+/**
+ * @brief Fail the currently running test.
+ *
+ * This is the function called from failed assertions and the like. You
+ * probably don't need to call it yourself.
+ */
 void ztest_test_fail(void);
 
+/**
+ * @brief Do nothing, successfully.
+ *
+ * Unit test / setup function / teardown function that does
+ * nothing, successfully. Can be used as a parameter to
+ * ztest_unit_test_setup_teardown().
+ */
 static inline void unit_test_noop(void)
 {
 }
 
+/**
+ * @brief Define a test with setup and teardown functions
+ *
+ * This should be called as an argument to ztest_test_suite. The test will
+ * be run in the following order: @a setup, @a fn, @a teardown.
+ *
+ * @param fn Main test function
+ * @param setup Setup function
+ * @param teardown Teardown function
+ */
+
 #define ztest_unit_test_setup_teardown(fn, setup, teardown) { \
 		STRINGIFY(fn), fn, setup, teardown \
 }
 
+
+/**
+ * @brief Define a test function
+ *
+ * This should be called as an argument to ztest_test_suite.
+ *
+ * @param fn Test function
+ */
+
 #define ztest_unit_test(fn) \
 	ztest_unit_test_setup_teardown(fn, unit_test_noop, unit_test_noop)
 
+/**
+ * @brief Define a test suite
+ *
+ * This function should be called in the following fashion:
+ * ```{.c}
+ *      ztest_test_suite(test_suite_name,
+ *              ztest_unit_test(test_function),
+ *              ztest_unit_test(test_other_function)
+ *      );
+ *
+ *      ztest_run_test_suite(test_suite_name);
+ * ```
+ *
+ * @param name Name of the testing suite
+ */
 #define ztest_test_suite(name, ...) \
 	struct unit_test _##name[] = { \
 		__VA_ARGS__, { 0 } \
 	}
+/**
+ * @brief Run the specified test suite.
+ *
+ * @param suite Test suite to run.
+ */
 #define ztest_run_test_suite(suite) \
 	_ztest_run_test_suite(#suite, _##suite)
 
+/**
+ * @}
+ */
+
 #endif /* __ZTEST_ASSERT_H__ */