cmake: Zephyr CMake package and CMake modules

Create a cmake/modules folder containing all Zephyr CMake modules.
All Zephyr cmake files that are included from boilerplate are now
converted into CMake modules which can be individually loaded.

The Zephyr CMake package is updated to support loading of individual
CMake modules using the COMPONENTS argument to `find_package(Zephyr)`.
If the COMPONENTS argument is not specified, the default Zephyr build
system will load.
If COMPONENTS is specified then, only those components and the
dependencies will be loaded.

If a Zephyr CMake module depends on another CMake module which has not
been loaded, it will automatically be loaded.

This allows us to modularize and reuse individual parts of the Zephyr
CMake build system in a more flexible way in future.

Such usage could be:
- Higher livel multi image build system
- Invocation of individual components, for example dts processing by
  twister without loading all build code
- Doc build
- Unittesting

With this new CMake package and CMake module scheme then direct
sourcing of boilerplate.cmake has been deprecated.

Signed-off-by: Torsten Rasmussen <Torsten.Rasmussen@nordicsemi.no>

cmake/app/boilerplate.cmake

@@ -1,12 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
-# This file must be included into the toplevel CMakeLists.txt file of
-# Zephyr applications.
-# Zephyr CMake package automatically includes this file when CMake function
-# find_package() is used.
-#
-# To ensure this file is loaded in a Zephyr application it must start with
-# one of those lines:
+######################################
+# The use of this file is deprecated #
+######################################
+
+# To build a Zephyr application it must start with one of those lines:
 #
 # find_package(Zephyr)
 # find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
@@ -14,99 +12,9 @@
 # The `REQUIRED HINTS $ENV{ZEPHYR_BASE}` variant is required for any application
 # inside the Zephyr repository.
 #
-# It exists to reduce boilerplate code that Zephyr expects to be in
-# application CMakeLists.txt code.
-
-# CMake version 3.20 is the real minimum supported version.
-#
-# Unfortunately CMake requires the toplevel CMakeLists.txt file to
-# define the required version, not even invoking it from an included
-# file, like boilerplate.cmake, is sufficient. It is however permitted
-# to have multiple invocations of cmake_minimum_required.
-#
-# Under these restraints we use a second 'cmake_minimum_required'
-# invocation in every toplevel CMakeLists.txt.
-cmake_minimum_required(VERSION 3.20.0)
-
-set(APPLICATION_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "Application Source Directory")
-set(APPLICATION_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} CACHE PATH "Application Binary Directory")
-
-set(__build_dir ${CMAKE_CURRENT_BINARY_DIR}/zephyr)
-
-set(PROJECT_BINARY_DIR ${__build_dir})
-
-message(STATUS "Application: ${APPLICATION_SOURCE_DIR}")
-
-set(ENV_ZEPHYR_BASE $ENV{ZEPHYR_BASE})
-# This add support for old style boilerplate include.
-if((NOT DEFINED ZEPHYR_BASE) AND (DEFINED ENV_ZEPHYR_BASE))
-  set(ZEPHYR_BASE ${ENV_ZEPHYR_BASE} CACHE PATH "Zephyr base")
-endif()
-
-find_package(ZephyrBuildConfiguration
-  QUIET NO_POLICY_SCOPE
-  NAMES ZephyrBuild
-  PATHS ${ZEPHYR_BASE}/../*
-  NO_CMAKE_PATH
-  NO_CMAKE_ENVIRONMENT_PATH
-  NO_SYSTEM_ENVIRONMENT_PATH
-  NO_CMAKE_PACKAGE_REGISTRY
-  NO_CMAKE_SYSTEM_PATH
-  NO_CMAKE_SYSTEM_PACKAGE_REGISTRY
+# Loading of this file directly is deprecated and only kept for backward compatibility.
+message(WARNING "Loading of Zephyr boilerplate.cmake directly is deprecated, "
+        "please use 'find_package(Zephyr)'"
 )
-#
-# Import more CMake functions and macros
-#
 
-include(CheckCCompilerFlag)
-include(CheckCXXCompilerFlag)
-include(${ZEPHYR_BASE}/cmake/extensions.cmake)
-include(${ZEPHYR_BASE}/cmake/version.cmake)  # depends on hex.cmake
-
-#
-# Find tools
-#
-
-include(${ZEPHYR_BASE}/cmake/python.cmake)
-include(${ZEPHYR_BASE}/cmake/west.cmake)
-include(${ZEPHYR_BASE}/cmake/ccache.cmake)
-
-include(${ZEPHYR_BASE}/cmake/root.cmake)
-#
-# Find Zephyr modules.
-# Those may contain additional DTS, BOARD, SOC, ARCH ROOTs.
-# Also create the Kconfig binary dir for generated Kconf files.
-#
-include(${ZEPHYR_BASE}/cmake/zephyr_module.cmake)
-
-include(${ZEPHYR_BASE}/cmake/boards.cmake)
-include(${ZEPHYR_BASE}/cmake/shields.cmake)
-include(${ZEPHYR_BASE}/cmake/arch.cmake)
-include(${ZEPHYR_BASE}/cmake/configuration_files.cmake)
-
-include(${ZEPHYR_BASE}/cmake/user_cache.cmake)
-include(${ZEPHYR_BASE}/cmake/verify-toolchain.cmake)
-include(${ZEPHYR_BASE}/cmake/host-tools.cmake)
-
-# Include board specific device-tree flags before parsing.
-include(${BOARD_DIR}/pre_dt_board.cmake OPTIONAL)
-
-# DTS should be close to kconfig because CONFIG_ variables from
-# kconfig and dts should be available at the same time.
-#
-# The DT system uses a C preprocessor for it's code generation needs.
-# This creates an awkward chicken-and-egg problem, because we don't
-# always know exactly which toolchain the user needs until we know
-# more about the target, e.g. after DT and Kconfig.
-#
-# To resolve this we find "some" C toolchain, configure it generically
-# with the minimal amount of configuration needed to have it
-# preprocess DT sources, and then, after we have finished processing
-# both DT and Kconfig we complete the target-specific configuration,
-# and possibly change the toolchain.
-include(${ZEPHYR_BASE}/cmake/generic_toolchain.cmake)
-include(${ZEPHYR_BASE}/cmake/dts.cmake)
-include(${ZEPHYR_BASE}/cmake/kconfig.cmake)
-include(${ZEPHYR_BASE}/cmake/soc.cmake)
-include(${ZEPHYR_BASE}/cmake/target_toolchain.cmake)
-include(${ZEPHYR_BASE}/cmake/kernel.cmake)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})

cmake/gen_version_h.cmake

@@ -22,5 +22,5 @@ if(NOT DEFINED BUILD_VERSION)
   endif()
 endif()
 
-include(${ZEPHYR_BASE}/cmake/version.cmake)
+include(${ZEPHYR_BASE}/cmake/modules/version.cmake)
 configure_file(${ZEPHYR_BASE}/version.h.in ${OUT_FILE})

cmake/modules/arch.cmake

@@ -23,6 +23,8 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
 # 'ARCH_ROOT' is a prioritized list of directories where archs may be
 # found. It always includes ${ZEPHYR_BASE} at the lowest priority.
 list(APPEND ARCH_ROOT ${ZEPHYR_BASE})

cmake/modules/boards.cmake

@@ -42,6 +42,11 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
+include(python)
+include(extensions)
+
 # Check that BOARD has been provided, and that it has not changed.
 # If user tries to change the BOARD, the BOARD value is reset to the BOARD_CACHED value.
 zephyr_check_cache(BOARD REQUIRED)

cmake/modules/ccache.cmake

@@ -3,6 +3,8 @@
 # Use ccache if it is installed, unless the user explicitly disables
 # it by setting USE_CCACHE=0.
 
+include_guard(GLOBAL)
+
 if(USE_CCACHE STREQUAL "0")
 else()
   find_program(CCACHE_FOUND ccache)

cmake/modules/configuration_files.cmake

@@ -20,6 +20,10 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
+include(extensions)
+
 if(DEFINED APPLICATION_CONFIG_DIR)
   string(CONFIGURE ${APPLICATION_CONFIG_DIR} APPLICATION_CONFIG_DIR)
   if(NOT IS_ABSOLUTE ${APPLICATION_CONFIG_DIR})

cmake/modules/dts.cmake

@@ -1,5 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(extensions)
+include(python)
+include(boards)
+include(generic_toolchain)
+
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/include/generated)
 
 # Zephyr code can configure itself based on a KConfig'uration with the

cmake/modules/extensions.cmake

@@ -1,5 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(user_cache)
+
+# Dependencies on CMake modules from the CMake distribution.
+include(CheckCCompilerFlag)
+include(CheckCXXCompilerFlag)
+
 ########################################################
 # Table of contents
 ########################################################

cmake/modules/generic_toolchain.cmake

@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(extensions)
+include(verify-toolchain)
+
 # Prevent CMake from testing the toolchain
 set(CMAKE_C_COMPILER_FORCED   1)
 set(CMAKE_CXX_COMPILER_FORCED 1)

cmake/modules/host-tools.cmake

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
 if(ZEPHYR_SDK_HOST_TOOLS)
   include(${ZEPHYR_BASE}/cmake/toolchain/zephyr/host-tools.cmake)
 endif()

cmake/modules/kconfig.cmake

@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(extensions)
+include(python)
+
 # autoconf.h is generated by Kconfig and placed in <build>/zephyr/include/generated/autoconf.h.
 # A project may request a custom location by setting AUTOCONF_H explicitly before
 # calling 'find_package(Zephyr)' or loading this module.
@@ -257,6 +262,11 @@ else()
   set(input_configs ${DOTCONFIG})
 endif()
 
+cmake_path(GET AUTOCONF_H PARENT_PATH autoconf_h_path)
+if(NOT EXISTS ${autoconf_h_path})
+  file(MAKE_DIRECTORY ${autoconf_h_path})
+endif()
+
 execute_process(
   COMMAND ${CMAKE_COMMAND} -E env
   ${COMMON_KCONFIG_ENV_SETTINGS}

cmake/modules/kernel.cmake

@@ -19,9 +19,11 @@
 # Important libraries:
 # - app: This is the main application library where the application can add
 #        source files that must be included when building Zephyr
-#
-# CMake module dependencies:
-# - All Zephyr CMake modules are required for this CMake module to work correctly
+
+include_guard(GLOBAL)
+
+# As this module is not intended for direct loading, but should be loaded through
+# find_package(Zephyr) then it won't be loading any Zephyr CMake modules by itself.
 
 define_property(GLOBAL PROPERTY ZEPHYR_LIBS
     BRIEF_DOCS "Global list of all Zephyr CMake libs that should be linked in"
@@ -208,8 +210,8 @@ add_subdirectory(${ZEPHYR_BASE} ${__build_dir})
 
 # Link 'app' with the Zephyr interface libraries.
 #
-# NB: This must be done in boilerplate.cmake because 'app' can only be
-# modified in the CMakeLists.txt file that created it. And it must be
+# NB: This must be done here because 'app' can only be modified in the
+# CMakeLists.txt file that created it. And it must be
 # done after 'add_subdirectory(${ZEPHYR_BASE} ${__build_dir})'
 # because interface libraries are defined while processing that
 # subdirectory.

cmake/modules/python.cmake

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
 # On Windows, instruct Python to output UTF-8 even when not
 # interacting with a terminal. This is required since Python scripts
 # are invoked by CMake code and, on Windows, standard I/O encoding defaults

cmake/modules/root.cmake

@@ -17,6 +17,10 @@
 # any relative path to absolute path and update the root list.
 # If a root is undefined it will still be undefined when this module has loaded.
 
+include_guard(GLOBAL)
+
+include(extensions)
+
 # Convert paths to absolute, relative from APPLICATION_SOURCE_DIR
 zephyr_file(APPLICATION_ROOT MODULE_EXT_ROOT)
 

cmake/modules/shields.cmake

@@ -22,6 +22,10 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
+include(extensions)
+
 # Check that SHIELD has not changed.
 zephyr_check_cache(SHIELD WATCH)
 

cmake/modules/soc.cmake

@@ -25,6 +25,10 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
+include(kconfig)
+
 # 'SOC_ROOT' is a prioritized list of directories where socs may be
 # found. It always includes ${ZEPHYR_BASE}/soc at the lowest priority.
 list(APPEND SOC_ROOT ${ZEPHYR_BASE})

cmake/modules/target_toolchain.cmake

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
 # Prevent CMake from testing the toolchain
 set(CMAKE_C_COMPILER_FORCED   1)
 set(CMAKE_CXX_COMPILER_FORCED 1)
@@ -60,5 +62,5 @@ add_custom_target(bintools)
 
 include(${TOOLCHAIN_ROOT}/cmake/compiler/${COMPILER}/target.cmake OPTIONAL)
 include(${TOOLCHAIN_ROOT}/cmake/linker/${LINKER}/target.cmake OPTIONAL)
-include(${CMAKE_CURRENT_LIST_DIR}/bintools/bintools_template.cmake)
+include(${ZEPHYR_BASE}/cmake/bintools/bintools_template.cmake)
 include(${TOOLCHAIN_ROOT}/cmake/bintools/${BINTOOLS}/target.cmake OPTIONAL)

cmake/modules/user_cache.cmake

@@ -25,6 +25,10 @@
 # Variables set by this module and not mentioned above are considered internal
 # use only and may be removed, renamed, or re-purposed without prior notice.
 
+include_guard(GLOBAL)
+
+include(python)
+
 function(find_appropriate_cache_directory dir)
   set(env_suffix_LOCALAPPDATA   .cache)
 

cmake/modules/verify-toolchain.cmake

@@ -15,6 +15,8 @@
 # it takes the following arguments:
 # FORMAT=json: Print the output as a json formatted string, useful for Python
 
+include_guard(GLOBAL)
+
 # This is the minimum required Zephyr-SDK version which supports CMake package
 set(TOOLCHAIN_ZEPHYR_MINIMUM_REQUIRED_VERSION 0.13.1)
 

cmake/modules/version.cmake

@@ -26,6 +26,11 @@
 # See also: independent and more dynamic ``BUILD_VERSION`` in
 # ``git.cmake``.
 
+# Note: version.cmake is loaded multiple times by ZephyrConfigVersion.cmake to
+# determine this Zephyr package version and thus the correct Zephyr CMake
+# package to load.
+# Therefore `version.cmake` should not use include_guard(GLOBAL).
+# The final load of `version.cmake` will setup correct build version values.
 
 include(${ZEPHYR_BASE}/cmake/hex.cmake)
 file(READ ${ZEPHYR_BASE}/VERSION ver)

cmake/modules/west.cmake

@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(python)
+
 # west is an optional dependency. We need to run west using the same
 # Python interpreter as everything else, though, so we play some extra
 # tricks.

cmake/modules/zephyr_default.cmake

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) 2021, Nordic Semiconductor ASA
+
+# This CMake module will load all Zephyr CMake modules in correct order for
+# default Zephyr build system.
+#
+# Outcome:
+# See individual CMake module descriptions
+
+include_guard(GLOBAL)
+
+# The code line below defines the real minimum supported CMake version.
+#
+# Unfortunately CMake requires the toplevel CMakeLists.txt file to define the
+# required version, not even invoking it from a CMake module is sufficient.
+# It is however permitted to have multiple invocations of cmake_minimum_required.
+cmake_minimum_required(VERSION 3.20.0)
+
+message(STATUS "Application: ${APPLICATION_SOURCE_DIR}")
+
+find_package(ZephyrBuildConfiguration
+  QUIET NO_POLICY_SCOPE
+  NAMES ZephyrBuild
+  PATHS ${ZEPHYR_BASE}/../*
+  NO_CMAKE_PATH
+  NO_CMAKE_ENVIRONMENT_PATH
+  NO_SYSTEM_ENVIRONMENT_PATH
+  NO_CMAKE_PACKAGE_REGISTRY
+  NO_CMAKE_SYSTEM_PATH
+  NO_CMAKE_SYSTEM_PACKAGE_REGISTRY
+)
+
+# Prepare user cache
+include(python)
+include(user_cache)
+
+# Load Zephyr extensions
+include(extensions)
+include(version)  # depends on hex.cmake
+
+#
+# Find tools
+#
+
+include(west)
+include(ccache)
+
+# Load default root settings
+include(root)
+
+#
+# Find Zephyr modules.
+# Those may contain additional DTS, BOARD, SOC, ARCH ROOTs.
+# Also create the Kconfig binary dir for generated Kconf files.
+#
+include(zephyr_module)
+
+include(boards)
+include(shields)
+include(arch)
+include(configuration_files)
+
+include(verify-toolchain)
+include(host-tools)
+
+# Include board specific device-tree flags before parsing.
+include(${BOARD_DIR}/pre_dt_board.cmake OPTIONAL)
+
+# DTS should be close to kconfig because CONFIG_ variables from
+# kconfig and dts should be available at the same time.
+#
+# The DT system uses a C preprocessor for it's code generation needs.
+# This creates an awkward chicken-and-egg problem, because we don't
+# always know exactly which toolchain the user needs until we know
+# more about the target, e.g. after DT and Kconfig.
+#
+# To resolve this we find "some" C toolchain, configure it generically
+# with the minimal amount of configuration needed to have it
+# preprocess DT sources, and then, after we have finished processing
+# both DT and Kconfig we complete the target-specific configuration,
+# and possibly change the toolchain.
+include(generic_toolchain)
+include(dts)
+include(kconfig)
+include(soc)
+include(target_toolchain)
+include(kernel)

cmake/modules/zephyr_module.cmake

@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
+include_guard(GLOBAL)
+
+include(extensions)
+include(python)
+
 # This cmake file provides functionality to import CMakeLists.txt and Kconfig
 # files for Zephyr modules into Zephyr build system.
 #

scripts/pylib/twister/twisterlib.py

@@ -3053,7 +3053,7 @@ class TestSuite(DisablePyTestCollectionMixin):
 
     @staticmethod
     def get_toolchain():
-        toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/verify-toolchain.cmake')
+        toolchain_script = Path(ZEPHYR_BASE) / Path('cmake/modules/verify-toolchain.cmake')
         result = CMake.run_cmake_script([toolchain_script, "FORMAT=json"])
 
         try:

share/zephyr-package/cmake/ZephyrConfig.cmake

@@ -11,17 +11,47 @@
 # It will be empty if not set in environment.
 
 macro(include_boilerplate location)
+  list(PREPEND CMAKE_MODULE_PATH ${ZEPHYR_BASE}/cmake/modules)
   if(ZEPHYR_UNITTEST)
     set(ZephyrUnittest_FOUND True)
     set(BOILERPLATE_FILE ${ZEPHYR_BASE}/subsys/testsuite/unittest.cmake)
   else()
     set(Zephyr_FOUND True)
-    set(BOILERPLATE_FILE ${ZEPHYR_BASE}/cmake/app/boilerplate.cmake)
   endif()
 
+  if(NOT DEFINED APPLICATION_SOURCE_DIR)
+    set(APPLICATION_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH
+        "Application Source Directory"
+    )
+  endif()
+
+  if(NOT DEFINED APPLICATION_BINARY_DIR)
+    set(APPLICATION_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} CACHE PATH
+        "Application Binary Directory"
+    )
+  endif()
+
+  set(__build_dir ${APPLICATION_BINARY_DIR}/zephyr)
+  set(PROJECT_BINARY_DIR ${__build_dir})
+
   if(NOT NO_BOILERPLATE)
+    list(LENGTH Zephyr_FIND_COMPONENTS components_length)
+    if(DEFINED BOILERPLATE_FILE)
       message("Including boilerplate (${location}): ${BOILERPLATE_FILE}")
       include(${BOILERPLATE_FILE} NO_POLICY_SCOPE)
+    elseif(components_length EQUAL 0)
+      message("Loading Zephyr default modules (${location}).")
+      include(zephyr_default NO_POLICY_SCOPE)
+    else()
+      message("Loading Zephyr module(s) (${location}): ${Zephyr_FIND_COMPONENTS}")
+      foreach(component ${Zephyr_FIND_COMPONENTS})
+        include(${component})
+      endforeach()
+    endif()
+  else()
+    message(WARNING "The NO_BOILERPLATE setting has been deprecated.\n"
+                    "Please use: 'find_package(Zephyr COMPONENTS <components>)'"
+    )
   endif()
 endmacro()
 

share/zephyr-package/cmake/ZephyrConfigVersion.cmake

@@ -56,7 +56,7 @@ if((DEFINED ZEPHYR_BASE) OR (DEFINED ENV_ZEPHYR_BASE))
     # We are the Zephyr to be used
 
     set(NO_PRINT_VERSION True)
-    include(${ZEPHYR_BASE}/cmake/version.cmake)
+    include(${ZEPHYR_BASE}/cmake/modules/version.cmake)
     # Zephyr uses project version, but CMake package uses PACKAGE_VERSION
     set(PACKAGE_VERSION ${PROJECT_VERSION})
     check_zephyr_version()
@@ -93,7 +93,7 @@ set(ZEPHYR_BASE ${CURRENT_ZEPHYR_DIR})
 # Tell version.cmake to not print as printing version for all Zephyr installations being tested
 # will lead to confusion on which is being used.
 set(NO_PRINT_VERSION True)
-include(${ZEPHYR_BASE}/cmake/version.cmake)
+include(${ZEPHYR_BASE}/cmake/modules/version.cmake)
 # Zephyr uses project version, but CMake package uses PACKAGE_VERSION
 set(PACKAGE_VERSION ${PROJECT_VERSION})
 set(ZEPHYR_BASE)