modules: introducing MODULE_EXT_ROOT to allow glue code in Zephyr repo

This commit introduces MODULE_EXT_ROOT which allows CMake and Kconfig
glue code to be placed outside of the Zephyr module repository.

This allows for placing glue code in Zephyr, but also allows users to
specify custom MODULE_EXT_ROOTs for glue code using either
`-DMODULE_EXT_ROOT` or `zephyr/module.yml` with
`build:settings:module_ext_root` settings.

MODULE_EXT_ROOT' is a list of directories, similar to other roots such
as BOARD_ROOT, DTS_ROOT, etc.
The Zephyr repo folder ${ZEPHYR_BASE} is always to the MODULE_EXT_ROOT
list as lowest priority.
For each MODULE_EXT_ROOT, the file
`<module_ext_root>/modules/modules.cmake` will be processed.

In Zephyr repo, the folder `modules/<module>/` contains CMakeLists.txt
and Kconfig glue code for the Zephyr module.

A Zephyr module can specify that CMakeLists.txt and Kconfig glue code is
placed in an external module root by specifying:
```
build:
  cmake-ext: True
  kconfig-ext: True
```

It is still possible to place the CMakeLists.txt and Kconfig files
directly in the Zephyr module using the existing:
```
build:
  cmake: <path>
  kconfig: <file>
```.

Signed-off-by: Torsten Rasmussen <Torsten.Rasmussen@nordicsemi.no>

Kconfig.zephyr

@@ -20,7 +20,6 @@ source "$(KCONFIG_BINARY_DIR)/Kconfig.soc.defconfig"
 
 menu "Modules"
 
-source "$(KCONFIG_BINARY_DIR)/Kconfig.modules"
 source "modules/Kconfig"
 
 endmenu

cmake/app/boilerplate.cmake

@@ -166,6 +166,13 @@ elseif(DEFINED ENV{ZEPHYR_EXTRA_MODULES})
   set(ZEPHYR_EXTRA_MODULES $ENV{ZEPHYR_EXTRA_MODULES})
 endif()
 
+# 'MODULE_EXT_ROOT' is a prioritized list of directories where module glue code
+# may be found. It always includes ${ZEPHYR_BASE} at the lowest priority.
+# For module roots, later entries may overrule module settings already defined
+# by processed module roots, hence first in list means lowest priority.
+zephyr_file(APPLICATION_ROOT MODULE_EXT_ROOT)
+list(INSERT MODULE_EXT_ROOT 0 ${ZEPHYR_BASE})
+
 #
 # Find Zephyr modules.
 # Those may contain additional DTS, BOARD, SOC, ARCH ROOTs.
@@ -190,6 +197,9 @@ add_custom_target(
 # Dummy add to generate files.
 zephyr_linker_sources(SECTIONS)
 
+zephyr_file(APPLICATION_ROOT BOARD_ROOT)
+list(APPEND BOARD_ROOT ${ZEPHYR_BASE})
+
 # 'BOARD_ROOT' is a prioritized list of directories where boards may
 # be found. It always includes ${ZEPHYR_BASE} at the lowest priority.
 zephyr_file(APPLICATION_ROOT BOARD_ROOT)

cmake/kconfig.cmake

@@ -74,6 +74,13 @@ foreach(module_name ${ZEPHYR_MODULE_NAMES})
        ZEPHYR_KCONFIG_MODULES_DIR
        "ZEPHYR_${MODULE_NAME_UPPER}_MODULE_DIR=${ZEPHYR_${MODULE_NAME_UPPER}_MODULE_DIR}"
   )
+
+  if(ZEPHYR_${MODULE_NAME_UPPER}_KCONFIG)
+    list(APPEND
+         ZEPHYR_KCONFIG_MODULES_DIR
+         "ZEPHYR_${MODULE_NAME_UPPER}_KCONFIG=${ZEPHYR_${MODULE_NAME_UPPER}_KCONFIG}"
+  )
+  endif()
 endforeach()
 
 # A list of common environment settings used when invoking Kconfig during CMake

cmake/zephyr_module.cmake

@@ -1,14 +1,27 @@
 # SPDX-License-Identifier: Apache-2.0
 
-# This cmake file provides functionality to import additional out-of-tree, OoT
+# This cmake file provides functionality to import CMakeLists.txt and Kconfig
-# CMakeLists.txt and Kconfig files into Zephyr build system.
+# files for Zephyr modules into Zephyr build system.
-# It uses -DZEPHYR_MODULES=<oot-path-to-module>[;<additional-oot-module(s)>]
+#
-# given to CMake for a list of folders to search.
+# CMakeLists.txt and Kconfig files can reside directly in the module or in a
-# It looks for: <oot-module>/zephyr/module.yml or
+# MODULE_EXT_ROOT.
-#               <oot-module>/zephyr/CMakeLists.txt
+# The `<module>/zephyr/module.yml` file specifies whether the build files are
-# to load the oot-module into Zephyr build system.
+# located in the module or in a MODULE_EXT_ROOT.
+#
+# A list of Zephyr modules can be provided to the build system using:
+#   -DZEPHYR_MODULES=<module-path>[;<additional-module(s)-path>]
+#
+# It looks for: <module>/zephyr/module.yml or
+#               <module>/zephyr/CMakeLists.txt
+# to load the module into Zephyr build system.
 # If west is available, it uses `west list` to obtain a list of projects to
 # search for zephyr/module.yml
+#
+# If the module.yml file specifies that build files are located in a
+# MODULE_EXT_ROOT then the variables:
+# - `ZEPHYR_<MODULE_NAME>_CMAKE_DIR` is used for inclusion of the CMakeLists.txt
+# - `ZEPHYR_<MODULE_NAME>_KCONFIG` is used for inclusion of the Kconfig
+# files into the build system.
 
 if(ZEPHYR_MODULES)
   set(ZEPHYR_MODULES_ARG "--modules" ${ZEPHYR_MODULES})
@@ -56,10 +69,25 @@ if(WEST OR ZEPHYR_MODULES)
       # lazy regexes (it supports greedy only).
       string(REGEX REPLACE "\"(.*)\":\".*\"" "\\1" key ${setting})
       string(REGEX REPLACE "\".*\":\"(.*)\"" "\\1" value ${setting})
+      # MODULE_EXT_ROOT is process order which means module roots processed
+      # later wins. To ensure ZEPHYR_BASE stays first, and command line settings
+      # are processed last, we insert at position 1.
+      if ("${key}" STREQUAL "MODULE_EXT_ROOT")
+        list(INSERT ${key} 1 ${value})
+      else()
         list(APPEND ${key} ${value})
+      endif()
     endforeach()
   endif()
 
+  foreach(root ${MODULE_EXT_ROOT})
+    if(NOT EXISTS ${root})
+      message(FATAL_ERROR "No `modules.cmake` found in module root `${root}`.")
+    endif()
+
+    include(${root}/modules/modules.cmake)
+  endforeach()
+
   if(EXISTS ${CMAKE_BINARY_DIR}/zephyr_modules.txt)
     file(STRINGS ${CMAKE_BINARY_DIR}/zephyr_modules.txt ZEPHYR_MODULES_TXT
          ENCODING UTF-8)
@@ -69,6 +97,7 @@ if(WEST OR ZEPHYR_MODULES)
       # Match "<name>":"<path>" for each line of file, each corresponding to
       # one module. The use of quotes is required due to CMake not supporting
       # lazy regexes (it supports greedy only).
+      string(CONFIGURE ${module} module)
       string(REGEX REPLACE "\"(.*)\":\".*\":\".*\"" "\\1" module_name ${module})
       string(REGEX REPLACE "\".*\":\"(.*)\":\".*\"" "\\1" module_path ${module})
       string(REGEX REPLACE "\".*\":\".*\":\"(.*)\"" "\\1" cmake_path ${module})

modules/Kconfig

@@ -1,6 +1,10 @@
 # Copyright (c) 2019 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+comment "Available modules."
+
+source "$(KCONFIG_BINARY_DIR)/Kconfig.modules"
+
 comment "Optional modules. Make sure they're installed, via the project manifest."
 
 source "modules/Kconfig.altera"
@@ -32,3 +36,23 @@ source "modules/Kconfig.tinycrypt"
 source "modules/Kconfig.vega"
 source "modules/Kconfig.xtensa"
 source "modules/Kconfig.mcuboot_bootutil"
+
+comment "Unavailable modules, please install those via the project manifest."
+
+# List of comments to display when Zephyr modules are not available, please
+# use the following syntax:
+# ---------------------------------------------------
+# comment "<module_name> module not available."
+#	depends on !ZEPHYR_<MODULE_NAME_UPPER>_MODULE
+#
+# Remember to add the following code inside the `<module>/Kconfig file:
+# ---------------------------------------------------
+# config ZEPHYR_<MODULE_NAME_UPPER>_MODULE
+# 	bool
+
+# This ensures that symbols are available in Kconfig for dependency checking
+# and referencing, while keeping the settings themselves unavailable when the
+# modules are not present in the workspace
+if 0
+osource "modules/*/Kconfig"
+endif

modules/modules.cmake

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+
+file(GLOB cmake_modules "${CMAKE_CURRENT_LIST_DIR}/*/CMakeLists.txt")
+
+foreach(module ${cmake_modules})
+  get_filename_component(module_dir  ${module} DIRECTORY)
+  get_filename_component(module_name ${module_dir} NAME)
+  string(TOUPPER ${module_name} MODULE_NAME_UPPER)
+  set(ZEPHYR_${MODULE_NAME_UPPER}_CMAKE_DIR ${module_dir})
+endforeach()
+
+file(GLOB kconfig_modules "${CMAKE_CURRENT_LIST_DIR}/*/Kconfig")
+
+foreach(module ${kconfig_modules})
+  get_filename_component(module_dir  ${module} DIRECTORY)
+  get_filename_component(module_name ${module_dir} NAME)
+  string(TOUPPER ${module_name} MODULE_NAME_UPPER)
+  set(ZEPHYR_${MODULE_NAME_UPPER}_KCONFIG ${module_dir}/Kconfig)
+endforeach()

scripts/zephyr_module.py

@@ -43,6 +43,14 @@ mapping:
       kconfig:
         required: false
         type: str
+      cmake-ext:
+        required: false
+        type: bool
+        default: false
+      kconfig-ext:
+        required: false
+        type: bool
+        default: false
       depends:
         required: false
         type: seq
@@ -64,6 +72,9 @@ mapping:
           arch_root:
             required: false
             type: str
+          module_ext_root:
+            required: false
+            type: str
   tests:
     required: false
     type: seq
@@ -126,6 +137,14 @@ def process_cmake(module, meta):
     section = meta.get('build', dict())
     module_path = PurePath(module)
     module_yml = module_path.joinpath('zephyr/module.yml')
+
+    cmake_extern = section.get('cmake-ext', False)
+    if cmake_extern:
+        return('\"{}\":\"{}\":\"{}\"\n'
+               .format(module_path.name,
+                       module_path.as_posix(),
+                       "${ZEPHYR_" + module_path.name.upper() + "_CMAKE_DIR}"))
+
     cmake_setting = section.get('cmake', None)
     if not validate_setting(cmake_setting, module, 'CMakeLists.txt'):
         sys.exit('ERROR: "cmake" key in {} has folder value "{}" which '
@@ -144,25 +163,41 @@ def process_cmake(module, meta):
                .format(module_path.name,
                        module_path.as_posix()))
 
+
 def process_settings(module, meta):
     section = meta.get('build', dict())
     build_settings = section.get('settings', None)
     out_text = ""
 
     if build_settings is not None:
-        for root in ['board', 'dts', 'soc', 'arch']:
+        for root in ['board', 'dts', 'soc', 'arch', 'module_ext']:
             setting = build_settings.get(root+'_root', None)
             if setting is not None:
                 root_path = PurePath(module) / setting
-                out_text += f'"{root.upper()}_ROOT":"{root_path.as_posix()}"\n'
+                out_text += f'"{root.upper()}_ROOT":'
+                out_text += f'"{root_path.as_posix()}"\n'
 
     return out_text
 
 
+def kconfig_snippet(path, kconfig_file=None):
+    snippet = (f'menu "{path.name} ({path})"',
+               f'osource "{kconfig_file.resolve().as_posix()}"' if kconfig_file
+               else f'osource "$(ZEPHYR_{path.name.upper()}_KCONFIG)"',
+               f'config ZEPHYR_{path.name.upper()}_MODULE',
+               '	bool',
+               '	default y',
+               'endmenu\n')
+    return '\n'.join(snippet)
+
+
 def process_kconfig(module, meta):
     section = meta.get('build', dict())
     module_path = PurePath(module)
     module_yml = module_path.joinpath('zephyr/module.yml')
+    kconfig_extern = section.get('kconfig-ext', False)
+    if kconfig_extern:
+        return kconfig_snippet(module_path)
 
     kconfig_setting = section.get('kconfig', None)
     if not validate_setting(kconfig_setting, module):
@@ -172,11 +207,11 @@ def process_kconfig(module, meta):
 
     kconfig_file = os.path.join(module, kconfig_setting or 'zephyr/Kconfig')
     if os.path.isfile(kconfig_file):
-        return 'osource "{}"\n\n'.format(Path(kconfig_file)
+        return kconfig_snippet(module_path, Path(kconfig_file))
-                                         .resolve().as_posix())
     else:
         return ""
 
+
 def process_twister(module, meta):
 
     out = ""