doc: use doxyrunner

doxyrunner plugin replaces a series of CMake+Python hacks. These include
input changes tracking and incremental build output simulation.

Signed-off-by: Gerard Marull-Paretas <gerard.marull@nordicsemi.no>

doc/CMakeLists.txt

@@ -93,11 +93,8 @@ set(I18NSPHINXOPTS  ${SPHINXOPTS})
 
 set(DOXYFILE_IN ${CMAKE_CURRENT_LIST_DIR}/zephyr.doxyfile.in)
 set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/zephyr.doxyfile)
-set(DOXY_PERM_OUT ${CMAKE_CURRENT_BINARY_DIR}/doxygen)
-set(DOXY_OUT ${CMAKE_CURRENT_BINARY_DIR}/latest_doxygen)
+set(DOXY_OUT ${CMAKE_CURRENT_BINARY_DIR}/doxygen)
 set(RST_OUT ${CMAKE_CURRENT_BINARY_DIR}/rst)
-set(DOC_LOG ${CMAKE_CURRENT_BINARY_DIR}/doc.log)
-set(DOXY_LOG ${CMAKE_CURRENT_BINARY_DIR}/doxy.log)
 set(DOC_WARN ${CMAKE_CURRENT_BINARY_DIR}/doc.warnings)
 set(CONTENT_OUTPUTS ${CMAKE_CURRENT_BINARY_DIR}/extracted-content.txt)
 
@@ -140,53 +137,11 @@ add_custom_target(
   COMMENT "Copying files to ${RST_OUT}"
 )
 
-# For incremental builds not to miss any source change this MUST be kept
-# a superset of INPUT= and FILE_PATTERNS= in zephyr.doxyfile.in
-#
-# NOTE: any changes here should be reflected in .github/workflows/doc-build.yml
-file(GLOB_RECURSE  DOXY_SOURCES
-  ${ZEPHYR_BASE}/include/*.[c,h,S]
-  ${ZEPHYR_BASE}/kernel/include/kernel_arch_interface.h
-  ${ZEPHYR_BASE}/lib/libc/*.[c,h,S]
-  ${ZEPHYR_BASE}/subsys/testsuite/ztest/include/*.[h,c,S]
-  ${ZEPHYR_BASE}/tests/*.[h,c,S]
-  )
-# For debug. Also find generated list in doc/_build/(build.ninja|CMakeFiles/)
-# message("DOXY_SOURCES= " ${DOXY_SOURCES})
-
-set(ARGS ${DOXYFILE_OUT})
-set(DOXY_RUN_TSTAMP ${CMAKE_CURRENT_BINARY_DIR}/last_doxy_run_tstamp)
-
-
-# Create timestamp first so we re-run if source files are edited while
-# doxygen is running
-add_custom_command(
-  OUTPUT ${DOXY_RUN_TSTAMP}
-  COMMAND cmake -E touch ${DOXY_RUN_TSTAMP}
-  COMMAND ${CMAKE_COMMAND}
-    -DCOMMAND=${DOXYGEN_EXECUTABLE}
-    -DARGS="${ARGS}"
-    -DOUTPUT_FILE=${DOXY_LOG}
-    -DERROR_FILE=${DOXY_LOG}
-    -DWORKING_DIRECTORY=${CMAKE_CURRENT_LIST_DIR}
-    -P ${ZEPHYR_BASE}/cmake/util/execute_process.cmake
-  DEPENDS ${DOXY_SOURCES}
-  COMMENT "Running ${DOXYGEN_EXECUTABLE}"
-)
-
-# Doxygen doesn't support incremental builds.
-# It could be ok because it's pretty fast.
-# But it's not because it has a cascade effect on sphinx:
-# https://sourceforge.net/p/doxygen/mailman/message/36580807/
-# For now this optimization speeds-up non-doxygen documentation work
-# only (by one order of magnitude).
 add_custom_target(
-  doxygen_sync
-  COMMAND ${CMAKE_COMMAND} -E env
-    ${PYTHON_EXECUTABLE} _scripts/doxygen_sync.py -o ${DOXY_PERM_OUT} -l ${DOXY_OUT}
+  doxygen
+  COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT}
   WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-  DEPENDS ${DOXY_RUN_TSTAMP}
-  COMMENT "Syncing Doxygen output"
+  COMMENT "Running ${DOXYGEN_EXECUTABLE}"
 )
 
 add_custom_target(
@@ -327,6 +282,7 @@ set(SPHINX_BUILD_HTML_COMMAND
   ${CMAKE_COMMAND} -E env
   ZEPHYR_BASE=${ZEPHYR_BASE}
   ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR}
+  DOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE}
   ${SPHINXBUILD} -W -N -t ${DOC_TAG} -b html ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_HTML})
 
 # The sphinx-html target is provided as a convenience for incremental
@@ -343,12 +299,6 @@ add_custom_target(
 )
 add_dependencies(sphinx-html content)
 
-# "breathe", the sphinx plugin that parses XML output from doxygen, has
-# an "everything on everything" dependency issue reported at:
-# https://github.com/michaeljones/breathe/issues/420 In other words
-# changing 1 source file costs the same build time than changing all
-# source files. breathe is fortunately smart enough not to run at all
-# when *nothing* has changed.
 add_custom_target(
   html
   COMMAND ${SPHINX_BUILD_HTML_COMMAND}
@@ -362,7 +312,9 @@ add_custom_target(
 #
 set(SPHINX_BUILD_LATEX_COMMAND
   ${CMAKE_COMMAND} -E env
+  ZEPHYR_BASE=${ZEPHYR_BASE}
   ZEPHYR_BUILD=${CMAKE_CURRENT_BINARY_DIR}
+  DOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE}
   ${SPHINXBUILD} -W -N -t ${DOC_TAG} -b latex -t svgconvert ${ALLSPHINXOPTS} ${RST_OUT}/doc ${SPHINX_OUTPUT_DIR_LATEX})
 
 # The sphinx-latex target works similarly to sphinx-html, and carries
@@ -428,20 +380,14 @@ endif()
 #
 # Dependencies and final targets
 #
-add_dependencies(html content doxygen_sync kconfig devicetree)
+add_dependencies(html content kconfig devicetree)
 
 add_custom_target(
   htmldocs
 )
 add_dependencies(htmldocs html)
 
-
-add_custom_target(
-  doxygen
-)
-add_dependencies(doxygen_sync doxygen)
-
-add_dependencies(latex content doxygen_sync kconfig devicetree)
+add_dependencies(latex content kconfig devicetree)
 
 add_custom_target(
   latexdocs

doc/_scripts/doxygen_sync.py

@@ -1,78 +0,0 @@
-"""
-Doxygen Sync Tool
-#################
-
-Utility to synchronize a new Doxygen build with a previous one. The aim of this
-script is to simulate incremental Doxygen builds so that tools post-processing
-Doxygen XML files can work in a more efficient manner.
-
-Copyright (c) 2021 Nordic Semiconductor ASA
-SPDX-License-Identifier: Apache-2.0
-"""
-
-import argparse
-import filecmp
-from pathlib import Path
-import shutil
-
-
-def doxygen_sync(outdir: Path, latest_outdir: Path) -> None:
-    """Synchronize Doxygen output with a previous build.
-
-    This function makes sure that only new, deleted or changed files are
-    actually modified in the Doxygen XML output. Latest HTML content is just
-    moved.
-
-    Args:
-        outdir: Doxygen output directory.
-        latest_outdir: Latest Doxygen build output directory.
-    """
-
-    htmldir = outdir / "html"
-    latest_htmldir = latest_outdir / "html"
-
-    xmldir = outdir / "xml"
-    latest_xmldir = latest_outdir / "xml"
-
-    if not latest_htmldir.exists() or not latest_xmldir.exists():
-        return
-
-    outdir.mkdir(exist_ok=True)
-
-    if htmldir.exists():
-        shutil.rmtree(htmldir)
-    latest_htmldir.rename(htmldir)
-
-    if xmldir.exists():
-        dcmp = filecmp.dircmp(latest_xmldir, xmldir)
-
-        for file in dcmp.right_only:
-            (Path(dcmp.right) / file).unlink()
-
-        for file in dcmp.left_only + dcmp.diff_files:
-            shutil.copy(Path(dcmp.left) / file, Path(dcmp.right) / file)
-
-        shutil.rmtree(latest_xmldir)
-    else:
-        latest_xmldir.rename(xmldir)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser("Doxygen Sync Tool")
-    parser.add_argument(
-        "-o",
-        dest="output",
-        type=Path,
-        required=True,
-        help="Doxygen build output path",
-    )
-    parser.add_argument(
-        "-l",
-        dest="latest_output",
-        type=Path,
-        required=True,
-        help="Latest Doxygen build output path",
-    )
-
-    args = parser.parse_args()
-    doxygen_sync(args.output, args.latest_output)

doc/conf.py

@@ -81,6 +81,7 @@ extensions = [
     "zephyr.link-roles",
     "sphinx_tabs.tabs",
     "zephyr.warnings_filter",
+    "zephyr.doxyrunner",
 ]
 
 # Only use SVG converter when it is really needed, e.g. LaTeX.
@@ -155,11 +156,19 @@ latex_documents = [
     ("index", "zephyr.tex", "Zephyr Project Documentation", "many", "manual"),
 ]
 
+# -- Options for zephyr.doxyrunner plugin ---------------------------------
+
+doxyrunner_doxygen = os.environ.get("DOXYGEN_EXECUTABLE", "doxygen")
+doxyrunner_doxyfile = ZEPHYR_BASE / "doc" / "zephyr.doxyfile.in"
+doxyrunner_outdir = ZEPHYR_BUILD / "doxygen"
+doxyrunner_fmt = True
+doxyrunner_fmt_vars = {"ZEPHYR_BASE": str(ZEPHYR_BASE)}
+
 # -- Options for Breathe plugin -------------------------------------------
 
 breathe_projects = {
-    "Zephyr": str(ZEPHYR_BUILD / "doxygen" / "xml"),
-    "doc-examples": str(ZEPHYR_BUILD / "doxygen" / "xml"),
+    "Zephyr": str(doxyrunner_outdir / "xml"),
+    "doc-examples": str(doxyrunner_outdir / "xml"),
 }
 breathe_default_project = "Zephyr"
 breathe_domain_by_extension = {