doc: extensions: add kconfig search extension

Add a new extension to handle Kconfig documentation. This means that no
more CMake hackery is required. However, the way it works differs from
the current approach. Instead of creating a single page for each Kconfig
option, the extension creates a JSON "database" which is then used on
the client side to render Kconfig options on a search page. The reason
to go for a single page choice is because Sphinx is significantly slow
when handling a lot of pages. Kconfig was responsible for an increase of
about ~10K pages.

Main features:

- Generates a Kconfig JSON database using kconfiglib APIs.
- Adds a new Sphinx domain for Kconfig. The domain provides a directive,
  :kconfig:search:: that can be used to insert a Kconfig search box onto
  any page. This page is where all Kconfig references inserted using the
  :kconfig:option: role will point to. The search functionality is
  implemented on the client side using Javascript. If the URL contains a
  hash with a Kconfig option (e.g. #CONFIG_SPI) it will load it.

Signed-off-by: Gerard Marull-Paretas <gerard.marull@nordicsemi.no>
This commit is contained in:
Gerard Marull-Paretas 2022-01-12 13:31:47 +01:00 committed by Carles Cufí
parent d2a56c5047
commit 8bdeac62bb
3 changed files with 863 additions and 0 deletions

View file

@ -0,0 +1,398 @@
"""
Kconfig Extension
#################
Copyright (c) 2022 Nordic Semiconductor ASA
SPDX-License-Identifier: Apache-2.0
Introduction
============
This extension adds a new domain (``kconfig``) for the Kconfig language. Unlike
many other domains, the Kconfig options are not rendered by Sphinx directly but
on the client side using a database built by the extension. A special directive
``.. kconfig:search::`` can be inserted on any page to render a search box that
allows to browse the database. References to Kconfig options can be created by
using the ``:kconfig:option:`` role. Kconfig options behave as regular domain
objects, so they can also be referenced by other projects using Intersphinx.
Options
=======
- kconfig_generate_db: Set to True if you want to generate the Kconfig database.
This is only required if you want to use the ``.. kconfig:search::``
directive, not if you just need support for Kconfig domain (e.g. when using
Intersphinx in another project). Defaults to False.
- kconfig_ext_paths: A list of base paths where to search for external modules
Kconfig files when they use ``kconfig-ext: True``. The extension will look for
${BASE_PATH}/modules/${MODULE_NAME}/Kconfig.
"""
from distutils.command.build import build
from itertools import chain
import json
from operator import mod
import os
from pathlib import Path
import re
import sys
from tempfile import TemporaryDirectory
from typing import Any, Dict, Iterable, List, Optional, Tuple
from docutils import nodes
from sphinx.addnodes import pending_xref
from sphinx.application import Sphinx
from sphinx.builders import Builder
from sphinx.domains import Domain, ObjType
from sphinx.environment import BuildEnvironment
from sphinx.errors import ExtensionError
from sphinx.roles import XRefRole
from sphinx.util import progress_message
from sphinx.util.docutils import SphinxDirective
from sphinx.util.nodes import make_refnode
__version__ = "0.1.0"
RESOURCES_DIR = Path(__file__).parent / "static"
ZEPHYR_BASE = Path(__file__).parents[4]
SCRIPTS = ZEPHYR_BASE / "scripts"
sys.path.insert(0, str(SCRIPTS))
KCONFIGLIB = SCRIPTS / "kconfig"
sys.path.insert(0, str(KCONFIGLIB))
import zephyr_module
import kconfiglib
def kconfig_load(app: Sphinx) -> Tuple[kconfiglib.Kconfig, Dict[str, str]]:
"""Load Kconfig"""
with TemporaryDirectory() as td:
projects = zephyr_module.west_projects()
projects = [p.posixpath for p in projects["projects"]] if projects else None
modules = zephyr_module.parse_modules(ZEPHYR_BASE, projects)
# generate Kconfig.modules file
kconfig = ""
for module in modules:
kconfig += zephyr_module.process_kconfig(module.project, module.meta)
with open(Path(td) / "Kconfig.modules", "w") as f:
f.write(kconfig)
# base environment
os.environ["ZEPHYR_BASE"] = str(ZEPHYR_BASE)
os.environ["srctree"] = str(ZEPHYR_BASE)
os.environ["KCONFIG_DOC_MODE"] = "1"
os.environ["KCONFIG_BINARY_DIR"] = td
# include all archs and boards
os.environ["ARCH_DIR"] = "arch"
os.environ["ARCH"] = "*"
os.environ["BOARD_DIR"] = "boards/*/*"
# insert external Kconfigs to the environment
module_paths = dict()
for module in modules:
name = module.meta["name"]
name_var = module.meta["name-sanitized"].upper()
module_paths[name] = module.project
build_conf = module.meta.get("build")
if not build_conf:
continue
if build_conf.get("kconfig"):
kconfig = Path(module.project) / build_conf["kconfig"]
os.environ[f"ZEPHYR_{name_var}_KCONFIG"] = str(kconfig)
elif build_conf.get("kconfig-ext"):
for path in app.config.kconfig_ext_paths:
kconfig = Path(path) / "modules" / name / "Kconfig"
if kconfig.exists():
os.environ[f"ZEPHYR_{name_var}_KCONFIG"] = str(kconfig)
return kconfiglib.Kconfig(ZEPHYR_BASE / "Kconfig"), module_paths
class KconfigSearchNode(nodes.Element):
@staticmethod
def html():
return '<div id="__kconfig-search"></div>'
def kconfig_search_visit_html(self, node: nodes.Node) -> None:
self.body.append(node.html())
raise nodes.SkipNode
def kconfig_search_visit_latex(self, node: nodes.Node) -> None:
self.body.append("Kconfig search is only available on HTML output")
raise nodes.SkipNode
class KconfigSearch(SphinxDirective):
"""Kconfig search directive"""
has_content = False
def run(self):
if not self.config.kconfig_generate_db:
raise ExtensionError(
"Kconfig search directive can not be used without database"
)
if "kconfig_search_inserted" in self.env.temp_data:
raise ExtensionError("Kconfig search directive can only be used once")
self.env.temp_data["kconfig_search_inserted"] = True
# register all options to the domain at this point, so that they all
# resolve to the page where the kconfig:search directive is inserted
domain = self.env.get_domain("kconfig")
unique = set({option["name"] for option in self.env.kconfig_db})
for option in unique:
domain.add_option(option)
return [KconfigSearchNode()]
class _FindKconfigSearchDirectiveVisitor(nodes.NodeVisitor):
def __init__(self, document):
super().__init__(document)
self._found = False
def unknown_visit(self, node: nodes.Node) -> None:
if self._found:
return
self._found = isinstance(node, KconfigSearchNode)
@property
def found_kconfig_search_directive(self) -> bool:
return self._found
class KconfigDomain(Domain):
"""Kconfig domain"""
name = "kconfig"
label = "Kconfig"
object_types = {"option": ObjType("option", "option")}
roles = {"option": XRefRole()}
directives = {"search": KconfigSearch}
initial_data: Dict[str, Any] = {"options": []}
def get_objects(self) -> Iterable[Tuple[str, str, str, str, str, int]]:
for obj in self.data["options"]:
yield obj
def merge_domaindata(self, docnames: List[str], otherdata: Dict) -> None:
self.data["options"] += otherdata["options"]
def resolve_xref(
self,
env: BuildEnvironment,
fromdocname: str,
builder: Builder,
typ: str,
target: str,
node: pending_xref,
contnode: nodes.Element,
) -> Optional[nodes.Element]:
match = [
(docname, anchor)
for name, _, _, docname, anchor, _ in self.get_objects()
if name == target
]
if match:
todocname, anchor = match[0]
return make_refnode(
builder, fromdocname, todocname, anchor, contnode, anchor
)
else:
return None
def add_option(self, option):
"""Register a new Kconfig option to the domain."""
self.data["options"].append(
(option, option, "option", self.env.docname, option, -1)
)
def sc_fmt(sc):
if isinstance(sc, kconfiglib.Symbol):
if sc.nodes:
return f'<a href="#CONFIG_{sc.name}">CONFIG_{sc.name}</a>'
elif isinstance(sc, kconfiglib.Choice):
if not sc.name:
return "&ltchoice&gt"
return f'&ltchoice <a href="#CONFIG_{sc.name}">CONFIG_{sc.name}</a>&gt'
return kconfiglib.standard_sc_expr_str(sc)
def kconfig_build_resources(app: Sphinx) -> None:
"""Build the Kconfig database and install HTML resources."""
if not app.config.kconfig_generate_db:
return
with progress_message("Building Kconfig database..."):
kconfig, module_paths = kconfig_load(app)
db = list()
for sc in chain(kconfig.unique_defined_syms, kconfig.unique_choices):
# skip nameless symbols
if not sc.name:
continue
# store alternative defaults (from defconfig files)
alt_defaults = list()
for node in sc.nodes:
if "defconfig" not in node.filename:
continue
for value, cond in node.orig_defaults:
fmt = kconfiglib.expr_str(value, sc_fmt)
if cond is not sc.kconfig.y:
fmt += f" if {kconfiglib.expr_str(cond, sc_fmt)}"
alt_defaults.append([fmt, node.filename])
# only process nodes with prompt or help
nodes = [node for node in sc.nodes if node.prompt or node.help]
inserted_paths = list()
for node in nodes:
# avoid duplicate symbols by forcing unique paths. this can
# happen due to dependencies on 0, a trick used by some modules
path = f"{node.filename}:{node.linenr}"
if path in inserted_paths:
continue
inserted_paths.append(path)
dependencies = None
if node.dep is not sc.kconfig.y:
dependencies = kconfiglib.expr_str(node.dep, sc_fmt)
defaults = list()
for value, cond in node.orig_defaults:
fmt = kconfiglib.expr_str(value, sc_fmt)
if cond is not sc.kconfig.y:
fmt += f" if {kconfiglib.expr_str(cond, sc_fmt)}"
defaults.append(fmt)
selects = list()
for value, cond in node.orig_selects:
fmt = kconfiglib.expr_str(value, sc_fmt)
if cond is not sc.kconfig.y:
fmt += f" if {kconfiglib.expr_str(cond, sc_fmt)}"
selects.append(fmt)
implies = list()
for value, cond in node.orig_implies:
fmt = kconfiglib.expr_str(value, sc_fmt)
if cond is not sc.kconfig.y:
fmt += f" if {kconfiglib.expr_str(cond, sc_fmt)}"
implies.append(fmt)
ranges = list()
for min, max, cond in node.orig_ranges:
fmt = (
f"[{kconfiglib.expr_str(min, sc_fmt)}, "
f"{kconfiglib.expr_str(max, sc_fmt)}]"
)
if cond is not sc.kconfig.y:
fmt += f" if {kconfiglib.expr_str(cond, sc_fmt)}"
ranges.append(fmt)
choices = list()
if isinstance(sc, kconfiglib.Choice):
for sym in sc.syms:
choices.append(kconfiglib.expr_str(sym, sc_fmt))
filename = node.filename
for name, path in module_paths.items():
if node.filename.startswith(path):
filename = node.filename.replace(path, f"<module:{name}>")
break
db.append(
{
"name": f"CONFIG_{sc.name}",
"prompt": node.prompt[0] if node.prompt else None,
"type": kconfiglib.TYPE_TO_STR[sc.type],
"help": node.help,
"dependencies": dependencies,
"defaults": defaults,
"alt_defaults": alt_defaults,
"selects": selects,
"implies": implies,
"ranges": ranges,
"choices": choices,
"filename": filename,
"linenr": node.linenr,
}
)
app.env.kconfig_db = db # type: ignore
outdir = Path(app.outdir) / "kconfig"
outdir.mkdir(exist_ok=True)
kconfig_db_file = outdir / "kconfig.json"
with open(kconfig_db_file, "w") as f:
json.dump(db, f)
app.config.html_extra_path.append(kconfig_db_file.as_posix())
app.config.html_static_path.append(RESOURCES_DIR.as_posix())
def kconfig_install(
app: Sphinx,
pagename: str,
templatename: str,
context: Dict,
doctree: Optional[nodes.Node],
) -> None:
"""Install the Kconfig library files on pages that require it."""
if (
not app.config.kconfig_generate_db
or app.builder.format != "html"
or not doctree
):
return
visitor = _FindKconfigSearchDirectiveVisitor(doctree)
doctree.walk(visitor)
if visitor.found_kconfig_search_directive:
app.add_css_file("kconfig.css")
app.add_js_file("kconfig.mjs", type="module")
def setup(app: Sphinx):
app.add_config_value("kconfig_generate_db", False, "env")
app.add_config_value("kconfig_ext_paths", [], "env")
app.add_node(
KconfigSearchNode,
html=(kconfig_search_visit_html, None),
latex=(kconfig_search_visit_latex, None),
)
app.add_domain(KconfigDomain)
app.connect("builder-inited", kconfig_build_resources)
app.connect("html-page-context", kconfig_install)
return {
"version": __version__,
"parallel_read_safe": True,
"parallel_write_safe": True,
}

View file

@ -0,0 +1,37 @@
/*
* Copyright (c) 2022 Nordic Semiconductor ASA
* SPDX-License-Identifier: Apache-2.0
*/
/* Kconfig search */
#__kconfig-search input {
border-radius: 5px;
border: 1px solid rgba(149, 157, 165, 0.2);
box-shadow: rgba(149, 157, 165, 0.2) 0px 8px 24px !important;
font-size: 18px;
margin-bottom: 0.5rem;
padding: 0.75rem;
width: 100%;
}
#__kconfig-search .search-summary {
margin: 0.25rem 0.1rem 1.5rem;
}
#__kconfig-search .search-nav {
display: flex;
justify-content: center;
align-items: center;
}
#__kconfig-search .search-nav > p {
padding: 0 1rem;
margin: 0;
}
/* Kconfig entries */
.kconfig ul {
margin-bottom: 0 !important;
}

View file

@ -0,0 +1,428 @@
/*
* Copyright (c) 2022 Nordic Semiconductor ASA
* SPDX-License-Identifier: Apache-2.0
*/
const DB_FILE = 'kconfig.json';
const MAX_RESULTS = 10;
/* search state */
let db;
let searchOffset;
/* elements */
let input;
let summaryText;
let results;
let navigation;
let navigationPagesText;
let navigationPrev;
let navigationNext;
/**
* Show an error message.
* @param {String} message Error message.
*/
function showError(message) {
const admonition = document.createElement('div');
admonition.className = 'admonition error';
results.replaceChildren(admonition);
const admonitionTitle = document.createElement('p');
admonitionTitle.className = 'admonition-title';
admonition.appendChild(admonitionTitle);
const admonitionTitleText = document.createTextNode('Error');
admonitionTitle.appendChild(admonitionTitleText);
const admonitionContent = document.createElement('p');
admonition.appendChild(admonitionContent);
const admonitionContentText = document.createTextNode(message);
admonitionContent.appendChild(admonitionContentText);
}
/**
* Show a progress message.
* @param {String} message Progress message.
*/
function showProgress(message) {
const p = document.createElement('p');
p.className = 'centered';
results.replaceChildren(p);
const pText = document.createTextNode(message);
p.appendChild(pText);
}
/**
* Render a Kconfig literal property.
* @param {Element} parent Parent element.
* @param {String} title Title.
* @param {String} content Content.
*/
function renderKconfigPropLiteral(parent, title, content) {
const term = document.createElement('dt');
parent.appendChild(term);
const termText = document.createTextNode(title);
term.appendChild(termText);
const details = document.createElement('dd');
parent.appendChild(details);
const code = document.createElement('code');
code.className = 'docutils literal';
details.appendChild(code);
const literal = document.createElement('span');
literal.className = 'pre';
code.appendChild(literal);
const literalText = document.createTextNode(content);
literal.appendChild(literalText);
}
/**
* Render a Kconfig list property.
* @param {Element} parent Parent element.
* @param {String} title Title.
* @param {list} elements List of elements.
* @returns
*/
function renderKconfigPropList(parent, title, elements) {
if (elements.length === 0) {
return;
}
const term = document.createElement('dt');
parent.appendChild(term);
const termText = document.createTextNode(title);
term.appendChild(termText);
const details = document.createElement('dd');
parent.appendChild(details);
const list = document.createElement('ul');
list.className = 'simple';
details.appendChild(list);
elements.forEach(element => {
const listItem = document.createElement('li');
list.appendChild(listItem);
/* using HTML since element content may be pre-formatted */
listItem.innerHTML = element;
});
}
/**
* Render a Kconfig list property.
* @param {Element} parent Parent element.
* @param {list} elements List of elements.
* @returns
*/
function renderKconfigDefaults(parent, defaults, alt_defaults) {
if (defaults.length === 0 && alt_defaults.length === 0) {
return;
}
const term = document.createElement('dt');
parent.appendChild(term);
const termText = document.createTextNode('Defaults');
term.appendChild(termText);
const details = document.createElement('dd');
parent.appendChild(details);
if (defaults.length > 0) {
const list = document.createElement('ul');
list.className = 'simple';
details.appendChild(list);
defaults.forEach(entry => {
const listItem = document.createElement('li');
list.appendChild(listItem);
/* using HTML since default content may be pre-formatted */
listItem.innerHTML = entry;
});
}
if (alt_defaults.length > 0) {
const list = document.createElement('ul');
list.className = 'simple';
list.style.display = 'none';
details.appendChild(list);
alt_defaults.forEach(entry => {
const listItem = document.createElement('li');
list.appendChild(listItem);
/* using HTML since default content may be pre-formatted */
listItem.innerHTML = `
${entry[0]}
<em>at</em>
<code class="docutils literal">
<span class"pre">${entry[1]}</span>
</code>`;
});
const show = document.createElement('a');
show.onclick = () => {
if (list.style.display === 'none') {
list.style.display = 'block';
} else {
list.style.display = 'none';
}
};
details.appendChild(show);
const showText = document.createTextNode('Show/Hide other defaults');
show.appendChild(showText);
}
}
/**
* Render a Kconfig entry.
* @param {Object} entry Kconfig entry.
*/
function renderKconfigEntry(entry) {
const container = document.createElement('dl');
container.className = 'kconfig';
/* title (name and permalink) */
const title = document.createElement('dt');
title.className = 'sig sig-object';
container.appendChild(title);
const name = document.createElement('span');
name.className = 'pre';
title.appendChild(name);
const nameText = document.createTextNode(entry.name);
name.appendChild(nameText);
const permalink = document.createElement('a');
permalink.className = 'headerlink';
permalink.href = '#' + entry.name;
title.appendChild(permalink);
const permalinkText = document.createTextNode('\uf0c1');
permalink.appendChild(permalinkText);
/* details */
const details = document.createElement('dd');
container.append(details);
/* prompt and help */
const prompt = document.createElement('p');
details.appendChild(prompt);
const promptTitle = document.createElement('em');
prompt.appendChild(promptTitle);
const promptTitleText = document.createTextNode('');
promptTitle.appendChild(promptTitleText);
if (entry.prompt) {
promptTitleText.nodeValue = entry.prompt;
} else {
promptTitleText.nodeValue = 'No prompt - not directly user assignable.';
}
if (entry.help) {
const help = document.createElement('p');
details.appendChild(help);
const helpText = document.createTextNode(entry.help);
help.appendChild(helpText);
}
/* symbol properties (defaults, selects, etc.) */
const props = document.createElement('dl');
props.className = 'field-list simple';
details.appendChild(props);
renderKconfigPropLiteral(props, 'Type', entry.type);
if (entry.dependencies) {
renderKconfigPropList(props, 'Dependencies', [entry.dependencies]);
}
renderKconfigDefaults(props, entry.defaults, entry.alt_defaults);
renderKconfigPropList(props, 'Selects', entry.selects);
renderKconfigPropList(props, 'Implies', entry.implies);
renderKconfigPropList(props, 'Ranges', entry.ranges);
renderKconfigPropList(props, 'Choices', entry.choices);
renderKconfigPropLiteral(props, 'Location', `${entry.filename}:${entry.linenr}`);
return container;
}
/** Perform a search and display the results. */
function doSearch() {
/* replace current state (to handle back button) */
history.replaceState({
value: input.value,
searchOffset: searchOffset
}, '', window.location);
/* nothing to search for */
if (!input.value) {
summaryText.nodeValue = '';
results.replaceChildren();
navigation.style.visibility = 'hidden';
return;
}
/* perform search */
let pattern = new RegExp(input.value, 'i');
let count = 0;
const searchResults = db.filter(entry => {
if (entry.name.match(pattern)) {
count++;
if (count > searchOffset && count <= (searchOffset + MAX_RESULTS)) {
return true;
}
}
return false;
});
/* show results count */
summaryText.nodeValue = `${count} options match your search.`;
/* update navigation */
navigation.style.visibility = 'visible';
navigationPrev.disabled = searchOffset - MAX_RESULTS < 0;
navigationNext.disabled = searchOffset + MAX_RESULTS > count;
const currentPage = Math.floor(searchOffset / MAX_RESULTS) + 1;
const totalPages = Math.floor(count / MAX_RESULTS) + 1;
navigationPagesText.nodeValue = `Page ${currentPage} of ${totalPages}`;
/* render Kconfig entries */
results.replaceChildren();
searchResults.forEach(entry => {
results.appendChild(renderKconfigEntry(entry));
});
}
/** Do a search from URL hash */
function doSearchFromURL() {
const rawOption = window.location.hash.substring(1);
if (!rawOption) {
return;
}
const option = rawOption.replace(/[^A-Za-z0-9_]+/g, '');
input.value = '^' + option + '$';
searchOffset = 0;
doSearch();
}
function setupKconfigSearch() {
/* populate kconfig-search container */
const container = document.getElementById('__kconfig-search');
if (!container) {
console.error("Couldn't find Kconfig search container");
return;
}
/* create input field */
input = document.createElement('input');
input.placeholder = 'Type a Kconfig option name (RegEx allowed)';
input.type = 'text';
container.appendChild(input);
/* create search summary */
const searchSummary = document.createElement('p');
searchSummary.className = 'search-summary';
container.appendChild(searchSummary);
summaryText = document.createTextNode('');
searchSummary.appendChild(summaryText);
/* create search results container */
results = document.createElement('div');
container.appendChild(results);
/* create search navigation */
navigation = document.createElement('div');
navigation.className = 'search-nav';
navigation.style.visibility = 'hidden';
container.appendChild(navigation);
navigationPrev = document.createElement('button');
navigationPrev.className = 'btn';
navigationPrev.disabled = true;
navigationPrev.onclick = () => {
searchOffset -= MAX_RESULTS;
doSearch();
window.scroll(0, 0);
}
navigation.appendChild(navigationPrev);
const navigationPrevText = document.createTextNode('Previous');
navigationPrev.appendChild(navigationPrevText);
const navigationPages = document.createElement('p');
navigation.appendChild(navigationPages);
navigationPagesText = document.createTextNode('');
navigationPages.appendChild(navigationPagesText);
navigationNext = document.createElement('button');
navigationNext.className = 'btn';
navigationNext.disabled = true;
navigationNext.onclick = () => {
searchOffset += MAX_RESULTS;
doSearch();
window.scroll(0, 0);
}
navigation.appendChild(navigationNext);
const navigationNextText = document.createTextNode('Next');
navigationNext.appendChild(navigationNextText);
/* load database */
showProgress('Loading database...');
fetch(DB_FILE)
.then(response => response.json())
.then(json => {
db = json;
results.replaceChildren();
/* perform initial search */
doSearchFromURL();
/* install event listeners */
input.addEventListener('keyup', () => {
searchOffset = 0;
doSearch();
});
/* install hash change listener (for links) */
window.addEventListener('hashchange', doSearchFromURL);
/* handle back/forward navigation */
window.addEventListener('popstate', (event) => {
if (!event.state) {
return;
}
input.value = event.state.value;
searchOffset = event.state.searchOffset;
doSearch();
});
})
.catch(error => {
showError(`Kconfig database could not be loaded (${error})`);
});
}
setupKconfigSearch();