dts: separate DT libraries from gen_defines.py

We are now in the process of extracting edtlib and dtlib into a
standalone source code library that we intend to share with other
projects.

Links related to the work making this standalone:

    https://pypi.org/project/devicetree/
    https://python-devicetree.readthedocs.io/en/latest/
    https://github.com/zephyrproject-rtos/python-devicetree

This standalone repo includes the same features as what we have in
Zephyr, but in its own 'devicetree' python package with PyPI
integration, etc.

To avoid making this a hard fork, move the code that's being made
standalone around in Zephyr into a new scripts/dts/python-devicetree
subdirectory, and handle the package and sys.path changes in the
various places in the tree that use it.

From now on, it will be possible to update the standalone repository
by just recursively copying scripts/dts/python-devicetree's contents
into it and committing the results.

This is an interim step; do NOT 'pip install devicetree' yet.
The code in the zephyr repository is still the canonical location.

(In the long term, people will get the devicetree package from PyPI
just like they do the 'yaml' package today, but that won't happen for
the foreseeable future.)

This commit is purely intended to avoid a hard fork for the standalone
code, and no functional changes besides the package structure and
location of the code itself are expected.

Signed-off-by: Martí Bolívar <marti.bolivar@nordicsemi.no>

.github/workflows/devicetree_checks.yml

@@ -57,8 +57,8 @@ jobs:
     - name: install python dependencies
       run: |
         pip3 install wheel
-        pip3 install pytest pyyaml
-    - name: run pytest
-      working-directory: scripts/dts
+        pip3 install pytest pyyaml tox
+    - name: run tox
+      working-directory: scripts/dts/python-devicetree
       run: |
-        python -m pytest testdtlib.py testedtlib.py
+        tox

doc/CMakeLists.txt

@@ -309,7 +309,7 @@ endif()
 add_custom_target(
   devicetree
   COMMAND ${CMAKE_COMMAND} -E env
-  PYTHONPATH=${ZEPHYR_BASE}/scripts/dts${SEP}$ENV{PYTHONPATH}
+  PYTHONPATH=${ZEPHYR_BASE}/scripts/dts/python-devicetree/src${SEP}$ENV{PYTHONPATH}
   ZEPHYR_BASE=${ZEPHYR_BASE}
   GEN_DEVICETREE_REST_ZEPHYR_DOCSET=${GEN_DEVICETREE_REST_ZEPHYR_DOCSET}
   ${PYTHON_EXECUTABLE} ${GEN_DEVICETREE_REST_SCRIPT}

doc/_scripts/gen_devicetree_rest.py

@@ -18,7 +18,7 @@ import re
 import sys
 import textwrap
 
-import edtlib
+from devicetree import edtlib
 
 import gen_helpers
 

scripts/dts/README.txt

@@ -0,0 +1,32 @@
+This directory used to contain the edtlib.py and dtlib.py libraries
+and tests, alongside the gen_defines.py script that uses them for
+converting DTS to the C macros used by Zephyr.
+
+The libraries and tests have now been moved to the 'python-devicetree'
+subdirectory.
+
+We are now in the process of extracting edtlib and dtlib into a
+standalone source code library that we intend to share with other
+projects.
+
+Links related to the work making this standalone:
+
+    https://pypi.org/project/devicetree/
+    https://python-devicetree.readthedocs.io/en/latest/
+    https://github.com/zephyrproject-rtos/python-devicetree
+
+The 'python-devicetree' subdirectory you find here next to this
+README.txt matches the standalone python-devicetree repository linked
+above.
+
+For now, the 'main' copy will continue to be hosted here in the zephyr
+repository. We will mirror changes into the standalone repository as
+needed; you can just ignore it for now.
+
+Code in the zephyr repository which needs these libraries will import
+devicetree.edtlib from now on, but the code will continue to be found
+by manipulating sys.path for now.
+
+Eventually, as APIs stabilize, the python-devicetree code in this
+repository will disappear, and a standalone repository will be the
+'main' one.

scripts/dts/gen_defines.py

@@ -27,7 +27,10 @@ import pickle
 import re
 import sys
 
-import edtlib
+sys.path.append(os.path.join(os.path.dirname(__file__), 'python-devicetree',
+                             'src'))
+
+from devicetree import edtlib
 
 class LogFormatter(logging.Formatter):
     '''A log formatter that prints the level name in lower case,

scripts/dts/python-devicetree/.gitignore

@@ -0,0 +1,7 @@
+dist/
+src/devicetree.egg-info/
+build/
+devicetree.egg-info/
+__pycache__/
+.tox/
+doc/build/

scripts/dts/python-devicetree/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme                # docs

scripts/dts/python-devicetree/setup.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2021, Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import setuptools
+
+long_description = '''
+Placeholder
+===========
+
+This is just a placeholder for moving Zephyr's devicetree libraries
+to PyPI.
+'''
+
+version = '0.0.1'
+
+setuptools.setup(
+    # TBD, just use these for now.
+    author='Zephyr Project',
+    author_email='devel@lists.zephyrproject.org',
+
+    name='devicetree',
+    version=version,
+    description='Python libraries for devicetree',
+    long_description=long_description,
+    # http://docutils.sourceforge.net/FAQ.html#what-s-the-official-mime-type-for-restructuredtext-data
+    long_description_content_type="text/x-rst",
+    url='https://github.com/zephyrproject-rtos/python-devicetree',
+    packages=setuptools.find_packages(where='src'),
+    package_dir={'': 'src'},
+    classifiers=[
+        'Programming Language :: Python :: 3 :: Only',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Operating System :: MacOS :: MacOS X',
+        'Operating System :: Microsoft :: Windows',
+    ],
+    install_requires=[
+        'PyYAML>=5.1',
+    ],
+    python_requires='>=3.6',
+)

scripts/dts/python-devicetree/src/devicetree/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2021 Nordic Semiconductor ASA
+# SPDX-License-Identifier: Apache-2.0
+
+__all__ = ['edtlib', 'dtlib']

scripts/dts/python-devicetree/src/devicetree/edtlib.py

@@ -80,10 +80,11 @@ try:
 except ImportError:
     from yaml import Loader
 
-from dtlib import DT, DTError, to_num, to_nums, TYPE_EMPTY, TYPE_BYTES, \
-                  TYPE_NUM, TYPE_NUMS, TYPE_STRING, TYPE_STRINGS, \
-                  TYPE_PHANDLE, TYPE_PHANDLES, TYPE_PHANDLES_AND_NUMS
-from grutils import Graph
+from devicetree.dtlib import \
+    DT, DTError, to_num, to_nums, TYPE_EMPTY, TYPE_BYTES, \
+    TYPE_NUM, TYPE_NUMS, TYPE_STRING, TYPE_STRINGS, \
+    TYPE_PHANDLE, TYPE_PHANDLES, TYPE_PHANDLES_AND_NUMS
+from devicetree.grutils import Graph
 
 
 #

scripts/dts/python-devicetree/tests/test_dtlib.py

@@ -8,13 +8,13 @@ import tempfile
 
 import pytest
 
-import dtlib
+from devicetree import dtlib
 
 # Test suite for dtlib.py.
 #
 # Run it using pytest (https://docs.pytest.org/en/stable/usage.html):
 #
-#   $ pytest testdtlib.py
+#   $ pytest tests/test_dtlib.py
 #
 # Extra options you can pass to pytest for debugging:
 #

scripts/dts/python-devicetree/tests/test_edtlib.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2019 Nordic Semiconductor ASA
 # SPDX-License-Identifier: BSD-3-Clause
 
+import contextlib
 import io
 from logging import WARNING
 import os
@@ -8,7 +9,7 @@ from pathlib import Path
 
 import pytest
 
-import edtlib
+from devicetree import edtlib
 
 # Test suite for edtlib.py.
 #
@@ -22,6 +23,18 @@ import edtlib
 # bindings. The tests mostly use string comparisons via the various __repr__()
 # methods.
 
+HERE = os.path.dirname(__file__)
+
+@contextlib.contextmanager
+def from_here():
+    # Convenience hack to minimize diff from zephyr.
+    cwd = os.getcwd()
+    try:
+        os.chdir(HERE)
+        yield
+    finally:
+        os.chdir(cwd)
+
 def hpath(filename):
     '''Convert 'filename' to the host path syntax.'''
     return os.fspath(Path(filename))
@@ -29,7 +42,7 @@ def hpath(filename):
 def test_warnings(caplog):
     '''Tests for situations that should cause warnings.'''
 
-    edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here(): edtlib.EDT("test.dts", ["test-bindings"])
 
     enums_hpath = hpath('test-bindings/enums.yaml')
     expected_warnings = [
@@ -40,12 +53,13 @@ def test_warnings(caplog):
         f"compatible 'enums' in binding '{enums_hpath}' has non-tokenizable enum for property 'string-enum': 'foo bar', 'foo_bar'",
         f"compatible 'enums' in binding '{enums_hpath}' has enum for property 'tokenizable-lower-enum' that is only tokenizable in lowercase: 'bar', 'BAR'",
     ]
-    assert caplog.record_tuples == [('edtlib', WARNING, warning_message)
+    assert caplog.record_tuples == [('devicetree.edtlib', WARNING, warning_message)
                                     for warning_message in expected_warnings]
 
 def test_interrupts():
     '''Tests for the interrupts property.'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
     filenames = {i: hpath(f'test-bindings/interrupt-{i}-cell.yaml')
                  for i in range(1, 4)}
 
@@ -66,7 +80,8 @@ def test_interrupts():
 
 def test_reg():
     '''Tests for the regs property'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert str(edt.get_node("/reg-zero-address-cells/node").regs) == \
         "[<Register, size: 0x1>, <Register, size: 0x2>]"
@@ -82,14 +97,16 @@ def test_reg():
 
 def test_pinctrl():
     '''Test 'pinctrl-<index>'.'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert str(edt.get_node("/pinctrl/dev").pinctrls) == \
         "[<PinCtrl, name: zero, configuration nodes: []>, <PinCtrl, name: one, configuration nodes: [<Node /pinctrl/pincontroller/state-1 in 'test.dts', no binding>]>, <PinCtrl, name: two, configuration nodes: [<Node /pinctrl/pincontroller/state-1 in 'test.dts', no binding>, <Node /pinctrl/pincontroller/state-2 in 'test.dts', no binding>]>]"
 
 def test_hierarchy():
     '''Test Node.parent and Node.children'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert edt.get_node("/").parent is None
 
@@ -106,7 +123,8 @@ def test_hierarchy():
 
 def test_include():
     '''Test 'include:' and the legacy 'inherits: !include ...' in bindings'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert str(edt.get_node("/binding-include").description) == \
         "Parent binding"
@@ -116,7 +134,8 @@ def test_include():
 
 def test_bus():
     '''Test 'bus:' and 'on-bus:' in bindings'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert edt.get_node("/buses/foo-bus").bus == "foo"
 
@@ -159,7 +178,8 @@ def test_bus():
 
 def test_child_binding():
     '''Test 'child-binding:' in bindings'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
     child1 = edt.get_node("/child-binding/child-1")
     child2 = edt.get_node("/child-binding/child-2")
     grandchild = edt.get_node("/child-binding/child-1/grandchild")
@@ -176,16 +196,18 @@ def test_child_binding():
     assert str(grandchild.description) == "grandchild node"
     assert str(grandchild.props) == "OrderedDict([('grandchild-prop', <Property, name: grandchild-prop, type: int, value: 2>)])"
 
-    binding_file = Path("test-bindings/child-binding.yaml").resolve()
-    top = edtlib.Binding(binding_file, {})
+    with from_here():
+        binding_file = Path("test-bindings/child-binding.yaml").resolve()
+        top = edtlib.Binding(binding_file, {})
     child = top.child_binding
     assert Path(top.path) == binding_file
     assert Path(child.path) == binding_file
     assert top.compatible == 'top-binding'
     assert child.compatible is None
 
-    binding_file = Path("test-bindings/child-binding-with-compat.yaml").resolve()
-    top = edtlib.Binding(binding_file, {})
+    with from_here():
+        binding_file = Path("test-bindings/child-binding-with-compat.yaml").resolve()
+        top = edtlib.Binding(binding_file, {})
     child = top.child_binding
     assert Path(top.path) == binding_file
     assert Path(child.path) == binding_file
@@ -194,7 +216,8 @@ def test_child_binding():
 
 def test_props():
     '''Test Node.props (derived from DT and 'properties:' in the binding)'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
     filenames = {i: hpath(f'test-bindings/phandle-array-controller-{i}.yaml')
                  for i in range(0, 4)}
 
@@ -242,7 +265,8 @@ def test_props():
 
 def test_nexus():
     '''Test <prefix>-map via gpio-map (the most common case).'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
     filename = hpath('test-bindings/gpio-dst.yaml')
 
     assert str(edt.get_node("/gpio-map/source").props["foo-gpios"]) == \
@@ -250,7 +274,8 @@ def test_nexus():
 
 def test_prop_defaults():
     '''Test property default values given in bindings'''
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
 
     assert str(edt.get_node("/defaults").props) == \
         r"OrderedDict([('int', <Property, name: int, type: int, value: 123>), ('array', <Property, name: array, type: array, value: [1, 2, 3]>), ('uint8-array', <Property, name: uint8-array, type: uint8-array, value: b'\x89\xab\xcd'>), ('string', <Property, name: string, type: string, value: 'hello'>), ('string-array', <Property, name: string-array, type: string-array, value: ['hello', 'there']>), ('default-not-used', <Property, name: default-not-used, type: int, value: 234>)])"
@@ -258,7 +283,8 @@ def test_prop_defaults():
 def test_prop_enums():
     '''test properties with enum: in the binding'''
 
-    edt = edtlib.EDT("test.dts", ["test-bindings"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"])
     props = edt.get_node('/enums').props
     int_enum = props['int-enum']
     string_enum = props['string-enum']
@@ -295,12 +321,14 @@ def test_prop_enums():
 def test_binding_inference():
     '''Test inferred bindings for special zephyr-specific nodes.'''
     warnings = io.StringIO()
-    edt = edtlib.EDT("test.dts", ["test-bindings"], warnings)
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"], warnings)
 
     assert str(edt.get_node("/zephyr,user").props) == r"OrderedDict()"
 
-    edt = edtlib.EDT("test.dts", ["test-bindings"], warnings,
-                     infer_binding_for_paths=["/zephyr,user"])
+    with from_here():
+        edt = edtlib.EDT("test.dts", ["test-bindings"], warnings,
+                         infer_binding_for_paths=["/zephyr,user"])
     filenames = {i: hpath(f'test-bindings/phandle-array-controller-{i}.yaml')
                  for i in range(1, 3)}
 
@@ -309,7 +337,8 @@ def test_binding_inference():
 
 def test_multi_bindings():
     '''Test having multiple directories with bindings'''
-    edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"])
+    with from_here():
+        edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"])
 
     assert str(edt.get_node("/in-dir-1").binding_path) == \
         hpath("test-bindings/multidir.yaml")
@@ -319,7 +348,8 @@ def test_multi_bindings():
 
 def test_dependencies():
     ''''Test dependency relations'''
-    edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"])
+    with from_here():
+        edt = edtlib.EDT("test-multidir.dts", ["test-bindings", "test-bindings-2"])
 
     assert edt.get_node("/").dep_ordinal == 0
     assert edt.get_node("/in-dir-1").dep_ordinal == 1

scripts/dts/python-devicetree/tox.ini

@@ -0,0 +1,11 @@
+[tox]
+envlist=py3
+
+[testenv]
+deps =
+    setuptools-scm
+    pytest
+setenv =
+    TOXTEMPDIR={envtmpdir}
+commands =
+  python -m pytest {posargs:tests}

scripts/gen_handles.py

@@ -38,6 +38,11 @@ from elftools.elf.elffile import ELFFile
 from elftools.elf.sections import SymbolTableSection
 import elftools.elf.enums
 
+# This is needed to load edt.pickle files.
+sys.path.append(os.path.join(os.path.dirname(__file__),
+                             'dts', 'python-devicetree', 'src'))
+from devicetree import edtlib  # pylint: disable=unused-import
+
 if LooseVersion(elftools.__version__) < LooseVersion('0.24'):
     sys.exit("pyelftools is out of date, need version 0.24 or later")
 

scripts/kconfig/kconfigfunctions.py

@@ -7,10 +7,11 @@ import os
 import pickle
 import sys
 
-ZEPHYR_BASE = os.environ.get("ZEPHYR_BASE")
-sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/dts"))
+ZEPHYR_BASE = os.environ["ZEPHYR_BASE"]
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts",
+                                "python-devicetree", "src"))
 
-import edtlib
+from devicetree import edtlib
 
 # Types we support
 # 'string', 'int', 'hex', 'bool'

scripts/pylib/twister/twisterlib.py

@@ -63,9 +63,9 @@ if not ZEPHYR_BASE:
     sys.exit("$ZEPHYR_BASE environment variable undefined")
 
 # This is needed to load edt.pickle files.
-sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts"))
-import edtlib  # pylint: disable=unused-import
-
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts",
+                                "python-devicetree", "src"))
+from devicetree import edtlib  # pylint: disable=unused-import
 
 # Use this for internal comparisons; that's what canonicalization is
 # for. Don't use it when invoking other components of the build system

scripts/west_commands/run_common.py

@@ -26,7 +26,7 @@ from zephyr_ext_common import ZEPHYR_SCRIPTS
 
 # Runners depend on edtlib. Make sure the copy in the tree is
 # available to them before trying to import any.
-sys.path.append(str(ZEPHYR_SCRIPTS / 'dts'))
+sys.path.append(str(ZEPHYR_SCRIPTS / 'dts' / 'python-devicetree' / 'src'))
 
 from runners import get_runner_cls, ZephyrBinaryRunner, MissingProgram
 from runners.core import RunnerConfig

scripts/west_commands/runners/bossac.py

@@ -14,7 +14,7 @@ from runners.core import ZephyrBinaryRunner, RunnerCaps, BuildConfiguration
 
 # This is needed to load edt.pickle files.
 try:
-    import edtlib  # pylint: disable=unused-import
+    from devicetree import edtlib  # pylint: disable=unused-import
     MISSING_EDTLIB = False
 except ImportError:
     # This can happen when building the documentation for the