[PATCH v6 23/23] dts: add doc generation

[Juraj Linkeš]

The tool used to generate developer docs is Sphinx, which is already
used in DPDK. The same configuration is used to preserve style, but it's
been augmented with doc-generating configuration and a change to how the
sidebar displays the content hierarchy.

Sphinx generates the documentation from Python docstrings. The docstring
format is the Google format [0] which requires the sphinx.ext.napoleon
extension. The other extension, sphinx.ext.intersphinx, enables linking
to object in external documentations, such as the Python documentation.

There are two requirements for building DTS docs:
* The same Python version as DTS or higher, because Sphinx imports the
  code.
* Also the same Python packages as DTS, for the same reason.

[0] https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings

Signed-off-by: Juraj Linkeš <juraj.linkes at pantheon.tech>
---
 buildtools/call-sphinx-build.py | 29 ++++++++++------
 doc/api/meson.build             |  1 +
 doc/guides/conf.py              | 34 ++++++++++++++++---
 doc/guides/meson.build          |  1 +
 doc/guides/tools/dts.rst        | 32 +++++++++++++++++-
 dts/doc/conf_yaml_schema.json   |  1 +
 dts/doc/index.rst               | 17 ++++++++++
 dts/doc/meson.build             | 60 +++++++++++++++++++++++++++++++++
 dts/meson.build                 | 16 +++++++++
 meson.build                     |  1 +
 10 files changed, 176 insertions(+), 16 deletions(-)
 create mode 120000 dts/doc/conf_yaml_schema.json
 create mode 100644 dts/doc/index.rst
 create mode 100644 dts/doc/meson.build
 create mode 100644 dts/meson.build

diff --git a/buildtools/call-sphinx-build.py b/buildtools/call-sphinx-build.py
index 39a60d09fa..c2f3acfb1d 100755
--- a/buildtools/call-sphinx-build.py
+++ b/buildtools/call-sphinx-build.py
@@ -3,37 +3,46 @@
 # Copyright(c) 2019 Intel Corporation
 #
 
+import argparse
 import sys
 import os
 from os.path import join
 from subprocess import run, PIPE, STDOUT
 from packaging.version import Version
 
-# assign parameters to variables
-(sphinx, version, src, dst, *extra_args) = sys.argv[1:]
+parser = argparse.ArgumentParser()
+parser.add_argument('sphinx')
+parser.add_argument('version')
+parser.add_argument('src')
+parser.add_argument('dst')
+parser.add_argument('--dts-root', default='.')
+args, extra_args = parser.parse_known_args()
 
 # set the version in environment for sphinx to pick up
-os.environ['DPDK_VERSION'] = version
+os.environ['DPDK_VERSION'] = args.version
+os.environ['DTS_ROOT'] = args.dts_root
 
 # for sphinx version >= 1.7 add parallelism using "-j auto"
-ver = run([sphinx, '--version'], stdout=PIPE,
+ver = run([args.sphinx, '--version'], stdout=PIPE,
           stderr=STDOUT).stdout.decode().split()[-1]
-sphinx_cmd = [sphinx] + extra_args
+sphinx_cmd = [args.sphinx] + extra_args
 if Version(ver) >= Version('1.7'):
     sphinx_cmd += ['-j', 'auto']
 
 # find all the files sphinx will process so we can write them as dependencies
 srcfiles = []
-for root, dirs, files in os.walk(src):
+for root, dirs, files in os.walk(args.src):
     srcfiles.extend([join(root, f) for f in files])
 
 # run sphinx, putting the html output in a "html" directory
-with open(join(dst, 'sphinx_html.out'), 'w') as out:
-    process = run(sphinx_cmd + ['-b', 'html', src, join(dst, 'html')],
-                  stdout=out)
+with open(join(args.dst, 'sphinx_html.out'), 'w') as out:
+    process = run(
+        sphinx_cmd + ['-b', 'html', args.src, join(args.dst, 'html')],
+        stdout=out
+    )
 
 # create a gcc format .d file giving all the dependencies of this doc build
-with open(join(dst, '.html.d'), 'w') as d:
+with open(join(args.dst, '.html.d'), 'w') as d:
     d.write('html: ' + ' '.join(srcfiles) + '\n')
 
 sys.exit(process.returncode)
diff --git a/doc/api/meson.build b/doc/api/meson.build
index 5b50692df9..92fe10d9e7 100644
--- a/doc/api/meson.build
+++ b/doc/api/meson.build
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Luca Boccassi <bluca at debian.org>
 
+doc_api_build_dir = meson.current_build_dir()
 doxygen = find_program('doxygen', required: get_option('enable_docs'))
 
 if not doxygen.found()
diff --git a/doc/guides/conf.py b/doc/guides/conf.py
index 0f7ff5282d..169b1d24bc 100644
--- a/doc/guides/conf.py
+++ b/doc/guides/conf.py
@@ -7,10 +7,9 @@
 from sphinx import __version__ as sphinx_version
 from os import listdir
 from os import environ
-from os.path import basename
-from os.path import dirname
+from os.path import basename, dirname
 from os.path import join as path_join
-from sys import argv, stderr
+from sys import argv, stderr, path
 
 import configparser
 
@@ -24,6 +23,31 @@
           file=stderr)
     pass
 
+extensions = ['sphinx.ext.napoleon', 'sphinx.ext.intersphinx']
+
+# Python docstring options
+autodoc_default_options = {
+    'members': True,
+    'member-order': 'bysource',
+    'show-inheritance': True,
+}
+autodoc_class_signature = 'separated'
+autodoc_typehints = 'both'
+autodoc_typehints_format = 'short'
+autodoc_typehints_description_target = 'documented'
+napoleon_numpy_docstring = False
+napoleon_attr_annotations = True
+napoleon_preprocess_types = True
+add_module_names = False
+toc_object_entries = False
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+
+# Sidebar config
+html_theme_options = {
+    'collapse_navigation': False,
+    'navigation_depth': -1,
+}
+
 stop_on_error = ('-W' in argv)
 
 project = 'Data Plane Development Kit'
@@ -35,8 +59,8 @@
 html_show_copyright = False
 highlight_language = 'none'
 
-release = environ.setdefault('DPDK_VERSION', "None")
-version = release
+path.append(environ.get('DTS_ROOT'))
+version = environ.setdefault('DPDK_VERSION', "None")
 
 master_doc = 'index'
 
diff --git a/doc/guides/meson.build b/doc/guides/meson.build
index 51f81da2e3..8933d75f6b 100644
--- a/doc/guides/meson.build
+++ b/doc/guides/meson.build
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Intel Corporation
 
+doc_guides_source_dir = meson.current_source_dir()
 sphinx = find_program('sphinx-build', required: get_option('enable_docs'))
 
 if not sphinx.found()
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index cd771a428c..77d9434c1c 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -283,7 +283,10 @@ When adding code to the DTS framework, pay attention to the rest of the code
 and try not to divert much from it. The :ref:`DTS developer tools <dts_dev_tools>` will issue
 warnings when some of the basics are not met.
 
-The code must be properly documented with docstrings. The style must conform to
+The API documentation, which is a helpful reference when developing, may be accessed
+in the code directly or generated with the `API docs build steps <building_api_docs>`_.
+
+Speaking of which, the code must be properly documented with docstrings. The style must conform to
 the `Google style <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
 See an example of the style
 `here <https://www.sphinx-doc.org/en/master/usage/extensions/example_google.html>`_.
@@ -408,3 +411,30 @@ There are three tools used in DTS to help with code checking, style and formatti
 These three tools are all used in ``devtools/dts-check-format.sh``,
 the DTS code check and format script.
 Refer to the script for usage: ``devtools/dts-check-format.sh -h``.
+
+
+.. _building_api_docs:
+
+Building DTS API docs
+---------------------
+
+To build DTS API docs, install the dependencies with Poetry, then enter its shell:
+
+.. code-block:: console
+
+   poetry install --with docs
+   poetry shell
+
+The documentation is built using the standard DPDK build system. After executing the meson command
+and entering Poetry's shell, build the documentation with:
+
+.. code-block:: console
+
+   ninja -C build dts-doc
+
+The output is generated in ``build/doc/api/dts/html``.
+
+.. Note::
+
+   Make sure to fix any Sphinx warnings when adding or updating docstrings. Also make sure to run
+   the ``devtools/dts-check-format.sh`` script and address any issues it finds.
diff --git a/dts/doc/conf_yaml_schema.json b/dts/doc/conf_yaml_schema.json
new file mode 120000
index 0000000000..d89eb81b72
--- /dev/null
+++ b/dts/doc/conf_yaml_schema.json
@@ -0,0 +1 @@
+../framework/config/conf_yaml_schema.json
\ No newline at end of file
diff --git a/dts/doc/index.rst b/dts/doc/index.rst
new file mode 100644
index 0000000000..f5dcd553f2
--- /dev/null
+++ b/dts/doc/index.rst
@@ -0,0 +1,17 @@
+.. DPDK Test Suite documentation.
+
+Welcome to DPDK Test Suite's documentation!
+===========================================
+
+.. toctree::
+   :titlesonly:
+   :caption: Contents:
+
+   framework
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/dts/doc/meson.build b/dts/doc/meson.build
new file mode 100644
index 0000000000..e11ab83843
--- /dev/null
+++ b/dts/doc/meson.build
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 PANTHEON.tech s.r.o.
+
+sphinx = find_program('sphinx-build', required: false)
+sphinx_apidoc = find_program('sphinx-apidoc', required: false)
+
+if not sphinx.found() or not sphinx_apidoc.found()
+    subdir_done()
+endif
+
+dts_api_framework_dir = join_paths(dts_dir, 'framework')
+dts_api_build_dir = join_paths(doc_api_build_dir, 'dts')
+if meson.version().version_compare('>=0.57.0')
+    dts_api_src = custom_target('dts_api_src',
+            output: 'modules.rst',
+            env: {'SPHINX_APIDOC_OPTIONS': 'members,show-inheritance'},
+            command: [sphinx_apidoc, '--append-syspath', '--force',
+                '--module-first', '--separate', '-V', meson.project_version(),
+                '--output-dir', dts_api_build_dir, '--no-toc', '--implicit-namespaces',
+                dts_api_framework_dir],
+            build_by_default: false)
+else
+    dts_api_src = custom_target('dts_api_src',
+            output: 'modules.rst',
+            command: ['SPHINX_APIDOC_OPTIONS=members,show-inheritance',
+                sphinx_apidoc, '--append-syspath', '--force',
+                '--module-first', '--separate', '-V', meson.project_version(),
+                '--output-dir', dts_api_build_dir, '--no-toc', '--implicit-namespaces',
+                dts_api_framework_dir],
+            build_by_default: false)
+endif
+doc_targets += dts_api_src
+doc_target_names += 'DTS_API_sphinx_sources'
+
+cp = find_program('cp')
+cp_index = custom_target('cp_index',
+        input: ['index.rst', 'conf_yaml_schema.json'],
+        output: 'index.rst',
+        depends: dts_api_src,
+        command: [cp, '--dereference', '@INPUT@', dts_api_build_dir],
+        build_by_default: false)
+doc_targets += cp_index
+doc_target_names += 'DTS_API_sphinx_index'
+
+extra_sphinx_args = ['-E', '-c', doc_guides_source_dir, '--dts-root', dts_dir]
+if get_option('werror')
+    extra_sphinx_args += '-W'
+endif
+
+htmldir = join_paths(get_option('datadir'), 'doc', 'dpdk')
+dts_api_html = custom_target('dts_api_html',
+        output: 'html',
+        depends: cp_index,
+        command: [sphinx_wrapper, sphinx, meson.project_version(),
+            dts_api_build_dir, dts_api_build_dir, extra_sphinx_args],
+        build_by_default: false,
+        install: false,
+        install_dir: htmldir)
+doc_targets += dts_api_html
+doc_target_names += 'DTS_API_HTML'
diff --git a/dts/meson.build b/dts/meson.build
new file mode 100644
index 0000000000..e8ce0f06ac
--- /dev/null
+++ b/dts/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 PANTHEON.tech s.r.o.
+
+doc_targets = []
+doc_target_names = []
+dts_dir = meson.current_source_dir()
+
+subdir('doc')
+
+if doc_targets.length() == 0
+    message = 'No docs targets found'
+else
+    message = 'Built docs:'
+endif
+run_target('dts-doc', command: [echo, message, doc_target_names],
+    depends: doc_targets)
diff --git a/meson.build b/meson.build
index 2e6e546d20..c391bf8c71 100644
--- a/meson.build
+++ b/meson.build
@@ -87,6 +87,7 @@ subdir('app')
 
 # build docs
 subdir('doc')
+subdir('dts')
 
 # build any examples explicitly requested - useful for developers - and
 # install any example code into the appropriate install path
-- 
2.34.1