doc: Vendor hawkmoth in a subdirectory

Co-authored-by: Rahix <rahix@rahix.de>

Documentation/conf.py

@@ -9,6 +9,7 @@ import sphinx.util.logging
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 sys.path.insert(0, os.path.abspath("../pycardium/modules/py"))
+sys.path.insert(0, os.path.abspath("./"))
 
 logger = sphinx.util.logging.getLogger("card10/conf.py")
 
@@ -41,7 +42,7 @@ extensions = [
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["output", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["output", "Thumbs.db", ".DS_Store", "hawkmoth"]
 
 # -- Options for HTML output ------------------------------------------------- {{{
 
@@ -83,8 +84,9 @@ try:
 
     cautodoc_root = os.path.abspath("..")
     has_hawkmoth = True
-except ImportError:
-    logger.warning("Hawkmoth was not found.  Documentation for Epicardium API will not be generated.")
+except ImportError as e:
+    if e.name == "clang":
+        logger.warning("hawkmoth requires the clang python module.  Documentation for Epicardium API will not be generated.")
 # }}}
 
 

Documentation/hawkmoth/LICENSE

+Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Documentation/hawkmoth/Makefile

+all:
+	$(MAKE) -C .. all
+
+.DEFAULT:
+	$(MAKE) -C .. $@

Documentation/hawkmoth/Makefile.local

+# -*- makefile -*-
+
+dir := hawkmoth
+
+CLEAN := $(CLEAN) $(dir)/hawkmoth.pyc $(dir)/cautodoc.pyc $(dir)/__init__.pyc

Documentation/hawkmoth/README.txt

+Hawkmoth - Sphinx Autodoc for C
+===============================
+
+Hawkmoth is a minimalistic Sphinx_ `C Domain`_ autodoc directive extension to
+incorporate formatted C source code comments written in reStructuredText_ into
+Sphinx based documentation. It uses Clang Python Bindings for parsing, and
+generates C Domain directives for C API documentation, and more. In short,
+Hawkmoth is Sphinx Autodoc for C.
+
+Hawkmoth aims to be a compelling alternative for documenting C projects using
+Sphinx, mainly through its simplicity of design, implementation and use.
+
+.. _Sphinx: http://www.sphinx-doc.org
+
+.. _C Domain: http://www.sphinx-doc.org/en/stable/domains.html
+
+.. _reStructuredText: http://docutils.sourceforge.net/rst.html
+
+Example
+-------
+
+Given C source code with rather familiar looking documentation comments::
+
+  /**
+   * Get foo out of bar.
+   */
+  void foobar();
+
+and a directive in the Sphinx project::
+
+  .. c:autodoc:: filename.c
+
+you can incorporate code documentation into Sphinx. It's as simple as that.
+
+You can document functions, their parameters and return values, structs, unions,
+their members, macros, function-like macros, enums, enumeration constants,
+typedefs, variables, as well as have generic documentation comments not attached
+to any symbols.
+
+Documentation
+-------------
+
+Documentation on how to configure Hawkmoth and write documentation comments,
+with examples, is available in the ``doc`` directory in the source tree,
+obviously in Sphinx format and using the directive extension. Pre-built
+documentation `showcasing what Hawkmoth can do`_ is available at `Read the
+Docs`_.
+
+.. _showcasing what Hawkmoth can do: https://hawkmoth.readthedocs.io/en/latest/examples.html
+
+.. _Read the Docs: https://hawkmoth.readthedocs.io/
+
+Installation
+------------
+
+You can install Hawkmoth from PyPI_ with::
+
+  pip install hawkmoth
+
+You'll additionally need to install Clang and Python 3 bindings for it through
+your distro's package manager; they are not available via PyPI. You may also
+need to set ``LD_LIBRARY_PATH`` so that the Clang library can be found. For
+example::
+
+  export LD_LIBRARY_PATH=$(llvm-config --libdir)
+
+Alternatively, installation packages are available for:
+
+* `Arch Linux`_
+
+In Sphinx ``conf.py``, add ``hawkmoth`` to ``extensions``, and point
+``cautodoc_root`` at the source tree. See the extension documentation for
+details.
+
+.. _PyPI: https://pypi.org/project/hawkmoth/
+
+.. _Arch Linux: https://aur.archlinux.org/packages/?K=hawkmoth
+
+Development and Contributing
+----------------------------
+
+Hawkmoth source code is available on GitHub_. The development version can be
+checked out via ``git`` using this command::
+
+  git clone https://github.com/jnikula/hawkmoth.git
+
+Please file bugs and feature requests as GitHub issues. Contributions are
+welcome both as emailed patches to the mailing list and as pull requests.
+
+.. _GitHub: https://github.com/jnikula/hawkmoth
+
+Dependencies
+------------
+
+- Python 3.4
+- Sphinx 1.8
+- Clang 6.0
+- Python 3 Bindings for Clang 6.0
+- sphinx-testing 1.0.0 (for development)
+
+These are the versions Hawkmoth is currently being developed and tested
+against. Other versions might work, but no guarantees.
+
+License
+-------
+
+Hawkmoth is free software, released under the `2-Clause BSD License`_.
+
+.. _2-Clause BSD License: https://opensource.org/licenses/BSD-2-Clause
+
+Contact
+-------
+
+IRC channel ``#hawkmoth`` on freenode_.
+
+Mailing list hawkmoth@freelists.org. Subscription information at the `list home
+page`_.
+
+.. _freenode: https://freenode.net/
+
+.. _list home page: https://www.freelists.org/list/hawkmoth

Documentation/hawkmoth/VERSION

+0.4

Documentation/hawkmoth/__init__.py

+# Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Hawkmoth
+========
+
+Sphinx C Domain autodoc directive extension.
+"""
+
+import glob
+import os
+import re
+import stat
+import subprocess
+import sys
+
+from docutils import nodes, statemachine
+from docutils.parsers.rst import directives, Directive
+from docutils.statemachine import ViewList
+from sphinx.util.nodes import nested_parse_with_titles
+from sphinx.util.docutils import switch_source_input
+from sphinx.util import logging
+
+from hawkmoth.parser import parse, ErrorLevel
+
+with open(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                       'VERSION')) as version_file:
+    __version__ = version_file.read().strip()
+
+class CAutoDocDirective(Directive):
+    """Extract all documentation comments from the specified file"""
+    required_argument = 1
+    optional_arguments = 1
+    logger = logging.getLogger(__name__)
+
+    # Allow passing a variable number of file patterns as arguments
+    final_argument_whitespace = True
+
+    option_spec = {
+        'compat': directives.unchanged_required,
+        'clang': directives.unchanged_required,
+    }
+    has_content = False
+
+    # Map verbosity levels to logger levels.
+    _log_lvl = {ErrorLevel.ERROR: logging.LEVEL_NAMES['ERROR'],
+                ErrorLevel.WARNING: logging.LEVEL_NAMES['WARNING'],
+                ErrorLevel.INFO: logging.LEVEL_NAMES['INFO'],
+                ErrorLevel.DEBUG: logging.LEVEL_NAMES['DEBUG']}
+
+    def __display_parser_diagnostics(self, errors):
+        env = self.state.document.settings.env
+
+        for (severity, filename, lineno, msg) in errors:
+            toprint = '{}:{}: {}'.format(filename, lineno, msg)
+
+            if severity.value <= env.app.verbosity:
+                self.logger.log(self._log_lvl[severity], toprint,
+                                location=(env.docname, self.lineno))
+
+    def __parse(self, viewlist, filename):
+        env = self.state.document.settings.env
+
+        compat = self.options.get('compat', env.config.cautodoc_compat)
+        clang = self.options.get('clang', env.config.cautodoc_clang)
+
+        comments, errors = parse(filename, compat=compat, clang=clang)
+
+        self.__display_parser_diagnostics(errors)
+
+        for (comment, meta) in comments:
+            lineoffset = meta['line'] - 1
+            lines = statemachine.string2lines(comment, 8,
+                                              convert_whitespace=True)
+            for line in lines:
+                viewlist.append(line, filename, lineoffset)
+                lineoffset += 1
+
+    def run(self):
+        env = self.state.document.settings.env
+
+        result = ViewList()
+
+        for pattern in self.arguments[0].split():
+            filenames = glob.glob(env.config.cautodoc_root + '/' + pattern)
+            if len(filenames) == 0:
+                fmt = 'Pattern "{pat}" does not match any files.'
+                self.logger.warning(fmt.format(pat=pattern),
+                                    location=(env.docname, self.lineno))
+                continue
+
+            for filename in filenames:
+                mode = os.stat(filename).st_mode
+                if stat.S_ISDIR(mode):
+                    fmt = 'Path "{name}" matching pattern "{pat}" is a directory.'
+                    self.logger.warning(fmt.format(name=filename, pat=pattern),
+                                        location=(env.docname, self.lineno))
+                    continue
+
+                # Tell Sphinx about the dependency and parse the file
+                env.note_dependency(os.path.abspath(filename))
+                self.__parse(result, filename)
+
+        # Parse the extracted reST
+        with switch_source_input(self.state, result):
+            node = nodes.section()
+            nested_parse_with_titles(self.state, result, node)
+
+        return node.children
+
+def setup(app):
+    app.require_sphinx('1.8')
+    app.add_config_value('cautodoc_root', app.confdir, 'env')
+    app.add_config_value('cautodoc_compat', None, 'env')
+    app.add_config_value('cautodoc_clang', None, 'env')
+    app.add_directive_to_domain('c', 'autodoc', CAutoDocDirective)
+
+    return dict(version = __version__,
+                parallel_read_safe = True, parallel_write_safe = True)

Documentation/hawkmoth/__main__.py

+# Copyright (c) 2016-2019 Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Hawkmoth parser debug tool
+==========================
+
+python3 -m hawkmoth
+"""
+
+import argparse
+import sys
+
+from hawkmoth.parser import parse
+
+def main():
+    parser = argparse.ArgumentParser(prog='hawkmoth', description="""
+    Hawkmoth parser debug tool. Print the documentation comments extracted
+    from FILE, along with the generated C Domain directives, to standard
+    output. Include metadata with verbose output.""")
+    parser.add_argument('file', metavar='FILE', type=str, action='store',
+                        help='The C source or header file to parse.')
+    parser.add_argument('--compat',
+                        choices=['none',
+                                 'javadoc-basic',
+                                 'javadoc-liberal',
+                                 'kernel-doc'],
+                        help='Compatibility options. See cautodoc_compat.')
+    parser.add_argument('--clang', metavar='PARAM[,PARAM,...]',
+                        help='Arguments to pass to clang. See cautodoc_clang.')
+    parser.add_argument('--verbose', dest='verbose', action='store_true',
+                        help='Verbose output.')
+    args = parser.parse_args()
+
+    docs, errors = parse(args.file, compat=args.compat, clang=args.clang)
+
+    for (doc, meta) in docs:
+        if args.verbose:
+            print('# {}'.format(meta))
+        print(doc)
+
+    for (severity, filename, lineno, msg) in errors:
+        print('{}: {}:{}: {}'.format(severity.name,
+                                     filename, lineno, msg), file=sys.stderr)
+
+main()

Documentation/hawkmoth/parser.py

+# Copyright (c) 2016-2017 Jani Nikula <jani@nikula.org>
+# Copyright (c) 2018-2019 Bruno Santos <brunomanuelsantos@tecnico.ulisboa.pt>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Documentation comment extractor
+===============================
+
+This module extracts relevant documentation comments, optionally reformatting
+them in reST syntax.
+
+This is the part that uses Clang Python Bindings to extract documentation
+comments from C source code. This module does not depend on Sphinx.
+
+There are two passes:
+
+#. Pass over the tokens to find all the comments, including ones that aren't
+   attached to cursors.
+
+#. Pass over the cursors to document them.
+
+There is minimal syntax parsing or input conversion:
+
+* Identification of documentation comment blocks, and stripping the comment
+  delimiters (``/**`` and ``*/``) and continuation line prefixes (e.g. ``␣*␣``).
+
+* Identification of function-like macros.
+
+* Indentation for reST C Domain directive blocks.
+
+* An optional external filter may be invoked to support different syntaxes.
+  These filters are expected to translate the comment into the reST format.
+
+Otherwise, documentation comments are passed through verbatim.
+"""
+
+import enum
+import itertools
+import sys
+
+from clang.cindex import CursorKind, TypeKind
+from clang.cindex import Index, TranslationUnit
+from clang.cindex import SourceLocation, SourceRange
+from clang.cindex import TokenKind, TokenGroup
+
+from hawkmoth.util import docstr, doccompat
+
+class ErrorLevel(enum.Enum):
+    """
+    Supported error levels in inverse numerical order of severity. The values
+    are chosen so that they map directly to a 'verbosity level'.
+    """
+    ERROR = 0
+    WARNING = 1
+    INFO = 2
+    DEBUG = 3
+
+def comment_extract(tu):
+
+    # FIXME: How to handle top level comments above a cursor that it does *not*
+    # describe? Parsing @file or @doc at this stage would not be a clean design.
+    # One idea is to use '/***' to denote them, but that might throw off editor
+    # highlighting. The workaround is to follow the top level comment with an
+    # empty '/**/' comment that gets attached to the cursor.
+
+    top_level_comments = []
+    comments = {}
+    cursor = None
+    current_comment = None
+    for token in tu.get_tokens(extent=tu.cursor.extent):
+        # handle all comments we come across
+        if token.kind == TokenKind.COMMENT:
+            # if we already have a comment, it wasn't related to a cursor
+            if current_comment and docstr.is_doc(current_comment.spelling):
+                top_level_comments.append(current_comment)
+            current_comment = token
+            continue
+
+        # cursors that are 1) never documented themselves, and 2) allowed
+        # between comment and the actual cursor being documented
+        if (token.cursor.kind == CursorKind.INVALID_FILE or
+            token.cursor.kind == CursorKind.TYPE_REF or
+            token.cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE or
+            token.cursor.kind == CursorKind.MACRO_INSTANTIATION):
+            continue
+
+        if cursor is not None and token.cursor == cursor:
+            continue
+
+        cursor = token.cursor
+
+        # Note: current_comment may be None
+        if current_comment != None and docstr.is_doc(current_comment.spelling):
+            comments[cursor.hash] = current_comment
+        current_comment = None
+
+    # comment at the end of file
+    if current_comment and docstr.is_doc(current_comment.spelling):
+        top_level_comments.append(current_comment)
+
+    return top_level_comments, comments
+
+def _result(comment, cursor=None, fmt=docstr.Type.TEXT, nest=0,
+            name=None, ttype=None, args=None, compat=None):
+
+    # FIXME: docstr.generate changes the number of lines in output. This impacts
+    # the error reporting via meta['line']. Adjust meta to take this into
+    # account.
+
+    doc = docstr.generate(text=comment.spelling, fmt=fmt,
+                          name=name, ttype=ttype, args=args, transform=compat)
+
+    doc = docstr.nest(doc, nest)
+
+    meta = {'line': comment.extent.start.line}
+    if cursor:
+        meta['cursor.kind']        = cursor.kind,
+        meta['cursor.displayname'] = cursor.displayname,
+        meta['cursor.spelling']    = cursor.spelling
+
+    return [(doc, meta)]
+
+# Return None for simple macros, a potentially empty list of arguments for
+# function-like macros
+def _get_macro_args(cursor):
+    if cursor.kind != CursorKind.MACRO_DEFINITION:
+        return None
+
+    # Use the first two tokens to make sure this starts with 'IDENTIFIER('
+    x = [token for token in itertools.islice(cursor.get_tokens(), 2)]
+    if (len(x) != 2 or x[0].spelling != cursor.spelling or
+        x[1].spelling != '(' or x[0].extent.end != x[1].extent.start):
+        return None
+
+    # Naïve parsing of macro arguments
+    # FIXME: This doesn't handle GCC named vararg extension FOO(vararg...)
+    args = []
+    for token in itertools.islice(cursor.get_tokens(), 2, None):
+        if token.spelling == ')':
+            return args
+        elif token.spelling == ',':
+            continue
+        elif token.kind == TokenKind.IDENTIFIER:
+            args.append(token.spelling)
+        elif token.spelling == '...':
+            args.append(token.spelling)
+        else:
+            break
+
+    return None
+
+def _recursive_parse(comments, cursor, nest, compat):
+    comment = comments[cursor.hash]
+    name = cursor.spelling
+    ttype = cursor.type.spelling
+
+    if cursor.kind == CursorKind.MACRO_DEFINITION:
+        # FIXME: check args against comment
+        args = _get_macro_args(cursor)
+        fmt = docstr.Type.MACRO if args is None else docstr.Type.MACRO_FUNC
+
+        return _result(comment, cursor=cursor, fmt=fmt,
+                       nest=nest, name=name, args=args, compat=compat)
+
+    elif cursor.kind == CursorKind.VAR_DECL:
+        fmt = docstr.Type.VAR
+
+        return _result(comment, cursor=cursor, fmt=fmt,
+                       nest=nest, name=name, ttype=ttype, compat=compat)
+
+    elif cursor.kind == CursorKind.TYPEDEF_DECL:
+        # FIXME: function pointers typedefs.
+        fmt = docstr.Type.TYPE
+
+        return _result(comment, cursor=cursor, fmt=fmt,
+                       nest=nest, name=ttype, compat=compat)
+
+    elif cursor.kind in [CursorKind.STRUCT_DECL, CursorKind.UNION_DECL,
+                         CursorKind.ENUM_DECL]:
+
+        # FIXME:
+        # Handle cases where variables are instantiated on type declaration,
+        # including anonymous cases. Idea is that if there is a variable
+        # instantiation, the documentation should be applied to the variable if
+        # the structure is anonymous or to the type otherwise.
+        #
+        # Due to the new recursiveness of the parser, fixing this here, _should_
+        # handle all cases (struct, union, enum).
+
+        # FIXME: Handle anonymous enumerators.
+
+        fmt = docstr.Type.TYPE
+        result = _result(comment, cursor=cursor, fmt=fmt,
+                         nest=nest, name=ttype, compat=compat)
+
+        nest += 1
+        for c in cursor.get_children():
+            if c.hash in comments:
+                result.extend(_recursive_parse(comments, c, nest, compat))
+
+        return result
+
+    elif cursor.kind == CursorKind.ENUM_CONSTANT_DECL:
+        fmt = docstr.Type.ENUM_VAL
+
+        return _result(comment, cursor=cursor, fmt=fmt,
+                       nest=nest, name=name, compat=compat)
+
+    elif cursor.kind == CursorKind.FIELD_DECL:
+        fmt = docstr.Type.MEMBER
+
+        return _result(comment, cursor=cursor, fmt=fmt,
+                       nest=nest, name=name, ttype=ttype, compat=compat)
+
+    elif cursor.kind == CursorKind.FUNCTION_DECL:
+        # FIXME: check args against comment
+        # FIXME: children may contain extra stuff if the return type is a
+        # typedef, for example
+        args = []
+
+        # Only fully prototyped functions will have argument lists to process.
+        if cursor.type.kind == TypeKind.FUNCTIONPROTO:
+            for c in cursor.get_children():
+                if c.kind == CursorKind.PARM_DECL:
+                    args.append('{ttype} {arg}'.format(ttype=c.type.spelling,
+                                                    arg=c.spelling))
+
+            if cursor.type.is_function_variadic():
+                args.append('...')
+
+        fmt = docstr.Type.FUNC
+        ttype = cursor.result_type.spelling
+
+        return _result(comment, cursor=cursor, fmt=fmt, nest=nest,
+                       name=name, ttype=ttype, args=args, compat=compat)
+
+    # FIXME: If we reach here, nothing matched. This is a warning or even error
+    # and it should be logged, but it should also return an empty list so that
+    # it doesn't break. I.e. the parser needs to pass warnings and errors to the
+    # Sphinx extension instead of polluting the generated output.
+    fmt = docstr.Type.TEXT
+    text = 'warning: unhandled cursor {kind} {name}\n'.format(
+        kind=str(cursor.kind),
+        name=cursor.spelling)
+
+    doc = docstr.generate(text=text, fmt=fmt)
+
+    meta = {
+        'line':               comment.extent.start.line,
+        'cursor.kind':        cursor.kind,
+        'cursor.displayname': cursor.displayname,
+        'cursor.spelling':    cursor.spelling
+    }
+
+    return [(doc, meta)]
+
+def clang_diagnostics(errors, diagnostics):
+    sev = {0: ErrorLevel.DEBUG,
+           1: ErrorLevel.DEBUG,
+           2: ErrorLevel.WARNING,
+           3: ErrorLevel.ERROR,
+           4: ErrorLevel.ERROR}
+
+    for diag in diagnostics:
+        errors.extend([(sev[diag.severity], diag.location.file.name,
+                        diag.location.line, diag.spelling)])
+
+# return a list of (comment, metadata) tuples
+# options - dictionary with directive options
+def parse(filename, **options):
+
+    errors = []
+    args = options.get('clang')
+    if args is not None:
+        args = [s.strip() for s in args.split(',') if len(s.strip()) > 0]
+        if len(args) == 0:
+            args = None
+
+    index = Index.create()
+
+    tu = index.parse(filename, args=args, options=
+                     TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD |
+                     TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
+
+    clang_diagnostics(errors, tu.diagnostics)
+
+    top_level_comments, comments = comment_extract(tu)
+
+    result = []
+    compat = lambda x: doccompat.convert(x, options.get('compat'))
+
+    for comment in top_level_comments:
+        result.extend(_result(comment, compat=compat))
+
+    for cursor in tu.cursor.get_children():
+        if cursor.hash in comments:
+            result.extend(_recursive_parse(comments, cursor, 0, compat))
+
+    # Sort all elements by order of appearance.
+    result.sort(key=lambda r: r[1]['line'])
+
+    return result, errors

Documentation/hawkmoth/util/doccompat.py

+# Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Alternative docstring syntax
+============================
+
+This module abstracts different compatibility options converting different
+syntaxes into 'native' reST ones.
+"""
+
+import re
+
+# Basic Javadoc/Doxygen/kernel-doc import
+#
+# FIXME: One of the design goals of Hawkmoth is to keep things simple. There's a
+# fine balance between sticking to that goal and adding compat code to
+# facilitate any kind of migration to Hawkmoth. The compat code could be turned
+# into a fairly simple plugin architecture, with some basic compat builtins, and
+# the users could still extend the compat features to fit their specific needs.
+def convert(comment, mode):
+    """Convert documentation from a supported syntax into reST."""
+    # FIXME: try to preserve whitespace better
+
+    if mode == 'javadoc-basic' or mode == 'javadoc-liberal':
+        # @param
+        comment = re.sub(r"(?m)^([ \t]*)@param([ \t]+)([a-zA-Z0-9_]+|\.\.\.)([ \t]+)",
+                         "\n\\1:param\\2\\3:\\4", comment)
+        # @param[direction]
+        comment = re.sub(r"(?m)^([ \t]*)@param\[([^]]*)\]([ \t]+)([a-zA-Z0-9_]+|\.\.\.)([ \t]+)",
+                         "\n\\1:param\\3\\4: *(\\2)* \\5", comment)
+        # @return
+        comment = re.sub(r"(?m)^([ \t]*)@returns?([ \t]+|$)",
+                         "\n\\1:return:\\2", comment)
+        # @code/@endcode blocks. Works if the code is indented.
+        comment = re.sub(r"(?m)^([ \t]*)@code([ \t]+|$)",
+                         "\n::\n", comment)
+        comment = re.sub(r"(?m)^([ \t]*)@endcode([ \t]+|$)",
+                         "\n", comment)
+        # Ignore @brief.
+        comment = re.sub(r"(?m)^([ \t]*)@brief[ \t]+", "\n\\1", comment)
+
+        # Ignore groups
+        comment = re.sub(r"(?m)^([ \t]*)@(defgroup|addtogroup)[ \t]+[a-zA-Z0-9_]+[ \t]*",
+                         "\n\\1", comment)
+        comment = re.sub(r"(?m)^([ \t]*)@(ingroup|{|}).*", "\n", comment)
+
+    if mode == 'javadoc-liberal':
+        # Liberal conversion of any @tags, will fail for @code etc. but don't
+        # care.
+        comment = re.sub(r"(?m)^([ \t]*)@([a-zA-Z0-9_]+)([ \t]+)",
+                         "\n\\1:\\2:\\3", comment)
+
+    if mode == 'kernel-doc':
+        # Basic kernel-doc convert, will document struct members as params, etc.
+        comment = re.sub(r"(?m)^([ \t]*)@(returns?|RETURNS?):([ \t]+|$)",
+                         "\n\\1:return:\\3", comment)
+        comment = re.sub(r"(?m)^([ \t]*)@([a-zA-Z0-9_]+|\.\.\.):([ \t]+)",
+                         "\n\\1:param \\2:\\3", comment)
+
+    return comment

Documentation/hawkmoth/util/docstr.py

+# Copyright (c) 2016-2017 Jani Nikula <jani@nikula.org>
+# Copyright (c) 2018-2019 Bruno Santos <brunomanuelsantos@tecnico.ulisboa.pt>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Documentation strings manipulation library
+==========================================
+
+This module allows a generic way of generating reST documentation for each C
+construct.
+"""
+
+import re
+from enum import Enum, auto
+
+class Type(Enum):
+    """Enumeration of supported formats."""
+    TEXT = auto()
+    VAR = auto()
+    TYPE = auto()
+    ENUM_VAL = auto()
+    MEMBER = auto()
+    MACRO = auto()
+    MACRO_FUNC = auto()
+    FUNC = auto()
+
+# Dictionary of tuples (text indentation level, format string).
+#
+# Text indentation is required for indenting the documentation body relative to
+# directive lines.
+_doc_fmt = {
+    Type.TEXT:       (0, '\n{text}\n'),
+    Type.VAR:        (1, '\n.. c:var:: {ttype} {name}\n\n{text}\n'),
+    Type.TYPE:       (1, '\n.. c:type:: {name}\n\n{text}\n'),
+    Type.ENUM_VAL:   (1, '\n.. c:macro:: {name}\n\n{text}\n'),
+    Type.MEMBER:     (1, '\n.. c:member:: {ttype} {name}\n\n{text}\n'),
+    Type.MACRO:      (1, '\n.. c:macro:: {name}\n\n{text}\n'),
+    Type.MACRO_FUNC: (1, '\n.. c:function:: {name}({args})\n\n{text}\n'),
+    Type.FUNC:       (1, '\n.. c:function:: {ttype} {name}({args})\n\n{text}\n')
+}
+
+def _strip(comment):
+    """Strip comment from comment markers."""
+    comment = re.sub(r'^/\*\*[ \t]?', '', comment)
+    comment = re.sub(r'\*/$', '', comment)
+    # Could look at first line of comment, and remove the leading stuff there
+    # from the rest.
+    comment = re.sub(r'(?m)^[ \t]*\*?[ \t]?', '', comment)
+    # Strip leading blank lines.
+    comment = re.sub(r'^[\n]*', '', comment)
+    return comment.strip()
+
+def is_doc(comment):
+    """Test if comment is a C documentation comment."""
+    return comment.startswith('/**') and comment != '/**/'
+
+def nest(text, nest):
+    """
+    Indent documentation block for nesting.
+
+    Args:
+        text (str): Documentation body.
+        nest (int): Nesting level. For each level, the final block is indented
+            one level. Useful for (e.g.) declaring structure members.
+
+    Returns:
+        str: Indented reST documentation string.
+    """
+    return re.sub('(?m)^(?!$)', '   ' * nest, text)
+
+def generate(text, fmt=Type.TEXT, name=None,
+             ttype=None, args=None, transform=None):
+    """
+    Generate reST documentation string.
+
+    Args:
+        text (str): Documentation body.
+        fmt (enum :py:class:`Type`): Format type to use. Different formats
+            require different arguments and ignores others if given.
+        name (str): Name of the documented token.
+        ttype (str): Type of the documented token.
+        args (list): List of arguments (str).
+        transform (func): Transformation function to be applied to the
+            documentation body. This is useful (e.g.) to extend the generator
+            with different syntaxes by converting them to reST. This is applied
+            on the documentation body after removing comment markers.
+
+    Returns:
+        str: reST documentation string.
+    """
+
+    text = _strip(text)
+
+    if transform:
+        text = transform(text)
+
+    if args is not None:
+        args = ', '.join(args)
+
+    (text_indent, fmt) = _doc_fmt[fmt]
+    text = nest(text, text_indent)
+    return fmt.format(text=text, name=name, ttype=ttype, args=args)