From 3ca87334942dccd7b22aeea3ae278728bfb4d323 Mon Sep 17 00:00:00 2001
From: ch3 <ch3@mailbox.org>
Date: Tue, 9 Jul 2019 20:32:25 +0200
Subject: [PATCH] doc: Vendor hawkmoth in a subdirectory
Co-authored-by: Rahix <rahix@rahix.de>
---
Documentation/conf.py | 8 +-
Documentation/hawkmoth/LICENSE | 25 ++
Documentation/hawkmoth/Makefile | 5 +
Documentation/hawkmoth/Makefile.local | 5 +
Documentation/hawkmoth/README.txt | 121 +++++++++
Documentation/hawkmoth/VERSION | 1 +
Documentation/hawkmoth/__init__.py | 119 +++++++++
Documentation/hawkmoth/__main__.py | 45 ++++
Documentation/hawkmoth/parser.py | 301 +++++++++++++++++++++++
Documentation/hawkmoth/util/__init__.py | 0
Documentation/hawkmoth/util/doccompat.py | 60 +++++
Documentation/hawkmoth/util/docstr.py | 101 ++++++++
12 files changed, 788 insertions(+), 3 deletions(-)
create mode 100644 Documentation/hawkmoth/LICENSE
create mode 100644 Documentation/hawkmoth/Makefile
create mode 100644 Documentation/hawkmoth/Makefile.local
create mode 100644 Documentation/hawkmoth/README.txt
create mode 100644 Documentation/hawkmoth/VERSION
create mode 100644 Documentation/hawkmoth/__init__.py
create mode 100644 Documentation/hawkmoth/__main__.py
create mode 100644 Documentation/hawkmoth/parser.py
create mode 100644 Documentation/hawkmoth/util/__init__.py
create mode 100644 Documentation/hawkmoth/util/doccompat.py
create mode 100644 Documentation/hawkmoth/util/docstr.py
diff --git a/Documentation/conf.py b/Documentation/conf.py
index 3c12ec794..8bdf2933d 100644
--- a/Documentation/conf.py
+++ b/Documentation/conf.py
@@ -9,6 +9,7 @@ import sphinx.util.logging
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
sys.path.insert(0, os.path.abspath("../pycardium/modules/py"))
+sys.path.insert(0, os.path.abspath("./"))
logger = sphinx.util.logging.getLogger("card10/conf.py")
@@ -41,7 +42,7 @@ extensions = [
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["output", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["output", "Thumbs.db", ".DS_Store", "hawkmoth"]
# -- Options for HTML output ------------------------------------------------- {{{
@@ -83,8 +84,9 @@ try:
cautodoc_root = os.path.abspath("..")
has_hawkmoth = True
-except ImportError:
- logger.warning("Hawkmoth was not found. Documentation for Epicardium API will not be generated.")
+except ImportError as e:
+ if e.name == "clang":
+ logger.warning("hawkmoth requires the clang python module. Documentation for Epicardium API will not be generated.")
# }}}
diff --git a/Documentation/hawkmoth/LICENSE b/Documentation/hawkmoth/LICENSE
new file mode 100644
index 000000000..6b0f4aa2d
--- /dev/null
+++ b/Documentation/hawkmoth/LICENSE
@@ -0,0 +1,25 @@
+Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/Documentation/hawkmoth/Makefile b/Documentation/hawkmoth/Makefile
new file mode 100644
index 000000000..fa25832e0
--- /dev/null
+++ b/Documentation/hawkmoth/Makefile
@@ -0,0 +1,5 @@
+all:
+ $(MAKE) -C .. all
+
+.DEFAULT:
+ $(MAKE) -C .. $@
diff --git a/Documentation/hawkmoth/Makefile.local b/Documentation/hawkmoth/Makefile.local
new file mode 100644
index 000000000..c6fdecb03
--- /dev/null
+++ b/Documentation/hawkmoth/Makefile.local
@@ -0,0 +1,5 @@
+# -*- makefile -*-
+
+dir := hawkmoth
+
+CLEAN := $(CLEAN) $(dir)/hawkmoth.pyc $(dir)/cautodoc.pyc $(dir)/__init__.pyc
diff --git a/Documentation/hawkmoth/README.txt b/Documentation/hawkmoth/README.txt
new file mode 100644
index 000000000..118ce99e2
--- /dev/null
+++ b/Documentation/hawkmoth/README.txt
@@ -0,0 +1,121 @@
+Hawkmoth - Sphinx Autodoc for C
+===============================
+
+Hawkmoth is a minimalistic Sphinx_ `C Domain`_ autodoc directive extension to
+incorporate formatted C source code comments written in reStructuredText_ into
+Sphinx based documentation. It uses Clang Python Bindings for parsing, and
+generates C Domain directives for C API documentation, and more. In short,
+Hawkmoth is Sphinx Autodoc for C.
+
+Hawkmoth aims to be a compelling alternative for documenting C projects using
+Sphinx, mainly through its simplicity of design, implementation and use.
+
+.. _Sphinx: http://www.sphinx-doc.org
+
+.. _C Domain: http://www.sphinx-doc.org/en/stable/domains.html
+
+.. _reStructuredText: http://docutils.sourceforge.net/rst.html
+
+Example
+-------
+
+Given C source code with rather familiar looking documentation comments::
+
+ /**
+ * Get foo out of bar.
+ */
+ void foobar();
+
+and a directive in the Sphinx project::
+
+ .. c:autodoc:: filename.c
+
+you can incorporate code documentation into Sphinx. It's as simple as that.
+
+You can document functions, their parameters and return values, structs, unions,
+their members, macros, function-like macros, enums, enumeration constants,
+typedefs, variables, as well as have generic documentation comments not attached
+to any symbols.
+
+Documentation
+-------------
+
+Documentation on how to configure Hawkmoth and write documentation comments,
+with examples, is available in the ``doc`` directory in the source tree,
+obviously in Sphinx format and using the directive extension. Pre-built
+documentation `showcasing what Hawkmoth can do`_ is available at `Read the
+Docs`_.
+
+.. _showcasing what Hawkmoth can do: https://hawkmoth.readthedocs.io/en/latest/examples.html
+
+.. _Read the Docs: https://hawkmoth.readthedocs.io/
+
+Installation
+------------
+
+You can install Hawkmoth from PyPI_ with::
+
+ pip install hawkmoth
+
+You'll additionally need to install Clang and Python 3 bindings for it through
+your distro's package manager; they are not available via PyPI. You may also
+need to set ``LD_LIBRARY_PATH`` so that the Clang library can be found. For
+example::
+
+ export LD_LIBRARY_PATH=$(llvm-config --libdir)
+
+Alternatively, installation packages are available for:
+
+* `Arch Linux`_
+
+In Sphinx ``conf.py``, add ``hawkmoth`` to ``extensions``, and point
+``cautodoc_root`` at the source tree. See the extension documentation for
+details.
+
+.. _PyPI: https://pypi.org/project/hawkmoth/
+
+.. _Arch Linux: https://aur.archlinux.org/packages/?K=hawkmoth
+
+Development and Contributing
+----------------------------
+
+Hawkmoth source code is available on GitHub_. The development version can be
+checked out via ``git`` using this command::
+
+ git clone https://github.com/jnikula/hawkmoth.git
+
+Please file bugs and feature requests as GitHub issues. Contributions are
+welcome both as emailed patches to the mailing list and as pull requests.
+
+.. _GitHub: https://github.com/jnikula/hawkmoth
+
+Dependencies
+------------
+
+- Python 3.4
+- Sphinx 1.8
+- Clang 6.0
+- Python 3 Bindings for Clang 6.0
+- sphinx-testing 1.0.0 (for development)
+
+These are the versions Hawkmoth is currently being developed and tested
+against. Other versions might work, but no guarantees.
+
+License
+-------
+
+Hawkmoth is free software, released under the `2-Clause BSD License`_.
+
+.. _2-Clause BSD License: https://opensource.org/licenses/BSD-2-Clause
+
+Contact
+-------
+
+IRC channel ``#hawkmoth`` on freenode_.
+
+Mailing list hawkmoth@freelists.org. Subscription information at the `list home
+page`_.
+
+.. _freenode: https://freenode.net/
+
+.. _list home page: https://www.freelists.org/list/hawkmoth
diff --git a/Documentation/hawkmoth/VERSION b/Documentation/hawkmoth/VERSION
new file mode 100644
index 000000000..bd73f4707
--- /dev/null
+++ b/Documentation/hawkmoth/VERSION
@@ -0,0 +1 @@
+0.4
diff --git a/Documentation/hawkmoth/__init__.py b/Documentation/hawkmoth/__init__.py
new file mode 100644
index 000000000..0896453ea
--- /dev/null
+++ b/Documentation/hawkmoth/__init__.py
@@ -0,0 +1,119 @@
+# Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Hawkmoth
+========
+
+Sphinx C Domain autodoc directive extension.
+"""
+
+import glob
+import os
+import re
+import stat
+import subprocess
+import sys
+
+from docutils import nodes, statemachine
+from docutils.parsers.rst import directives, Directive
+from docutils.statemachine import ViewList
+from sphinx.util.nodes import nested_parse_with_titles
+from sphinx.util.docutils import switch_source_input
+from sphinx.util import logging
+
+from hawkmoth.parser import parse, ErrorLevel
+
+with open(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ 'VERSION')) as version_file:
+ __version__ = version_file.read().strip()
+
+class CAutoDocDirective(Directive):
+ """Extract all documentation comments from the specified file"""
+ required_argument = 1
+ optional_arguments = 1
+ logger = logging.getLogger(__name__)
+
+ # Allow passing a variable number of file patterns as arguments
+ final_argument_whitespace = True
+
+ option_spec = {
+ 'compat': directives.unchanged_required,
+ 'clang': directives.unchanged_required,
+ }
+ has_content = False
+
+ # Map verbosity levels to logger levels.
+ _log_lvl = {ErrorLevel.ERROR: logging.LEVEL_NAMES['ERROR'],
+ ErrorLevel.WARNING: logging.LEVEL_NAMES['WARNING'],
+ ErrorLevel.INFO: logging.LEVEL_NAMES['INFO'],
+ ErrorLevel.DEBUG: logging.LEVEL_NAMES['DEBUG']}
+
+ def __display_parser_diagnostics(self, errors):
+ env = self.state.document.settings.env
+
+ for (severity, filename, lineno, msg) in errors:
+ toprint = '{}:{}: {}'.format(filename, lineno, msg)
+
+ if severity.value <= env.app.verbosity:
+ self.logger.log(self._log_lvl[severity], toprint,
+ location=(env.docname, self.lineno))
+
+ def __parse(self, viewlist, filename):
+ env = self.state.document.settings.env
+
+ compat = self.options.get('compat', env.config.cautodoc_compat)
+ clang = self.options.get('clang', env.config.cautodoc_clang)
+
+ comments, errors = parse(filename, compat=compat, clang=clang)
+
+ self.__display_parser_diagnostics(errors)
+
+ for (comment, meta) in comments:
+ lineoffset = meta['line'] - 1
+ lines = statemachine.string2lines(comment, 8,
+ convert_whitespace=True)
+ for line in lines:
+ viewlist.append(line, filename, lineoffset)
+ lineoffset += 1
+
+ def run(self):
+ env = self.state.document.settings.env
+
+ result = ViewList()
+
+ for pattern in self.arguments[0].split():
+ filenames = glob.glob(env.config.cautodoc_root + '/' + pattern)
+ if len(filenames) == 0:
+ fmt = 'Pattern "{pat}" does not match any files.'
+ self.logger.warning(fmt.format(pat=pattern),
+ location=(env.docname, self.lineno))
+ continue
+
+ for filename in filenames:
+ mode = os.stat(filename).st_mode
+ if stat.S_ISDIR(mode):
+ fmt = 'Path "{name}" matching pattern "{pat}" is a directory.'
+ self.logger.warning(fmt.format(name=filename, pat=pattern),
+ location=(env.docname, self.lineno))
+ continue
+
+ # Tell Sphinx about the dependency and parse the file
+ env.note_dependency(os.path.abspath(filename))
+ self.__parse(result, filename)
+
+ # Parse the extracted reST
+ with switch_source_input(self.state, result):
+ node = nodes.section()
+ nested_parse_with_titles(self.state, result, node)
+
+ return node.children
+
+def setup(app):
+ app.require_sphinx('1.8')
+ app.add_config_value('cautodoc_root', app.confdir, 'env')
+ app.add_config_value('cautodoc_compat', None, 'env')
+ app.add_config_value('cautodoc_clang', None, 'env')
+ app.add_directive_to_domain('c', 'autodoc', CAutoDocDirective)
+
+ return dict(version = __version__,
+ parallel_read_safe = True, parallel_write_safe = True)
diff --git a/Documentation/hawkmoth/__main__.py b/Documentation/hawkmoth/__main__.py
new file mode 100644
index 000000000..96f6ce7f2
--- /dev/null
+++ b/Documentation/hawkmoth/__main__.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2016-2019 Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Hawkmoth parser debug tool
+==========================
+
+python3 -m hawkmoth
+"""
+
+import argparse
+import sys
+
+from hawkmoth.parser import parse
+
+def main():
+ parser = argparse.ArgumentParser(prog='hawkmoth', description="""
+ Hawkmoth parser debug tool. Print the documentation comments extracted
+ from FILE, along with the generated C Domain directives, to standard
+ output. Include metadata with verbose output.""")
+ parser.add_argument('file', metavar='FILE', type=str, action='store',
+ help='The C source or header file to parse.')
+ parser.add_argument('--compat',
+ choices=['none',
+ 'javadoc-basic',
+ 'javadoc-liberal',
+ 'kernel-doc'],
+ help='Compatibility options. See cautodoc_compat.')
+ parser.add_argument('--clang', metavar='PARAM[,PARAM,...]',
+ help='Arguments to pass to clang. See cautodoc_clang.')
+ parser.add_argument('--verbose', dest='verbose', action='store_true',
+ help='Verbose output.')
+ args = parser.parse_args()
+
+ docs, errors = parse(args.file, compat=args.compat, clang=args.clang)
+
+ for (doc, meta) in docs:
+ if args.verbose:
+ print('# {}'.format(meta))
+ print(doc)
+
+ for (severity, filename, lineno, msg) in errors:
+ print('{}: {}:{}: {}'.format(severity.name,
+ filename, lineno, msg), file=sys.stderr)
+
+main()
diff --git a/Documentation/hawkmoth/parser.py b/Documentation/hawkmoth/parser.py
new file mode 100644
index 000000000..a070ad1cd
--- /dev/null
+++ b/Documentation/hawkmoth/parser.py
@@ -0,0 +1,301 @@
+# Copyright (c) 2016-2017 Jani Nikula <jani@nikula.org>
+# Copyright (c) 2018-2019 Bruno Santos <brunomanuelsantos@tecnico.ulisboa.pt>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Documentation comment extractor
+===============================
+
+This module extracts relevant documentation comments, optionally reformatting
+them in reST syntax.
+
+This is the part that uses Clang Python Bindings to extract documentation
+comments from C source code. This module does not depend on Sphinx.
+
+There are two passes:
+
+#. Pass over the tokens to find all the comments, including ones that aren't
+ attached to cursors.
+
+#. Pass over the cursors to document them.
+
+There is minimal syntax parsing or input conversion:
+
+* Identification of documentation comment blocks, and stripping the comment
+ delimiters (``/**`` and ``*/``) and continuation line prefixes (e.g. ``â£*â£``).
+
+* Identification of function-like macros.
+
+* Indentation for reST C Domain directive blocks.
+
+* An optional external filter may be invoked to support different syntaxes.
+ These filters are expected to translate the comment into the reST format.
+
+Otherwise, documentation comments are passed through verbatim.
+"""
+
+import enum
+import itertools
+import sys
+
+from clang.cindex import CursorKind, TypeKind
+from clang.cindex import Index, TranslationUnit
+from clang.cindex import SourceLocation, SourceRange
+from clang.cindex import TokenKind, TokenGroup
+
+from hawkmoth.util import docstr, doccompat
+
+class ErrorLevel(enum.Enum):
+ """
+ Supported error levels in inverse numerical order of severity. The values
+ are chosen so that they map directly to a 'verbosity level'.
+ """
+ ERROR = 0
+ WARNING = 1
+ INFO = 2
+ DEBUG = 3
+
+def comment_extract(tu):
+
+ # FIXME: How to handle top level comments above a cursor that it does *not*
+ # describe? Parsing @file or @doc at this stage would not be a clean design.
+ # One idea is to use '/***' to denote them, but that might throw off editor
+ # highlighting. The workaround is to follow the top level comment with an
+ # empty '/**/' comment that gets attached to the cursor.
+
+ top_level_comments = []
+ comments = {}
+ cursor = None
+ current_comment = None
+ for token in tu.get_tokens(extent=tu.cursor.extent):
+ # handle all comments we come across
+ if token.kind == TokenKind.COMMENT:
+ # if we already have a comment, it wasn't related to a cursor
+ if current_comment and docstr.is_doc(current_comment.spelling):
+ top_level_comments.append(current_comment)
+ current_comment = token
+ continue
+
+ # cursors that are 1) never documented themselves, and 2) allowed
+ # between comment and the actual cursor being documented
+ if (token.cursor.kind == CursorKind.INVALID_FILE or
+ token.cursor.kind == CursorKind.TYPE_REF or
+ token.cursor.kind == CursorKind.PREPROCESSING_DIRECTIVE or
+ token.cursor.kind == CursorKind.MACRO_INSTANTIATION):
+ continue
+
+ if cursor is not None and token.cursor == cursor:
+ continue
+
+ cursor = token.cursor
+
+ # Note: current_comment may be None
+ if current_comment != None and docstr.is_doc(current_comment.spelling):
+ comments[cursor.hash] = current_comment
+ current_comment = None
+
+ # comment at the end of file
+ if current_comment and docstr.is_doc(current_comment.spelling):
+ top_level_comments.append(current_comment)
+
+ return top_level_comments, comments
+
+def _result(comment, cursor=None, fmt=docstr.Type.TEXT, nest=0,
+ name=None, ttype=None, args=None, compat=None):
+
+ # FIXME: docstr.generate changes the number of lines in output. This impacts
+ # the error reporting via meta['line']. Adjust meta to take this into
+ # account.
+
+ doc = docstr.generate(text=comment.spelling, fmt=fmt,
+ name=name, ttype=ttype, args=args, transform=compat)
+
+ doc = docstr.nest(doc, nest)
+
+ meta = {'line': comment.extent.start.line}
+ if cursor:
+ meta['cursor.kind'] = cursor.kind,
+ meta['cursor.displayname'] = cursor.displayname,
+ meta['cursor.spelling'] = cursor.spelling
+
+ return [(doc, meta)]
+
+# Return None for simple macros, a potentially empty list of arguments for
+# function-like macros
+def _get_macro_args(cursor):
+ if cursor.kind != CursorKind.MACRO_DEFINITION:
+ return None
+
+ # Use the first two tokens to make sure this starts with 'IDENTIFIER('
+ x = [token for token in itertools.islice(cursor.get_tokens(), 2)]
+ if (len(x) != 2 or x[0].spelling != cursor.spelling or
+ x[1].spelling != '(' or x[0].extent.end != x[1].extent.start):
+ return None
+
+ # Naïve parsing of macro arguments
+ # FIXME: This doesn't handle GCC named vararg extension FOO(vararg...)
+ args = []
+ for token in itertools.islice(cursor.get_tokens(), 2, None):
+ if token.spelling == ')':
+ return args
+ elif token.spelling == ',':
+ continue
+ elif token.kind == TokenKind.IDENTIFIER:
+ args.append(token.spelling)
+ elif token.spelling == '...':
+ args.append(token.spelling)
+ else:
+ break
+
+ return None
+
+def _recursive_parse(comments, cursor, nest, compat):
+ comment = comments[cursor.hash]
+ name = cursor.spelling
+ ttype = cursor.type.spelling
+
+ if cursor.kind == CursorKind.MACRO_DEFINITION:
+ # FIXME: check args against comment
+ args = _get_macro_args(cursor)
+ fmt = docstr.Type.MACRO if args is None else docstr.Type.MACRO_FUNC
+
+ return _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=name, args=args, compat=compat)
+
+ elif cursor.kind == CursorKind.VAR_DECL:
+ fmt = docstr.Type.VAR
+
+ return _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=name, ttype=ttype, compat=compat)
+
+ elif cursor.kind == CursorKind.TYPEDEF_DECL:
+ # FIXME: function pointers typedefs.
+ fmt = docstr.Type.TYPE
+
+ return _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=ttype, compat=compat)
+
+ elif cursor.kind in [CursorKind.STRUCT_DECL, CursorKind.UNION_DECL,
+ CursorKind.ENUM_DECL]:
+
+ # FIXME:
+ # Handle cases where variables are instantiated on type declaration,
+ # including anonymous cases. Idea is that if there is a variable
+ # instantiation, the documentation should be applied to the variable if
+ # the structure is anonymous or to the type otherwise.
+ #
+ # Due to the new recursiveness of the parser, fixing this here, _should_
+ # handle all cases (struct, union, enum).
+
+ # FIXME: Handle anonymous enumerators.
+
+ fmt = docstr.Type.TYPE
+ result = _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=ttype, compat=compat)
+
+ nest += 1
+ for c in cursor.get_children():
+ if c.hash in comments:
+ result.extend(_recursive_parse(comments, c, nest, compat))
+
+ return result
+
+ elif cursor.kind == CursorKind.ENUM_CONSTANT_DECL:
+ fmt = docstr.Type.ENUM_VAL
+
+ return _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=name, compat=compat)
+
+ elif cursor.kind == CursorKind.FIELD_DECL:
+ fmt = docstr.Type.MEMBER
+
+ return _result(comment, cursor=cursor, fmt=fmt,
+ nest=nest, name=name, ttype=ttype, compat=compat)
+
+ elif cursor.kind == CursorKind.FUNCTION_DECL:
+ # FIXME: check args against comment
+ # FIXME: children may contain extra stuff if the return type is a
+ # typedef, for example
+ args = []
+
+ # Only fully prototyped functions will have argument lists to process.
+ if cursor.type.kind == TypeKind.FUNCTIONPROTO:
+ for c in cursor.get_children():
+ if c.kind == CursorKind.PARM_DECL:
+ args.append('{ttype} {arg}'.format(ttype=c.type.spelling,
+ arg=c.spelling))
+
+ if cursor.type.is_function_variadic():
+ args.append('...')
+
+ fmt = docstr.Type.FUNC
+ ttype = cursor.result_type.spelling
+
+ return _result(comment, cursor=cursor, fmt=fmt, nest=nest,
+ name=name, ttype=ttype, args=args, compat=compat)
+
+ # FIXME: If we reach here, nothing matched. This is a warning or even error
+ # and it should be logged, but it should also return an empty list so that
+ # it doesn't break. I.e. the parser needs to pass warnings and errors to the
+ # Sphinx extension instead of polluting the generated output.
+ fmt = docstr.Type.TEXT
+ text = 'warning: unhandled cursor {kind} {name}\n'.format(
+ kind=str(cursor.kind),
+ name=cursor.spelling)
+
+ doc = docstr.generate(text=text, fmt=fmt)
+
+ meta = {
+ 'line': comment.extent.start.line,
+ 'cursor.kind': cursor.kind,
+ 'cursor.displayname': cursor.displayname,
+ 'cursor.spelling': cursor.spelling
+ }
+
+ return [(doc, meta)]
+
+def clang_diagnostics(errors, diagnostics):
+ sev = {0: ErrorLevel.DEBUG,
+ 1: ErrorLevel.DEBUG,
+ 2: ErrorLevel.WARNING,
+ 3: ErrorLevel.ERROR,
+ 4: ErrorLevel.ERROR}
+
+ for diag in diagnostics:
+ errors.extend([(sev[diag.severity], diag.location.file.name,
+ diag.location.line, diag.spelling)])
+
+# return a list of (comment, metadata) tuples
+# options - dictionary with directive options
+def parse(filename, **options):
+
+ errors = []
+ args = options.get('clang')
+ if args is not None:
+ args = [s.strip() for s in args.split(',') if len(s.strip()) > 0]
+ if len(args) == 0:
+ args = None
+
+ index = Index.create()
+
+ tu = index.parse(filename, args=args, options=
+ TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD |
+ TranslationUnit.PARSE_SKIP_FUNCTION_BODIES)
+
+ clang_diagnostics(errors, tu.diagnostics)
+
+ top_level_comments, comments = comment_extract(tu)
+
+ result = []
+ compat = lambda x: doccompat.convert(x, options.get('compat'))
+
+ for comment in top_level_comments:
+ result.extend(_result(comment, compat=compat))
+
+ for cursor in tu.cursor.get_children():
+ if cursor.hash in comments:
+ result.extend(_recursive_parse(comments, cursor, 0, compat))
+
+ # Sort all elements by order of appearance.
+ result.sort(key=lambda r: r[1]['line'])
+
+ return result, errors
diff --git a/Documentation/hawkmoth/util/__init__.py b/Documentation/hawkmoth/util/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Documentation/hawkmoth/util/doccompat.py b/Documentation/hawkmoth/util/doccompat.py
new file mode 100644
index 000000000..e687bc4b4
--- /dev/null
+++ b/Documentation/hawkmoth/util/doccompat.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2016-2017, Jani Nikula <jani@nikula.org>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Alternative docstring syntax
+============================
+
+This module abstracts different compatibility options converting different
+syntaxes into 'native' reST ones.
+"""
+
+import re
+
+# Basic Javadoc/Doxygen/kernel-doc import
+#
+# FIXME: One of the design goals of Hawkmoth is to keep things simple. There's a
+# fine balance between sticking to that goal and adding compat code to
+# facilitate any kind of migration to Hawkmoth. The compat code could be turned
+# into a fairly simple plugin architecture, with some basic compat builtins, and
+# the users could still extend the compat features to fit their specific needs.
+def convert(comment, mode):
+ """Convert documentation from a supported syntax into reST."""
+ # FIXME: try to preserve whitespace better
+
+ if mode == 'javadoc-basic' or mode == 'javadoc-liberal':
+ # @param
+ comment = re.sub(r"(?m)^([ \t]*)@param([ \t]+)([a-zA-Z0-9_]+|\.\.\.)([ \t]+)",
+ "\n\\1:param\\2\\3:\\4", comment)
+ # @param[direction]
+ comment = re.sub(r"(?m)^([ \t]*)@param\[([^]]*)\]([ \t]+)([a-zA-Z0-9_]+|\.\.\.)([ \t]+)",
+ "\n\\1:param\\3\\4: *(\\2)* \\5", comment)
+ # @return
+ comment = re.sub(r"(?m)^([ \t]*)@returns?([ \t]+|$)",
+ "\n\\1:return:\\2", comment)
+ # @code/@endcode blocks. Works if the code is indented.
+ comment = re.sub(r"(?m)^([ \t]*)@code([ \t]+|$)",
+ "\n::\n", comment)
+ comment = re.sub(r"(?m)^([ \t]*)@endcode([ \t]+|$)",
+ "\n", comment)
+ # Ignore @brief.
+ comment = re.sub(r"(?m)^([ \t]*)@brief[ \t]+", "\n\\1", comment)
+
+ # Ignore groups
+ comment = re.sub(r"(?m)^([ \t]*)@(defgroup|addtogroup)[ \t]+[a-zA-Z0-9_]+[ \t]*",
+ "\n\\1", comment)
+ comment = re.sub(r"(?m)^([ \t]*)@(ingroup|{|}).*", "\n", comment)
+
+ if mode == 'javadoc-liberal':
+ # Liberal conversion of any @tags, will fail for @code etc. but don't
+ # care.
+ comment = re.sub(r"(?m)^([ \t]*)@([a-zA-Z0-9_]+)([ \t]+)",
+ "\n\\1:\\2:\\3", comment)
+
+ if mode == 'kernel-doc':
+ # Basic kernel-doc convert, will document struct members as params, etc.
+ comment = re.sub(r"(?m)^([ \t]*)@(returns?|RETURNS?):([ \t]+|$)",
+ "\n\\1:return:\\3", comment)
+ comment = re.sub(r"(?m)^([ \t]*)@([a-zA-Z0-9_]+|\.\.\.):([ \t]+)",
+ "\n\\1:param \\2:\\3", comment)
+
+ return comment
diff --git a/Documentation/hawkmoth/util/docstr.py b/Documentation/hawkmoth/util/docstr.py
new file mode 100644
index 000000000..2e6572f92
--- /dev/null
+++ b/Documentation/hawkmoth/util/docstr.py
@@ -0,0 +1,101 @@
+# Copyright (c) 2016-2017 Jani Nikula <jani@nikula.org>
+# Copyright (c) 2018-2019 Bruno Santos <brunomanuelsantos@tecnico.ulisboa.pt>
+# Licensed under the terms of BSD 2-Clause, see LICENSE for details.
+"""
+Documentation strings manipulation library
+==========================================
+
+This module allows a generic way of generating reST documentation for each C
+construct.
+"""
+
+import re
+from enum import Enum, auto
+
+class Type(Enum):
+ """Enumeration of supported formats."""
+ TEXT = auto()
+ VAR = auto()
+ TYPE = auto()
+ ENUM_VAL = auto()
+ MEMBER = auto()
+ MACRO = auto()
+ MACRO_FUNC = auto()
+ FUNC = auto()
+
+# Dictionary of tuples (text indentation level, format string).
+#
+# Text indentation is required for indenting the documentation body relative to
+# directive lines.
+_doc_fmt = {
+ Type.TEXT: (0, '\n{text}\n'),
+ Type.VAR: (1, '\n.. c:var:: {ttype} {name}\n\n{text}\n'),
+ Type.TYPE: (1, '\n.. c:type:: {name}\n\n{text}\n'),
+ Type.ENUM_VAL: (1, '\n.. c:macro:: {name}\n\n{text}\n'),
+ Type.MEMBER: (1, '\n.. c:member:: {ttype} {name}\n\n{text}\n'),
+ Type.MACRO: (1, '\n.. c:macro:: {name}\n\n{text}\n'),
+ Type.MACRO_FUNC: (1, '\n.. c:function:: {name}({args})\n\n{text}\n'),
+ Type.FUNC: (1, '\n.. c:function:: {ttype} {name}({args})\n\n{text}\n')
+}
+
+def _strip(comment):
+ """Strip comment from comment markers."""
+ comment = re.sub(r'^/\*\*[ \t]?', '', comment)
+ comment = re.sub(r'\*/$', '', comment)
+ # Could look at first line of comment, and remove the leading stuff there
+ # from the rest.
+ comment = re.sub(r'(?m)^[ \t]*\*?[ \t]?', '', comment)
+ # Strip leading blank lines.
+ comment = re.sub(r'^[\n]*', '', comment)
+ return comment.strip()
+
+def is_doc(comment):
+ """Test if comment is a C documentation comment."""
+ return comment.startswith('/**') and comment != '/**/'
+
+def nest(text, nest):
+ """
+ Indent documentation block for nesting.
+
+ Args:
+ text (str): Documentation body.
+ nest (int): Nesting level. For each level, the final block is indented
+ one level. Useful for (e.g.) declaring structure members.
+
+ Returns:
+ str: Indented reST documentation string.
+ """
+ return re.sub('(?m)^(?!$)', ' ' * nest, text)
+
+def generate(text, fmt=Type.TEXT, name=None,
+ ttype=None, args=None, transform=None):
+ """
+ Generate reST documentation string.
+
+ Args:
+ text (str): Documentation body.
+ fmt (enum :py:class:`Type`): Format type to use. Different formats
+ require different arguments and ignores others if given.
+ name (str): Name of the documented token.
+ ttype (str): Type of the documented token.
+ args (list): List of arguments (str).
+ transform (func): Transformation function to be applied to the
+ documentation body. This is useful (e.g.) to extend the generator
+ with different syntaxes by converting them to reST. This is applied
+ on the documentation body after removing comment markers.
+
+ Returns:
+ str: reST documentation string.
+ """
+
+ text = _strip(text)
+
+ if transform:
+ text = transform(text)
+
+ if args is not None:
+ args = ', '.join(args)
+
+ (text_indent, fmt) = _doc_fmt[fmt]
+ text = nest(text, text_indent)
+ return fmt.format(text=text, name=name, ttype=ttype, args=args)
--
GitLab