Merge of documenter and sphinx extension
- documenter script functionality was merged into sphinx extension - heavy refactoring of the documenter code - grouping of overload methods in documentation - several minor fixes in doc strings Change-Id: I9bccd6b1ff1750d966d8c39558d204fcaa4ad185
This commit is contained in:
Stan Lagun
parent
fa798e1e15
commit
81ad11b60e
@ -12,14 +12,177 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import importlib
|
||||
import operator
|
||||
import pkgutil
|
||||
import traceback
|
||||
import types
|
||||
|
||||
import six
|
||||
from docutils import nodes
|
||||
from docutils.parsers import rst
|
||||
from docutils import utils
|
||||
|
||||
|
||||
TAG = ':yaql:'
|
||||
|
||||
|
||||
def _get_modules_names(package):
|
||||
"""Get names of modules in package"""
|
||||
|
||||
return sorted(
|
||||
map(operator.itemgetter(1),
|
||||
pkgutil.walk_packages(package.__path__,
|
||||
'{0}.'.format(package.__name__))))
|
||||
|
||||
|
||||
def _get_functions_names(module):
|
||||
"""Get names of the functions in the current module"""
|
||||
|
||||
return [name for name in dir(module) if
|
||||
isinstance(getattr(module, name, None), types.FunctionType)]
|
||||
|
||||
|
||||
def write_method_doc(method, output):
|
||||
"""Construct method documentation from a docstring.
|
||||
|
||||
1) Strip TAG
|
||||
2) Embolden function name
|
||||
3) Add :callAs: after :signature:
|
||||
"""
|
||||
|
||||
msg = "Error: function {0} has no valid YAQL documentation."
|
||||
|
||||
if method.__doc__:
|
||||
doc = method.__doc__
|
||||
try:
|
||||
# strip TAG
|
||||
doc = doc[doc.index(TAG) + len(TAG):]
|
||||
|
||||
# embolden function name
|
||||
line_break = doc.index('\n')
|
||||
yaql_name = doc[:line_break]
|
||||
(emit_header, is_overload) = yield yaql_name
|
||||
if emit_header:
|
||||
output.write(yaql_name)
|
||||
output.write('\n')
|
||||
output.write('~' * len(yaql_name))
|
||||
output.write('\n')
|
||||
doc = doc[line_break:]
|
||||
|
||||
# add :callAs: parameter
|
||||
try:
|
||||
signature_index = doc.index(':signature:')
|
||||
position = doc.index(' :', signature_index +
|
||||
len(':signature:'))
|
||||
if hasattr(method, '__yaql_function__'):
|
||||
if (method.__yaql_function__.name and
|
||||
'operator' in method.__yaql_function__.name):
|
||||
call_as = 'operator'
|
||||
elif (method.__yaql_function__.is_function and
|
||||
method.__yaql_function__.is_method):
|
||||
call_as = 'function or method'
|
||||
elif method.__yaql_function__.is_method:
|
||||
call_as = 'method'
|
||||
else:
|
||||
call_as = 'function'
|
||||
else:
|
||||
call_as = 'function'
|
||||
|
||||
call_as_str = ' :callAs: {0}\n'.format(call_as)
|
||||
text = doc[:position] + call_as_str + doc[position:]
|
||||
except ValueError:
|
||||
text = doc
|
||||
if is_overload:
|
||||
text = '* ' + '\n '.join(text.split('\n'))
|
||||
output.write(text)
|
||||
else:
|
||||
output.write(text)
|
||||
except ValueError:
|
||||
yield method.func_name
|
||||
output.write(msg.format(method.func_name))
|
||||
|
||||
|
||||
def write_module_doc(module, output):
|
||||
"""Generate and write rst document for module.
|
||||
|
||||
Generate and write rst document for the single module.
|
||||
|
||||
:parameter module: takes a Python module which should be documented.
|
||||
:type module: Python module
|
||||
|
||||
:parameter output: takes file to which generated document will be written.
|
||||
:type output: file
|
||||
"""
|
||||
functions_names = _get_functions_names(module)
|
||||
if module.__doc__:
|
||||
output.write(module.__doc__)
|
||||
output.write('\n')
|
||||
seq = []
|
||||
for name in functions_names:
|
||||
method = getattr(module, name)
|
||||
it = write_method_doc(method, output)
|
||||
try:
|
||||
name = six.next(it)
|
||||
seq.append((name, it))
|
||||
except StopIteration:
|
||||
pass
|
||||
seq.sort(key=operator.itemgetter(0))
|
||||
prev_name = None
|
||||
for i, item in enumerate(seq):
|
||||
name = item[0]
|
||||
emit_header = name != prev_name
|
||||
prev_name = name
|
||||
if emit_header:
|
||||
overload = i < len(seq) - 1 and seq[i + 1][0] == name
|
||||
else:
|
||||
overload = True
|
||||
|
||||
try:
|
||||
item[1].send((emit_header, overload))
|
||||
except StopIteration:
|
||||
pass
|
||||
output.write('\n\n')
|
||||
output.write('\n')
|
||||
|
||||
|
||||
def write_package_doc(package, output):
|
||||
"""Writes rst document for the package.
|
||||
|
||||
Generate and write rst document for the modules in the given package.
|
||||
|
||||
:parameter package: takes a Python package which should be documented
|
||||
:type package: Python module
|
||||
|
||||
:parameter output: takes file to which generated document will be written.
|
||||
:type output: file
|
||||
"""
|
||||
|
||||
modules = _get_modules_names(package)
|
||||
for module_name in modules:
|
||||
module = importlib.import_module(module_name)
|
||||
write_module_doc(module, output)
|
||||
|
||||
|
||||
def generate_doc(source):
|
||||
try:
|
||||
package = importlib.import_module(source)
|
||||
except ImportError:
|
||||
return 'Error: No such module {0}'.format(source)
|
||||
out = six.StringIO()
|
||||
try:
|
||||
if hasattr(package, '__path__'):
|
||||
write_package_doc(package, out)
|
||||
else:
|
||||
write_module_doc(package, out)
|
||||
res = out.getvalue()
|
||||
return res
|
||||
|
||||
except Exception as e:
|
||||
return '.. code-block:: python\n\n Error: {0}\n {1}\n\n'.format(
|
||||
str(e), '\n '.join([''] + traceback.format_exc().split('\n')))
|
||||
|
||||
|
||||
class YaqlDocNode(nodes.General, nodes.Element):
|
||||
source = None
|
||||
|
||||
@ -39,21 +202,11 @@ class YaqlDocDirective(rst.Directive):
|
||||
def render(app, doctree, fromdocname):
|
||||
for node in doctree.traverse(YaqlDocNode):
|
||||
new_doc = utils.new_document('YAQL', doctree.settings)
|
||||
content = run_documenter(node.source)
|
||||
content = generate_doc(node.source)
|
||||
rst.Parser().parse(content, new_doc)
|
||||
node.replace_self(new_doc.children)
|
||||
|
||||
|
||||
def run_documenter(source):
|
||||
path = os.path.join(os.path.abspath('.'), 'yaql/contrib/documenter.py')
|
||||
proc = subprocess.Popen(['python', path, source, '--no-header'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
stdout, stderr = proc.communicate()
|
||||
retcode = proc.poll()
|
||||
return stdout if not retcode else stderr
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.info('Loading the yaql documenter extension')
|
||||
app.add_node(YaqlDocNode)
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
You can adapt this file completely to your liking, but it should at least
|
||||
contain the root `toctree` directive.
|
||||
|
||||
Welcome to yaql's documentation!
|
||||
================================
|
||||
Welcome to yaql documentation!
|
||||
==============================
|
||||
|
||||
Introduction
|
||||
~~~~~~~~~~~~
|
||||
|
||||
@ -1,11 +1,16 @@
|
||||
Standard YAQL Library
|
||||
=====================
|
||||
|
||||
Conditions and boolean logic functions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
.. yaqldoc:: yaql.standard_library.boolean
|
||||
Comparison operators
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
.. yaqldoc:: yaql.standard_library.common
|
||||
|
||||
|
||||
Boolean logic functions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
.. yaqldoc:: yaql.standard_library.boolean
|
||||
|
||||
|
||||
Working with collections
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@ -1,207 +0,0 @@
|
||||
# Copyright (c) 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import importlib
|
||||
import operator
|
||||
import pkgutil
|
||||
import sys
|
||||
import types
|
||||
|
||||
|
||||
TAG = ':yaql:'
|
||||
|
||||
|
||||
def _get_name(module):
|
||||
"""Get name of the module in the library directory"""
|
||||
|
||||
name_stub = module.__name__.split('.')
|
||||
name = name_stub[-1].capitalize()
|
||||
|
||||
return name
|
||||
|
||||
|
||||
def _get_modules_names(package):
|
||||
"""Get names of modules in package"""
|
||||
|
||||
for _, name, _ in pkgutil.walk_packages(package.__path__,
|
||||
'{0}.'.format(package.__name__)):
|
||||
yield name
|
||||
|
||||
|
||||
def _get_functions_names(module):
|
||||
"""Get names of the functions in the current module"""
|
||||
|
||||
return [name for name in dir(module) if
|
||||
isinstance(getattr(module, name, None),
|
||||
types.FunctionType)]
|
||||
|
||||
|
||||
def _construct_method_docs(method):
|
||||
"""Construct method documentation from a docstring.
|
||||
|
||||
1) Strip TAG
|
||||
2) Embolden function name
|
||||
3) Add :callAs: after :signature:
|
||||
"""
|
||||
|
||||
msg = "Function {0} has no valid YAQL documentation."
|
||||
|
||||
if method.__doc__:
|
||||
doc = method.__doc__
|
||||
try:
|
||||
# strip TAG
|
||||
doc = doc[doc.index(TAG) + len(TAG):]
|
||||
|
||||
# embolden function name
|
||||
line_break = doc.index('\n')
|
||||
doc = '**{0}**{1}'.format(doc[:line_break], doc[line_break:])
|
||||
|
||||
# add :callAs: parameter
|
||||
signature_index = doc.index(':signature:')
|
||||
position = doc.index(' :', signature_index +
|
||||
len(':signature:'))
|
||||
|
||||
if hasattr(method, '__yaql_function__'):
|
||||
if (method.__yaql_function__.name and
|
||||
'operator' in method.__yaql_function__.name):
|
||||
call_as = 'operator'
|
||||
elif (method.__yaql_function__.is_function and
|
||||
method.__yaql_function__.is_method):
|
||||
call_as = 'function or method'
|
||||
elif method.__yaql_function__.is_method:
|
||||
call_as = 'method'
|
||||
else:
|
||||
call_as = 'function'
|
||||
else:
|
||||
call_as = 'function'
|
||||
|
||||
call_as_str = ' :callAs: {0}\n'.format(call_as)
|
||||
return doc[:position] + call_as_str + doc[position:]
|
||||
except ValueError:
|
||||
return msg.format(method.func_name)
|
||||
|
||||
|
||||
def _get_functions_docs(module):
|
||||
"""Collect YAQL docstrings.
|
||||
|
||||
Collect functions docstrings after TAG.
|
||||
"""
|
||||
functions_names = _get_functions_names(module)
|
||||
if module.__doc__:
|
||||
docs_list = [module.__doc__]
|
||||
func_docs_list = []
|
||||
for name in functions_names:
|
||||
method = getattr(module, name)
|
||||
method_docs = _construct_method_docs(method)
|
||||
if method_docs:
|
||||
func_docs_list.append('\n{0}\n'.format(method_docs))
|
||||
func_docs_list.sort()
|
||||
docs_list.extend(func_docs_list)
|
||||
else:
|
||||
docs_list = ['\nModule is not documented yet']
|
||||
return docs_list
|
||||
|
||||
|
||||
def _add_markup(obj):
|
||||
body = ''
|
||||
subtitle = '{0} functions\n'.format(obj['module_name'])
|
||||
markup = '{0}\n'.format('~' * (len(subtitle) - 1))
|
||||
body = ''.join(obj['documentation'])
|
||||
return '{0}{1}{2}\n\n'.format(subtitle, markup, body)
|
||||
|
||||
|
||||
def _write_to_doc(output, header, stub):
|
||||
if header:
|
||||
output.write("{0}\n{1}\n\n".format(header,
|
||||
'=' * len(header)))
|
||||
sorted_stub = sorted(stub, key=operator.itemgetter('module_name'))
|
||||
for elem in sorted_stub:
|
||||
if elem:
|
||||
markuped = _add_markup(elem)
|
||||
output.write(markuped)
|
||||
|
||||
|
||||
def generate_doc_for_module(module, output):
|
||||
"""Generate and write rst document for module.
|
||||
|
||||
Generate and write rst document for the single module. By default it will
|
||||
print to stdout.
|
||||
|
||||
:parameter module: takes a Python module which should be documented.
|
||||
:type module: Python module
|
||||
|
||||
:parameter output: takes file to which generated document will be written.
|
||||
:type output: file
|
||||
"""
|
||||
doc_stub = []
|
||||
docs_for_module = _get_functions_docs(module)
|
||||
doc_dict = {'module_name': _get_name(module),
|
||||
'documentation': docs_for_module}
|
||||
doc_stub.append(doc_dict)
|
||||
doc_name = module.__name__.rsplit('.', 1)[-1]
|
||||
doc_header = doc_name.replace("_", " ").capitalize()
|
||||
_write_to_doc(output, doc_header, doc_stub)
|
||||
|
||||
|
||||
def generate_doc_for_package(package, output, no_header):
|
||||
"""Generate and write rst document for package.
|
||||
|
||||
Generate and write rst document for the modules in the given package. By
|
||||
default it will print to stdout.
|
||||
|
||||
:parameter package: takes a Python package which should be documented
|
||||
:type package: Python module
|
||||
|
||||
:parameter output: takes file to which generated document will be written.
|
||||
:type output: file
|
||||
"""
|
||||
|
||||
modules = _get_modules_names(package)
|
||||
doc_stub = []
|
||||
for module_name in modules:
|
||||
current_module = importlib.import_module(module_name)
|
||||
docs_for_module = _get_functions_docs(current_module)
|
||||
doc_dict = {'module_name': _get_name(current_module),
|
||||
'documentation': docs_for_module}
|
||||
doc_stub.append(doc_dict)
|
||||
if no_header:
|
||||
doc_header = None
|
||||
else:
|
||||
doc_name = package.__name__.rsplit('.', 1)[-1]
|
||||
doc_header = doc_name.replace("_", " ").capitalize()
|
||||
_write_to_doc(output, doc_header, doc_stub)
|
||||
|
||||
|
||||
def main(args):
|
||||
try:
|
||||
package = importlib.import_module(args.package)
|
||||
except ImportError:
|
||||
raise ValueError("No such package {0}".format(args.package))
|
||||
try:
|
||||
getattr(package, '__path__')
|
||||
generate_doc_for_package(package, args.output, args.no_header)
|
||||
except AttributeError:
|
||||
generate_doc_for_module(package, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('package', help="A package/module to be documented")
|
||||
parser.add_argument('--output', help="A file to output",
|
||||
type=argparse.FileType('w'), default=sys.stdout)
|
||||
parser.add_argument('--no-header', help="Do not generate package header",
|
||||
action='store_true')
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@ -12,7 +12,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
"""
|
||||
Main functions for collection module.
|
||||
Functions that produce or consume finite collections - lists, dicts and sets.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
|
||||
@ -21,7 +21,7 @@ from yaql.language import specs
|
||||
|
||||
@specs.name('*equal')
|
||||
def eq(left, right):
|
||||
""":yaql:equal
|
||||
""":yaql:operator =
|
||||
|
||||
Returns true if left and right are equal, false otherwise.
|
||||
|
||||
@ -33,7 +33,7 @@ def eq(left, right):
|
||||
|
||||
@specs.name('*not_equal')
|
||||
def neq(left, right):
|
||||
""":yaql:notEqual
|
||||
""":yaql:operator !=
|
||||
|
||||
Returns true if left and right are not equal, false otherwise.
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user