From 81ad11b60ee4049964af580c129f9b590b63977d Mon Sep 17 00:00:00 2001
From: Stan Lagun <slagun@mirantis.com>
Date: Thu, 10 Nov 2016 00:35:49 -0800
Subject: [PATCH] Merge of documenter and sphinx extension

- documenter script functionality was merged
   into sphinx extension
- heavy refactoring of the documenter code
- grouping of overload methods in documentation
- several minor fixes in doc strings

Change-Id: I9bccd6b1ff1750d966d8c39558d204fcaa4ad185
---
 doc/source/_exts/yaqlautodoc.py      | 179 +++++++++++++++++++++--
 doc/source/index.rst                 |   4 +-
 doc/source/standard_library.rst      |  11 +-
 yaql/contrib/documenter.py           | 207 ---------------------------
 yaql/standard_library/collections.py |   2 +-
 yaql/standard_library/common.py      |   4 +-
 6 files changed, 179 insertions(+), 228 deletions(-)
 delete mode 100644 yaql/contrib/documenter.py

diff --git a/doc/source/_exts/yaqlautodoc.py b/doc/source/_exts/yaqlautodoc.py
index e313e25..9fe286d 100644
--- a/doc/source/_exts/yaqlautodoc.py
+++ b/doc/source/_exts/yaqlautodoc.py
@@ -12,14 +12,177 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import os
-import subprocess
+import importlib
+import operator
+import pkgutil
+import traceback
+import types
 
+import six
 from docutils import nodes
 from docutils.parsers import rst
 from docutils import utils
 
 
+TAG = ':yaql:'
+
+
+def _get_modules_names(package):
+    """Get names of modules in package"""
+
+    return sorted(
+        map(operator.itemgetter(1),
+            pkgutil.walk_packages(package.__path__,
+                                  '{0}.'.format(package.__name__))))
+
+
+def _get_functions_names(module):
+    """Get names of the functions in the current module"""
+
+    return [name for name in dir(module) if
+            isinstance(getattr(module, name, None), types.FunctionType)]
+
+
+def write_method_doc(method, output):
+    """Construct method documentation from a docstring.
+
+    1) Strip TAG
+    2) Embolden function name
+    3) Add :callAs: after :signature:
+    """
+
+    msg = "Error: function {0} has no valid YAQL documentation."
+
+    if method.__doc__:
+        doc = method.__doc__
+        try:
+            # strip TAG
+            doc = doc[doc.index(TAG) + len(TAG):]
+
+            # embolden function name
+            line_break = doc.index('\n')
+            yaql_name = doc[:line_break]
+            (emit_header, is_overload) = yield yaql_name
+            if emit_header:
+                output.write(yaql_name)
+                output.write('\n')
+                output.write('~' * len(yaql_name))
+                output.write('\n')
+            doc = doc[line_break:]
+
+            # add :callAs: parameter
+            try:
+                signature_index = doc.index(':signature:')
+                position = doc.index('    :', signature_index +
+                                     len(':signature:'))
+                if hasattr(method, '__yaql_function__'):
+                    if (method.__yaql_function__.name and
+                            'operator' in method.__yaql_function__.name):
+                        call_as = 'operator'
+                    elif (method.__yaql_function__.is_function and
+                            method.__yaql_function__.is_method):
+                        call_as = 'function or method'
+                    elif method.__yaql_function__.is_method:
+                        call_as = 'method'
+                    else:
+                        call_as = 'function'
+                else:
+                    call_as = 'function'
+
+                call_as_str = '    :callAs: {0}\n'.format(call_as)
+                text = doc[:position] + call_as_str + doc[position:]
+            except ValueError:
+                text = doc
+            if is_overload:
+                text = '*  ' + '\n   '.join(text.split('\n'))
+                output.write(text)
+            else:
+                output.write(text)
+        except ValueError:
+            yield method.func_name
+            output.write(msg.format(method.func_name))
+
+
+def write_module_doc(module, output):
+    """Generate and write rst document for module.
+
+    Generate and write rst document for the single module.
+
+    :parameter module: takes a Python module which should be documented.
+    :type module: Python module
+
+    :parameter output: takes file to which generated document will be written.
+    :type output: file
+    """
+    functions_names = _get_functions_names(module)
+    if module.__doc__:
+        output.write(module.__doc__)
+        output.write('\n')
+    seq = []
+    for name in functions_names:
+        method = getattr(module, name)
+        it = write_method_doc(method, output)
+        try:
+            name = six.next(it)
+            seq.append((name, it))
+        except StopIteration:
+            pass
+    seq.sort(key=operator.itemgetter(0))
+    prev_name = None
+    for i, item in enumerate(seq):
+        name = item[0]
+        emit_header = name != prev_name
+        prev_name = name
+        if emit_header:
+            overload = i < len(seq) - 1 and seq[i + 1][0] == name
+        else:
+            overload = True
+
+        try:
+            item[1].send((emit_header, overload))
+        except StopIteration:
+            pass
+        output.write('\n\n')
+    output.write('\n')
+
+
+def write_package_doc(package, output):
+    """Writes rst document for the package.
+
+    Generate and write rst document for the modules in the given package.
+
+    :parameter package: takes a Python package which should be documented
+    :type package: Python module
+
+    :parameter output: takes file to which generated document will be written.
+    :type output: file
+    """
+
+    modules = _get_modules_names(package)
+    for module_name in modules:
+        module = importlib.import_module(module_name)
+        write_module_doc(module, output)
+
+
+def generate_doc(source):
+    try:
+        package = importlib.import_module(source)
+    except ImportError:
+        return 'Error: No such module {0}'.format(source)
+    out = six.StringIO()
+    try:
+        if hasattr(package, '__path__'):
+            write_package_doc(package, out)
+        else:
+            write_module_doc(package, out)
+        res = out.getvalue()
+        return res
+
+    except Exception as e:
+        return '.. code-block:: python\n\n    Error: {0}\n    {1}\n\n'.format(
+            str(e), '\n    '.join([''] + traceback.format_exc().split('\n')))
+
+
 class YaqlDocNode(nodes.General, nodes.Element):
     source = None
 
@@ -39,21 +202,11 @@ class YaqlDocDirective(rst.Directive):
 def render(app, doctree, fromdocname):
     for node in doctree.traverse(YaqlDocNode):
         new_doc = utils.new_document('YAQL', doctree.settings)
-        content = run_documenter(node.source)
+        content = generate_doc(node.source)
         rst.Parser().parse(content, new_doc)
         node.replace_self(new_doc.children)
 
 
-def run_documenter(source):
-    path = os.path.join(os.path.abspath('.'), 'yaql/contrib/documenter.py')
-    proc = subprocess.Popen(['python', path, source, '--no-header'],
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-    stdout, stderr = proc.communicate()
-    retcode = proc.poll()
-    return stdout if not retcode else stderr
-
-
 def setup(app):
     app.info('Loading the yaql documenter extension')
     app.add_node(YaqlDocNode)
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 6484f0d..d535594 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -3,8 +3,8 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to yaql's documentation!
-================================
+Welcome to yaql documentation!
+==============================
 
 Introduction
 ~~~~~~~~~~~~
diff --git a/doc/source/standard_library.rst b/doc/source/standard_library.rst
index 48eef8e..917ebfe 100644
--- a/doc/source/standard_library.rst
+++ b/doc/source/standard_library.rst
@@ -1,11 +1,16 @@
 Standard YAQL Library
 =====================
 
-Conditions and boolean logic functions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. yaqldoc:: yaql.standard_library.boolean
+Comparison operators
+~~~~~~~~~~~~~~~~~~~~
 .. yaqldoc:: yaql.standard_library.common
 
+
+Boolean logic functions
+~~~~~~~~~~~~~~~~~~~~~~~
+.. yaqldoc:: yaql.standard_library.boolean
+
+
 Working with collections
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/yaql/contrib/documenter.py b/yaql/contrib/documenter.py
deleted file mode 100644
index c031b68..0000000
--- a/yaql/contrib/documenter.py
+++ /dev/null
@@ -1,207 +0,0 @@
-#    Copyright (c) 2016 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import argparse
-import importlib
-import operator
-import pkgutil
-import sys
-import types
-
-
-TAG = ':yaql:'
-
-
-def _get_name(module):
-    """Get name of the module in the library directory"""
-
-    name_stub = module.__name__.split('.')
-    name = name_stub[-1].capitalize()
-
-    return name
-
-
-def _get_modules_names(package):
-    """Get names of modules in package"""
-
-    for _, name, _ in pkgutil.walk_packages(package.__path__,
-                                            '{0}.'.format(package.__name__)):
-        yield name
-
-
-def _get_functions_names(module):
-    """Get names of the functions in the current module"""
-
-    return [name for name in dir(module) if
-            isinstance(getattr(module, name, None),
-                       types.FunctionType)]
-
-
-def _construct_method_docs(method):
-    """Construct method documentation from a docstring.
-
-    1) Strip TAG
-    2) Embolden function name
-    3) Add :callAs: after :signature:
-    """
-
-    msg = "Function {0} has no valid YAQL documentation."
-
-    if method.__doc__:
-        doc = method.__doc__
-        try:
-            # strip TAG
-            doc = doc[doc.index(TAG) + len(TAG):]
-
-            # embolden function name
-            line_break = doc.index('\n')
-            doc = '**{0}**{1}'.format(doc[:line_break], doc[line_break:])
-
-            # add :callAs: parameter
-            signature_index = doc.index(':signature:')
-            position = doc.index('    :', signature_index +
-                                 len(':signature:'))
-
-            if hasattr(method, '__yaql_function__'):
-                if (method.__yaql_function__.name and
-                        'operator' in method.__yaql_function__.name):
-                    call_as = 'operator'
-                elif (method.__yaql_function__.is_function and
-                        method.__yaql_function__.is_method):
-                    call_as = 'function or method'
-                elif method.__yaql_function__.is_method:
-                    call_as = 'method'
-                else:
-                    call_as = 'function'
-            else:
-                call_as = 'function'
-
-            call_as_str = '    :callAs: {0}\n'.format(call_as)
-            return doc[:position] + call_as_str + doc[position:]
-        except ValueError:
-            return msg.format(method.func_name)
-
-
-def _get_functions_docs(module):
-    """Collect YAQL docstrings.
-
-    Collect functions docstrings after TAG.
-    """
-    functions_names = _get_functions_names(module)
-    if module.__doc__:
-        docs_list = [module.__doc__]
-        func_docs_list = []
-        for name in functions_names:
-            method = getattr(module, name)
-            method_docs = _construct_method_docs(method)
-            if method_docs:
-                func_docs_list.append('\n{0}\n'.format(method_docs))
-        func_docs_list.sort()
-        docs_list.extend(func_docs_list)
-    else:
-        docs_list = ['\nModule is not documented yet']
-    return docs_list
-
-
-def _add_markup(obj):
-    body = ''
-    subtitle = '{0} functions\n'.format(obj['module_name'])
-    markup = '{0}\n'.format('~' * (len(subtitle) - 1))
-    body = ''.join(obj['documentation'])
-    return '{0}{1}{2}\n\n'.format(subtitle, markup, body)
-
-
-def _write_to_doc(output, header, stub):
-    if header:
-        output.write("{0}\n{1}\n\n".format(header,
-                                           '=' * len(header)))
-    sorted_stub = sorted(stub, key=operator.itemgetter('module_name'))
-    for elem in sorted_stub:
-        if elem:
-            markuped = _add_markup(elem)
-            output.write(markuped)
-
-
-def generate_doc_for_module(module, output):
-    """Generate and write rst document for module.
-
-    Generate and write rst document for the single module. By default it will
-    print to stdout.
-
-    :parameter module: takes a Python module which should be documented.
-    :type module: Python module
-
-    :parameter output: takes file to which generated document will be written.
-    :type output: file
-    """
-    doc_stub = []
-    docs_for_module = _get_functions_docs(module)
-    doc_dict = {'module_name': _get_name(module),
-                'documentation': docs_for_module}
-    doc_stub.append(doc_dict)
-    doc_name = module.__name__.rsplit('.', 1)[-1]
-    doc_header = doc_name.replace("_", " ").capitalize()
-    _write_to_doc(output, doc_header, doc_stub)
-
-
-def generate_doc_for_package(package, output, no_header):
-    """Generate and write rst document for package.
-
-    Generate and write rst document for the modules in the given package. By
-    default it will print to stdout.
-
-    :parameter package: takes a Python package which should be documented
-    :type package: Python module
-
-    :parameter output: takes file to which generated document will be written.
-    :type output: file
-    """
-
-    modules = _get_modules_names(package)
-    doc_stub = []
-    for module_name in modules:
-        current_module = importlib.import_module(module_name)
-        docs_for_module = _get_functions_docs(current_module)
-        doc_dict = {'module_name': _get_name(current_module),
-                    'documentation': docs_for_module}
-        doc_stub.append(doc_dict)
-    if no_header:
-        doc_header = None
-    else:
-        doc_name = package.__name__.rsplit('.', 1)[-1]
-        doc_header = doc_name.replace("_", " ").capitalize()
-    _write_to_doc(output, doc_header, doc_stub)
-
-
-def main(args):
-    try:
-        package = importlib.import_module(args.package)
-    except ImportError:
-        raise ValueError("No such package {0}".format(args.package))
-    try:
-        getattr(package, '__path__')
-        generate_doc_for_package(package, args.output, args.no_header)
-    except AttributeError:
-        generate_doc_for_module(package, args.output)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument('package', help="A package/module to be documented")
-    parser.add_argument('--output', help="A file to output",
-                        type=argparse.FileType('w'), default=sys.stdout)
-    parser.add_argument('--no-header', help="Do not generate package header",
-                        action='store_true')
-    args = parser.parse_args()
-    main(args)
diff --git a/yaql/standard_library/collections.py b/yaql/standard_library/collections.py
index 3969b10..8965914 100644
--- a/yaql/standard_library/collections.py
+++ b/yaql/standard_library/collections.py
@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 """
-Main functions for collection module.
+Functions that produce or consume finite collections - lists, dicts and sets.
 """
 
 import itertools
diff --git a/yaql/standard_library/common.py b/yaql/standard_library/common.py
index 91c2a03..76f429f 100644
--- a/yaql/standard_library/common.py
+++ b/yaql/standard_library/common.py
@@ -21,7 +21,7 @@ from yaql.language import specs
 
 @specs.name('*equal')
 def eq(left, right):
-    """:yaql:equal
+    """:yaql:operator =
 
     Returns true if left and right are equal, false otherwise.
 
@@ -33,7 +33,7 @@ def eq(left, right):
 
 @specs.name('*not_equal')
 def neq(left, right):
-    """:yaql:notEqual
+    """:yaql:operator !=
 
     Returns true if left and right are not equal, false otherwise.