Update config generator from Oslo

This gets rid of the unnecessary option count at the end of the
file and ensures that the config groups are ordered in a repeatable
way so adding an option won't shuffle the entire sample config.

The new config generator uses stevedore so I added that to
test-requirements too.

Change-Id: I5fa0e21ca19a4736ebb3ea9c31b683be5324e052

etc/tuskar/tuskar.conf.sample

@@ -174,10 +174,12 @@
 
 # (Optional) Name of log file to output to. If no default is
 # set, logging will go to stdout. (string value)
+# Deprecated group/name - [DEFAULT]/logfile
 #log_file=<None>
 
 # (Optional) The base directory used for relative --log-file
 # paths (string value)
+# Deprecated group/name - [DEFAULT]/logdir
 #log_dir=<None>
 
 # Use syslog for logging. (boolean value)
@@ -410,10 +412,12 @@
 #
 
 # The backend to use for db (string value)
+# Deprecated group/name - [DEFAULT]/db_backend
 #backend=sqlalchemy
 
 # Enable the experimental use of thread pooling for all DB API
 # calls (boolean value)
+# Deprecated group/name - [DEFAULT]/dbapi_use_tpool
 #use_tpool=false
 
 
@@ -423,41 +427,66 @@
 
 # The SQLAlchemy connection string used to connect to the
 # database (string value)
+# Deprecated group/name - [DEFAULT]/sql_connection
 #connection=sqlite:////tuskar/openstack/common/db/$sqlite_db
 
 # timeout before idle sql connections are reaped (integer
 # value)
+# Deprecated group/name - [DEFAULT]/sql_idle_timeout
 #idle_timeout=3600
 
 # Minimum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_min_pool_size
 #min_pool_size=1
 
 # Maximum number of SQL connections to keep open in a pool
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_pool_size
 #max_pool_size=5
 
 # maximum db connection retries during startup. (setting -1
 # implies an infinite retry count) (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_retries
 #max_retries=10
 
 # interval between retries of opening a sql connection
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_retry_interval
 #retry_interval=10
 
 # If set, use this value for max_overflow with sqlalchemy
 # (integer value)
+# Deprecated group/name - [DEFAULT]/sql_max_overflow
 #max_overflow=<None>
 
 # Verbosity of SQL debugging information. 0=None,
 # 100=Everything (integer value)
+# Deprecated group/name - [DEFAULT]/sql_connection_debug
 #connection_debug=0
 
 # Add python stack traces to SQL as comment strings (boolean
 # value)
+# Deprecated group/name - [DEFAULT]/sql_connection_trace
 #connection_trace=false
 
 
+[heat]
+
+#
+# Options defined in tuskar.heat.client
+#
+
+# Name of the overcloud Heat stack (string value)
+#stack_name=overcloud
+
+# Heat API service type registered in keystone (string value)
+#service_type=orchestration
+
+# Heat API service endpoint type in keystone (string value)
+#endpoint_type=publicURL
+
+
 [heat_keystone]
 
 #
@@ -483,32 +512,6 @@
 #insecure=true
 
 
-[rpc_notifier2]
-
-#
-# Options defined in tuskar.openstack.common.notifier.rpc_notifier2
-#
-
-# AMQP topic(s) used for openstack notifications (list value)
-#topics=notifications
-
-
-[heat]
-
-#
-# Options defined in tuskar.heat.client
-#
-
-# Name of the overcloud Heat stack (string value)
-#stack_name=overcloud
-
-# Heat API service type registered in keystone (string value)
-#service_type=orchestration
-
-# Heat API service endpoint type in keystone (string value)
-#endpoint_type=publicURL
-
-
 [matchmaker_redis]
 
 #
@@ -532,7 +535,17 @@
 #
 
 # Matchmaker ring file (JSON) (string value)
+# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
 #ringfile=/etc/oslo/matchmaker_ring.json
 
 
-# Total option count: 109
+[rpc_notifier2]
+
+#
+# Options defined in tuskar.openstack.common.notifier.rpc_notifier2
+#
+
+# AMQP topic(s) used for openstack notifications (list value)
+#topics=notifications
+
+

test-requirements.txt

@@ -8,6 +8,7 @@ mock>=1.0
 mox>=0.5.3
 MySQL-python
 python-subunit
+stevedore>=0.14
 testrepository>=0.0.17
 testtools>=0.9.32
 unittest2

tools/config/check_uptodate.sh

@@ -1,9 +1,25 @@
-#!/bin/sh
+#!/usr/bin/env bash
-TEMPDIR=`mktemp -d`
+
-CFGFILE=tuskar.conf.sample
+PROJECT_NAME=${PROJECT_NAME:-tuskar}
-tools/config/generate_sample.sh -b ./ -p tuskar -o $TEMPDIR
+CFGFILE_NAME=${PROJECT_NAME}.conf.sample
-if ! diff $TEMPDIR/$CFGFILE etc/tuskar/$CFGFILE
+
-then
+if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then
-    echo "E: tuskar.conf.sample is not up to date, please run tools/config/generate_sample.sh"
+    CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME}
-    exit 42
+elif [ -e etc/${CFGFILE_NAME} ]; then
+    CFGFILE=etc/${CFGFILE_NAME}
+else
+    echo "${0##*/}: can not find config file"
+    exit 1
+fi
+
+TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX`
+trap "rm -rf $TEMPDIR" EXIT
+
+tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR}
+
+if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE}
+then
+   echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date."
+   echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh."
+   exit 1
 fi

tools/config/generate_sample.sh

@@ -4,8 +4,8 @@ print_hint() {
     echo "Try \`${0##*/} --help' for more information." >&2
 }
 
-PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \
+PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \
-                 --long help,base-dir:,package-name:,output-dir: -- "$@")
+                 --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@")
 
 if [ $? != 0 ] ; then print_hint ; exit 1 ; fi
 
@@ -21,6 +21,8 @@ while true; do
             echo "-b, --base-dir=DIR        project base directory"
             echo "-p, --package-name=NAME   project package name"
             echo "-o, --output-dir=DIR      file output directory"
+            echo "-m, --module=MOD          extra python module to interrogate for options"
+            echo "-l, --library=LIB         extra library that registers options for discovery"
             exit 0
             ;;
         -b|--base-dir)
@@ -38,6 +40,16 @@ while true; do
             OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'`
             shift
             ;;
+        -m|--module)
+            shift
+            MODULES="$MODULES -m $1"
+            shift
+            ;;
+        -l|--library)
+            shift
+            LIBRARIES="$LIBRARIES -l $1"
+            shift
+            ;;
         --)
             break
             ;;
@@ -77,12 +89,20 @@ find $TARGETDIR -type f -name "*.pyc" -delete
 FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \
         -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u)
 
-EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc"
+RC_FILE="`dirname $0`/oslo.config.generator.rc"
-if test -r "$EXTRA_MODULES_FILE"
+if test -r "$RC_FILE"
 then
-    source "$EXTRA_MODULES_FILE"
+    source "$RC_FILE"
 fi
 
+for mod in ${TUSKAR_CONFIG_GENERATOR_EXTRA_MODULES}; do
+    MODULES="$MODULES -m $mod"
+done
+
+for lib in ${TUSKAR_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do
+    LIBRARIES="$LIBRARIES -l $lib"
+done
+
 export EVENTLET_NO_GREENDNS=yes
 
 OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
@@ -90,7 +110,7 @@ OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs)
 DEFAULT_MODULEPATH=tuskar.openstack.common.config.generator
 MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH}
 OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample
-python -m $MODULEPATH $FILES > $OUTPUTFILE
+python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE
 
 # Hook to allow projects to append custom config file snippets
 CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null)

tuskar/openstack/common/config/generator.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 SINA Corporation
+# Copyright 2014 Cisco Systems, Inc.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -16,10 +14,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-# @author: Zhongyue Luo, SINA Corporation.
+
-#
 """Extracts OpenStack config option info from module(s)."""
 
+from __future__ import print_function
+
+import argparse
 import imp
 import os
 import re
@@ -28,6 +28,8 @@ import sys
 import textwrap
 
 from oslo.config import cfg
+import six
+import stevedore.named
 
 from tuskar.openstack.common import gettextutils
 from tuskar.openstack.common import importutils
@@ -39,6 +41,7 @@ BOOLOPT = "BoolOpt"
 INTOPT = "IntOpt"
 FLOATOPT = "FloatOpt"
 LISTOPT = "ListOpt"
+DICTOPT = "DictOpt"
 MULTISTROPT = "MultiStrOpt"
 
 OPT_TYPES = {
@@ -47,12 +50,12 @@ OPT_TYPES = {
     INTOPT: 'integer value',
     FLOATOPT: 'floating point value',
     LISTOPT: 'list value',
+    DICTOPT: 'dict value',
     MULTISTROPT: 'multi valued',
 }
 
-OPTION_COUNT = 0
 OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
-                                              FLOATOPT, LISTOPT,
+                                              FLOATOPT, LISTOPT, DICTOPT,
                                               MULTISTROPT]))
 
 PY_EXT = ".py"
@@ -61,24 +64,55 @@ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
 WORDWRAP_WIDTH = 60
 
 
-def generate(srcfiles):
+def generate(argv):
+    parser = argparse.ArgumentParser(
+        description='generate sample configuration file',
+    )
+    parser.add_argument('-m', dest='modules', action='append')
+    parser.add_argument('-l', dest='libraries', action='append')
+    parser.add_argument('srcfiles', nargs='*')
+    parsed_args = parser.parse_args(argv)
+
     mods_by_pkg = dict()
-    for filepath in srcfiles:
+    for filepath in parsed_args.srcfiles:
         pkg_name = filepath.split(os.sep)[1]
         mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
                             os.path.basename(filepath).split('.')[0]])
         mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
     # NOTE(lzyeval): place top level modules before packages
-    pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
+    pkg_names = sorted(pkg for pkg in mods_by_pkg if pkg.endswith(PY_EXT))
-    pkg_names.sort()
+    ext_names = sorted(pkg for pkg in mods_by_pkg if pkg not in pkg_names)
-    ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
-    ext_names.sort()
     pkg_names.extend(ext_names)
 
     # opts_by_group is a mapping of group name to an options list
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
+    if parsed_args.modules:
+        for module_name in parsed_args.modules:
+            module = _import_module(module_name)
+            if module:
+                for group, opts in _list_opts(module):
+                    opts_by_group.setdefault(group, []).append((module_name,
+                                                                opts))
+
+    # Look for entry points defined in libraries (or applications) for
+    # option discovery, and include their return values in the output.
+    #
+    # Each entry point should be a function returning an iterable
+    # of pairs with the group name (or None for the default group)
+    # and the list of Opt instances for that group.
+    if parsed_args.libraries:
+        loader = stevedore.named.NamedExtensionManager(
+            'oslo.config.opts',
+            names=list(set(parsed_args.libraries)),
+            invoke_on_load=False,
+        )
+        for ext in loader:
+            for group, opts in ext.plugin():
+                opt_list = opts_by_group.setdefault(group or 'DEFAULT', [])
+                opt_list.append((ext.name, opts))
+
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
         mods.sort()
@@ -88,16 +122,14 @@ def generate(srcfiles):
 
             mod_obj = _import_module(mod_str)
             if not mod_obj:
-                continue
+                raise RuntimeError("Unable to import module %s" % mod_str)
 
             for group, opts in _list_opts(mod_obj):
                 opts_by_group.setdefault(group, []).append((mod_str, opts))
 
     print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
-    for group, opts in opts_by_group.items():
+    for group in sorted(opts_by_group.keys()):
-        print_group_opts(group, opts)
+        print_group_opts(group, opts_by_group[group])
-
-    print "# Total option count: %d" % OPTION_COUNT
 
 
 def _import_module(mod_str):
@@ -107,17 +139,17 @@ def _import_module(mod_str):
             return sys.modules[mod_str[4:]]
         else:
             return importutils.import_module(mod_str)
-    except ImportError as ie:
+    except Exception as e:
-        sys.stderr.write("%s\n" % str(ie))
+        sys.stderr.write("Error importing module %s: %s\n" % (mod_str, str(e)))
-        return None
-    except Exception:
         return None
 
 
 def _is_in_group(opt, group):
     "Check if opt is in group."
-    for key, value in group._opts.items():
+    for value in group._opts.values():
-        if value['opt'] == opt:
+        # NOTE(llu): Temporary workaround for bug #1262148, wait until
+        # newly released oslo.config support '==' operator.
+        if not(value['opt'] != opt):
             return True
     return False
 
@@ -128,7 +160,7 @@ def _guess_groups(opt, mod_obj):
         return 'DEFAULT'
 
     # what other groups is it in?
-    for key, value in cfg.CONF.items():
+    for value in cfg.CONF.values():
         if isinstance(value, cfg.CONF.GroupAttr):
             if _is_in_group(opt, value._group):
                 return value._group.name
@@ -161,18 +193,16 @@ def _list_opts(obj):
 
 
 def print_group_opts(group, opts_by_module):
-    print "[%s]" % group
+    print("[%s]" % group)
-    print
+    print('')
-    global OPTION_COUNT
     for mod, opts in opts_by_module:
-        OPTION_COUNT += len(opts)
+        print('#')
-        print '#'
+        print('# Options defined in %s' % mod)
-        print '# Options defined in %s' % mod
+        print('#')
-        print '#'
+        print('')
-        print
         for opt in opts:
             _print_opt(opt)
-        print
+        print('')
 
 
 def _get_my_ip():
@@ -186,25 +216,31 @@ def _get_my_ip():
         return None
 
 
-def _sanitize_default(s):
+def _sanitize_default(name, value):
     """Set up a reasonably sensible default for pybasedir, my_ip and host."""
-    if s.startswith(BASEDIR):
+    if value.startswith(sys.prefix):
-        return s.replace(BASEDIR, '/usr/lib/python/site-packages')
+        # NOTE(jd) Don't use os.path.join, because it is likely to think the
-    elif BASEDIR in s:
+        # second part is an absolute pathname and therefore drop the first
-        return s.replace(BASEDIR, '')
+        # part.
-    elif s == _get_my_ip():
+        value = os.path.normpath("/usr/" + value[len(sys.prefix):])
+    elif value.startswith(BASEDIR):
+        return value.replace(BASEDIR, '/usr/lib/python/site-packages')
+    elif BASEDIR in value:
+        return value.replace(BASEDIR, '')
+    elif value == _get_my_ip():
         return '10.0.0.1'
-    elif s == socket.gethostname():
+    elif value == socket.gethostname() and 'host' in name:
         return 'tuskar'
-    elif s.strip() != s:
+    elif value.strip() != value:
-        return '"%s"' % s
+        return '"%s"' % value
-    return s
+    return value
 
 
 def _print_opt(opt):
     opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help
     if not opt_help:
         sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
+        opt_help = ""
     opt_type = None
     try:
         opt_type = OPTION_REGEX.search(str(type(opt))).group(0)
@@ -212,42 +248,53 @@ def _print_opt(opt):
         sys.stderr.write("%s\n" % str(err))
         sys.exit(1)
     opt_help += ' (' + OPT_TYPES[opt_type] + ')'
-    print '#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))
+    print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
+    if opt.deprecated_opts:
+        for deprecated_opt in opt.deprecated_opts:
+            if deprecated_opt.name:
+                deprecated_group = (deprecated_opt.group if
+                                    deprecated_opt.group else "DEFAULT")
+                print('# Deprecated group/name - [%s]/%s' %
+                      (deprecated_group,
+                       deprecated_opt.name))
     try:
         if opt_default is None:
-            print '#%s=<None>' % opt_name
+            print('#%s=<None>' % opt_name)
         elif opt_type == STROPT:
-            assert(isinstance(opt_default, basestring))
+            assert(isinstance(opt_default, six.string_types))
-            print '#%s=%s' % (opt_name, _sanitize_default(opt_default))
+            print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
+                                                          opt_default)))
         elif opt_type == BOOLOPT:
             assert(isinstance(opt_default, bool))
-            print '#%s=%s' % (opt_name, str(opt_default).lower())
+            print('#%s=%s' % (opt_name, str(opt_default).lower()))
         elif opt_type == INTOPT:
             assert(isinstance(opt_default, int) and
                    not isinstance(opt_default, bool))
-            print '#%s=%s' % (opt_name, opt_default)
+            print('#%s=%s' % (opt_name, opt_default))
         elif opt_type == FLOATOPT:
             assert(isinstance(opt_default, float))
-            print '#%s=%s' % (opt_name, opt_default)
+            print('#%s=%s' % (opt_name, opt_default))
         elif opt_type == LISTOPT:
             assert(isinstance(opt_default, list))
-            print '#%s=%s' % (opt_name, ','.join(opt_default))
+            print('#%s=%s' % (opt_name, ','.join(opt_default)))
+        elif opt_type == DICTOPT:
+            assert(isinstance(opt_default, dict))
+            opt_default_strlist = [str(key) + ':' + str(value)
+                                   for (key, value) in opt_default.items()]
+            print('#%s=%s' % (opt_name, ','.join(opt_default_strlist)))
         elif opt_type == MULTISTROPT:
             assert(isinstance(opt_default, list))
             if not opt_default:
                 opt_default = ['']
             for default in opt_default:
-                print '#%s=%s' % (opt_name, default)
+                print('#%s=%s' % (opt_name, default))
-        print
+        print('')
     except Exception:
         sys.stderr.write('Error in option "%s"\n' % opt_name)
         sys.exit(1)
 
 
 def main():
-    if len(sys.argv) < 2:
-        print "usage: %s [srcfile]...\n" % sys.argv[0]
-        sys.exit(0)
     generate(sys.argv[1:])
 
 if __name__ == '__main__':