Add oslo.log library

Add oslo.log lib in application.
Move API related options from refstack/api/config.py
to the oslo_config's configuration file.
Make regeneration of example config file with oslo.log namespace
and with moved API options.

Change-Id: Id0db95fe34eb8faa284e9d716cbeebdf6d25c183

bin/refstack-manage

@@ -20,12 +20,16 @@ Command-line utility for database manage
 
 import sys
 
-from oslo.config import cfg
+from oslo_config import cfg
+from oslo_log import log
 
 from refstack.db import migration
 
+LOG = log.getLogger(__name__)
 CONF = cfg.CONF
 
+log.register_options(CONF)
+
 
 class DatabaseManager(object):
 
@@ -91,4 +95,5 @@ CONF.register_cli_opt(command_opt)
 
 if __name__ == '__main__':
     CONF(sys.argv[1:], project='refstack')
+    log.setup(CONF, 'refstack')
     CONF.command.func()

etc/refstack.conf.sample

@@ -1,5 +1,91 @@
 [DEFAULT]
 
+#
+# From oslo.log
+#
+
+# Print debugging output (set logging level to DEBUG instead of
+# default WARNING level). (boolean value)
+#debug = false
+
+# Print more verbose output (set logging level to INFO instead of
+# default WARNING level). (boolean value)
+#verbose = false
+
+# The name of a logging configuration file. This file is appended to
+# any existing logging configuration files. For details about logging
+# configuration files, see the Python logging module documentation.
+# (string value)
+# Deprecated group/name - [DEFAULT]/log_config
+#log_config_append = <None>
+
+# DEPRECATED. A logging.Formatter log message format string which may
+# use any of the available logging.LogRecord attributes. This option
+# is deprecated.  Please use logging_context_format_string and
+# logging_default_format_string instead. (string value)
+#log_format = <None>
+
+# Format string for %%(asctime)s in log records. Default: %(default)s
+# . (string value)
+#log_date_format = %Y-%m-%d %H:%M:%S
+
+# (Optional) Name of log file to output to. If no default is set,
+# logging will go to stdout. (string value)
+# Deprecated group/name - [DEFAULT]/logfile
+#log_file = <None>
+
+# (Optional) The base directory used for relative --log-file paths.
+# (string value)
+# Deprecated group/name - [DEFAULT]/logdir
+#log_dir = <None>
+
+# Use syslog for logging. Existing syslog format is DEPRECATED during
+# I, and will change in J to honor RFC5424. (boolean value)
+#use_syslog = false
+
+# (Optional) Enables or disables syslog rfc5424 format for logging. If
+# enabled, prefixes the MSG part of the syslog message with APP-NAME
+# (RFC5424). The format without the APP-NAME is deprecated in I, and
+# will be removed in J. (boolean value)
+#use_syslog_rfc_format = false
+
+# Syslog facility to receive log lines. (string value)
+#syslog_log_facility = LOG_USER
+
+# Log output to standard error. (boolean value)
+#use_stderr = true
+
+# Format string to use for log messages with context. (string value)
+#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
+
+# Format string to use for log messages without context. (string
+# value)
+#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
+
+# Data to append to log format when level is DEBUG. (string value)
+#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
+
+# Prefix each line of exception output with this format. (string
+# value)
+#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s
+
+# List of logger=LEVEL pairs. (list value)
+#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN
+
+# Enables or disables publication of error events. (boolean value)
+#publish_errors = false
+
+# Enables or disables fatal status of deprecations. (boolean value)
+#fatal_deprecations = false
+
+# The format for an instance that is passed with the log message.
+# (string value)
+#instance_format = "[instance: %(uuid)s] "
+
+# The format for an instance UUID that is passed with the log message.
+# (string value)
+#instance_uuid_format = "[instance: %(uuid)s] "
+
 #
 # From refstack
 #
@@ -8,6 +94,35 @@
 #db_backend = sqlalchemy
 
 
+[api]
+
+#
+# From refstack
+#
+
+# The directory where your static files can be found. Pecan comes with
+# middleware that can be used to serve static files (like CSS and
+# Javascript files) during development. %(project_root)s is special
+# variable that point to the root directory of Refstack project. Value
+# of this option must contain %(project_root)s variable. Directory
+# with static files specified relative the project root. (string
+# value)
+#static_root = %(project_root)s/static
+
+# Points to the directory where your template files live.
+# %(project_root)s is special variable that point to the root
+# directory of Refstack project. Value of this option must contain
+# %(project_root)s variable. Directory with template files specified
+# relative the project root. (string value)
+#template_path = %(project_root)s/templates
+
+# Switch Refstack app into debug mode. Helpful for development. In
+# debug mode static file will be served by pecan application. Also,
+# server responses will contain some details with debug information.
+# (boolean value)
+#app_dev_mode = false
+
+
 [database]
 
 #

refstack/api/app.py

@@ -20,24 +20,64 @@ import logging
 import os
 
 from oslo_config import cfg
+from oslo_log import log
+from oslo_log import loggers
 import pecan
 from pecan import hooks
 import webob
 
-from refstack import utils
+LOG = log.getLogger(__name__)
 
-logger = logging.getLogger(__name__)
+PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                            os.pardir)
+
+API_OPTS = [
+    cfg.StrOpt('static_root',
+               default='%(project_root)s/static',
+               help='The directory where your static files can '
+                    'be found. Pecan comes with middleware that can be used '
+                    'to serve static files (like CSS and Javascript files) '
+                    'during development. %(project_root)s is special variable '
+                    'that point to the root directory of Refstack project. '
+                    'Value of this option must contain %(project_root)s '
+                    'variable. Directory with static files specified relative '
+                    'the project root.'
+               ),
+    cfg.StrOpt('template_path',
+               default='%(project_root)s/templates',
+               help='Points to the directory where your template files live. '
+                    '%(project_root)s is special variable that point to the '
+                    'root directory of Refstack project. Value of this option '
+                    'must contain %(project_root)s variable. Directory with '
+                    'template files specified relative the project root.'
+               ),
+    cfg.BoolOpt('app_dev_mode',
+                default=False,
+                help='Switch Refstack app into debug mode. Helpful for '
+                     'development. In debug mode static file will be served '
+                     'by pecan application. Also, server responses will '
+                     'contain some details with debug information.'
+                ),
+]
 
 CONF = cfg.CONF
 
+opt_group = cfg.OptGroup(name='api',
+                         title='Options for the Refstack API')
+
+CONF.register_group(opt_group)
+CONF.register_opts(API_OPTS, opt_group)
+
+log.register_options(CONF)
+
 
 class JSONErrorHook(hooks.PecanHook):
     """
     A pecan hook that translates webob HTTP errors into a JSON format.
     """
-    def __init__(self, app_config):
+    def __init__(self):
         """Hook init."""
-        self.debug = app_config.get('debug', False)
+        self.debug = CONF.api.app_dev_mode
 
     def on_error(self, state, exc):
         """Request error handler."""
@@ -52,7 +92,7 @@ class JSONErrorHook(hooks.PecanHook):
                 content_type='application/json'
             )
         else:
-            logger.exception(exc)
+            LOG.exception(exc)
             body = {'code': 500,
                     'title': 'Internal Server Error'}
             if self.debug:
@@ -66,18 +106,6 @@ class JSONErrorHook(hooks.PecanHook):
 
 def setup_app(config):
     """App factory."""
-    app_conf = dict(config.app)
-
-    app = pecan.make_app(
-        app_conf.pop('root'),
-        logging=getattr(config, 'logging', {}),
-        hooks=[JSONErrorHook(app_conf), hooks.RequestViewerHook(
-            {'items': ['status', 'method', 'controller', 'path', 'body']},
-            headers=False, writer=utils.LogWriter(logger, logging.DEBUG)
-        )],
-        **app_conf
-    )
-
     # By default we expect path to oslo config file in environment variable
     # REFSTACK_OSLO_CONFIG (option for testing and development)
     # If it is empty we look up those config files
@@ -86,14 +114,30 @@ def setup_app(config):
     #   ~/
     #   /etc/${project}/
     #   /etc/
+
     default_config_files = ((os.getenv('REFSTACK_OSLO_CONFIG'), )
                             if os.getenv('REFSTACK_OSLO_CONFIG')
                             else cfg.find_config_files('refstack'))
-
     CONF('',
          project='refstack',
          default_config_files=default_config_files)
 
-    CONF.log_opt_values(logger, logging.DEBUG)
+    log.setup(CONF, 'refstack')
+    CONF.log_opt_values(LOG, logging.DEBUG)
+
+    template_path = CONF.api.template_path % {'project_root': PROJECT_ROOT}
+    static_root = CONF.api.static_root % {'project_root': PROJECT_ROOT}
+
+    app_conf = dict(config.app)
+    app = pecan.make_app(
+        app_conf.pop('root'),
+        debug=CONF.api.app_dev_mode,
+        static_root=static_root,
+        template_path=template_path,
+        hooks=[JSONErrorHook(), hooks.RequestViewerHook(
+            {'items': ['status', 'method', 'controller', 'path', 'body']},
+            headers=False, writer=loggers.WritableLogger(LOG, logging.DEBUG)
+        )]
+    )
 
     return app

refstack/api/config.py

@@ -34,33 +34,4 @@ server = {
 app = {
     'root': 'refstack.api.controllers.root.RootController',
     'modules': ['refstack.api'],
-    'static_root': '%(confdir)s/../static',
-    'template_path': '%(confdir)s/../templates',
-    # The 'debug' option should be false in production servers, but needs to be
-    # true in development in order to allow the static_root option to work.
-    'debug': False,
-    'errors': {
-        '404': '/error/404',
-        '__force_dict__': True
-    }
-}
-
-logging = {
-    'loggers': {
-        'root': {'level': 'INFO', 'handlers': ['console']},
-        'refstack': {'level': 'DEBUG', 'handlers': ['console']}
-    },
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'simple'
-        }
-    },
-    'formatters': {
-        'simple': {
-            'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]'
-                       '[%(threadName)s] %(message)s')
-        }
-    }
 }

refstack/api/controllers/v1.py

@@ -14,15 +14,14 @@
 #    under the License.
 
 """Version 1 of the API."""
-import logging
+from oslo_log import log
-
 import pecan
 from pecan import rest
 
 from refstack import db
 from refstack.common import validators
 
-logger = logging.getLogger(__name__)
+LOG = log.getLogger(__name__)
 
 
 class RestControllerWithValidation(rest.RestController):

refstack/db/utils.py

@@ -14,13 +14,11 @@
 #    under the License.
 
 """Utilities for database."""
-
-import logging
-
 from oslo_config import cfg
+from oslo_log import log
 
 CONF = cfg.CONF
-LOG = logging.getLogger(__name__)
+LOG = log.getLogger(__name__)
 
 
 class PluggableBackend(object):

refstack/opts.py

@@ -31,6 +31,7 @@
         ]
     ...
 """
+import refstack.api.app
 import refstack.db.api
 
 
@@ -38,4 +39,5 @@ def list_opts():
     return [
         # Keep a list in alphabetical order
         ('DEFAULT', refstack.db.api.db_opts),
+        ('api', refstack.api.app.API_OPTS),
     ]

refstack/utils.py

@@ -20,7 +20,6 @@
 from datetime import datetime
 import os
 import random
-import re
 import string
 
 
@@ -61,18 +60,6 @@ SEX_TYPE = {
 STRING_LEN = 64
 
 
-class LogWriter(object):
-    """Stream-like API to logger"""
-
-    def __init__(self, logger, level):
-        self.logger = logger
-        self.level = level
-
-    def write(self, s):
-        if re.sub('[\n ]', '', s):
-            self.logger.log(self.level, '\n' + s)
-
-
 def get_current_time():
     return datetime.utcnow()
 

requirements.txt

@@ -4,8 +4,9 @@ alembic==0.5.0
 gunicorn==18
 oslo.config>=1.6.0 # Apache-2.0
 oslo.db>=1.4.1 # Apache-2.0
+oslo.log
 pecan>=0.8.2
 pyOpenSSL==0.13
 pycrypto==2.6
 requests==1.2.3
-jsonschema>=2.0.0,<3.0.0
+jsonschema>=2.0.0,<3.0.0

tox.ini

@@ -40,7 +40,8 @@ distribute = false
 commands =
     oslo-config-generator --output-file etc/refstack.conf.sample \
     --namespace refstack \
-    --namespace oslo.db
+    --namespace oslo.db \
+    --namespace oslo.log
 
 [testenv:venv]
 commands = {posargs}