Add a json reformatter command

It would save a lot of space to only write JSON logs, rather than also
write formatted logs. This utility turns the former into the latter
so we can do this without removing the ability of a sysadmin to read
logs in a sensible format.

You can either provide a filename or pipe via stdin. `tail -f` works
as stdin. Each record must occupy precisely one line.

Change-Id: I334e1e52d442f82bf68da9e581ce44bc3465208b
Co-Authored-By: Alexis Lee <lxsli@hpe.com>

oslo_log/cmds/convert_json.py

@@ -0,0 +1,138 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import print_function
+
+import argparse
+import collections
+import functools
+import sys
+
+from oslo_serialization import jsonutils
+from oslo_utils import importutils
+import six
+
+termcolor = importutils.try_import('termcolor')
+
+from oslo_log import log
+
+
+_USE_COLOR = False
+
+
+def main():
+    global _USE_COLOR
+    args = parse_args()
+    _USE_COLOR = args.color
+    formatter = functools.partial(console_format, args.prefix, args.locator)
+    for line in reformat_json(args.file, formatter):
+        print(line)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file",
+                        nargs='?', default=sys.stdin,
+                        type=argparse.FileType(),
+                        help="JSON log file to read from (if not provided"
+                             " standard input is used instead)")
+    parser.add_argument("--prefix",
+                        default='%(asctime)s.%(msecs)03d'
+                                ' %(process)s %(levelname)s %(name)s',
+                        help="Message prefixes")
+    parser.add_argument("--locator",
+                        default='[%(funcname)s %(pathname)s:%(lineno)s]',
+                        help="Locator to append to DEBUG records")
+    parser.add_argument("-c", "--color",
+                        action='store_true', default=False,
+                        help="Color log levels (requires `termcolor`)")
+    args = parser.parse_args()
+    if args.color and not termcolor:
+        raise ImportError("Coloring requested but `termcolor` is not"
+                          " importable")
+    return args
+
+
+def colorise(key, text=None):
+    if text is None:
+        text = key
+    if not _USE_COLOR:
+        return text
+    colors = {
+        'exc': ('red', ['reverse', 'bold']),
+        'FATAL': ('red', ['reverse', 'bold']),
+        'ERROR': ('red', ['bold']),
+        'WARNING': ('yellow', ['bold']),
+        'WARN': ('yellow', ['bold']),
+        'INFO': ('white', ['bold']),
+    }
+    color, attrs = colors.get(key, ('', []))
+    if color:
+        return termcolor.colored(text, color=color, attrs=attrs)
+    return text
+
+
+def warn(prefix, msg):
+    return "%s: %s" % (colorise('exc', prefix), msg)
+
+
+def reformat_json(fh, formatter):
+    # using readline allows interactive stdin to respond to every line
+    while True:
+        line = fh.readline()
+        if not line:
+            break
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            record = jsonutils.loads(line)
+        except ValueError:
+            yield warn("Not JSON", line)
+            continue
+        for out_line in formatter(record):
+            yield out_line
+
+
+def console_format(prefix, locator, record):
+    # Provide an empty string to format-specifiers the record is
+    # missing, instead of failing. Doesn't work for non-string
+    # specifiers.
+    record = collections.defaultdict(str, record)
+    levelname = record.get('levelname')
+    if levelname:
+        record['levelname'] = colorise(levelname)
+
+    try:
+        prefix = prefix % record
+    except TypeError:
+        # Thrown when a non-string format-specifier can't be filled in.
+        # Dict comprehension cleans up the output
+        yield warn('Missing non-string placeholder in record',
+                   {str(k): str(v) if isinstance(v, six.string_types) else v
+                    for k, v in six.iteritems(record)})
+        return
+
+    locator = ''
+    if (record.get('levelno', 100) <= log.DEBUG or levelname == 'DEBUG'):
+        locator = locator % record
+
+    yield ' '.join(x for x in [prefix, record['message'], locator] if x)
+
+    tb = record.get('traceback')
+    if tb:
+        for tb_line in tb.splitlines():
+            yield ' '.join([prefix, tb_line])
+
+
+if __name__ == '__main__':
+    main()

oslo_log/tests/unit/test_convert_json.py

@@ -0,0 +1,70 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+from oslo_log.cmds import convert_json
+from oslo_serialization import jsonutils
+from oslotest import base as test_base
+
+
+TRIVIAL_RECORD = {'message': 'msg'}
+DEBUG_LEVELNAME_RECORD = {
+    'message': 'msg',
+    'levelname': 'DEBUG',
+}
+DEBUG_LEVELNO_RECORD = {
+    'message': 'msg',
+    'levelno': 0,
+}
+TRACEBACK_RECORD = {
+    'message': 'msg',
+    'traceback': "abc\ndef",
+}
+
+
+class ConvertJsonTestCase(test_base.BaseTestCase):
+    def setUp(self):
+        super(ConvertJsonTestCase, self).setUp()
+
+    def _reformat(self, text):
+        fh = six.StringIO(text)
+        return list(convert_json.reformat_json(fh, lambda x: [x]))
+
+    def test_reformat_json_single(self):
+        text = jsonutils.dumps(TRIVIAL_RECORD)
+        self.assertEqual([TRIVIAL_RECORD], self._reformat(text))
+
+    def test_reformat_json_blanks(self):
+        text = jsonutils.dumps(TRIVIAL_RECORD)
+        self.assertEqual([TRIVIAL_RECORD], self._reformat(text + "\n\n"))
+
+    def test_reformat_json_double(self):
+        text = jsonutils.dumps(TRIVIAL_RECORD)
+        self.assertEqual(
+            [TRIVIAL_RECORD, TRIVIAL_RECORD],
+            self._reformat("\n".join([text, text])))
+
+    def _lines(self, record, pre='pre', loc='loc'):
+        return list(convert_json.console_format(pre, loc, record))
+
+    def test_console_format_trivial(self):
+        lines = self._lines(TRIVIAL_RECORD)
+        self.assertEqual(['pre msg'], lines)
+
+    def test_console_format_debug_levelname(self):
+        lines = self._lines(DEBUG_LEVELNAME_RECORD)
+        self.assertEqual(['pre msg'], lines)
+
+    def test_console_format_debug_levelno(self):
+        lines = self._lines(DEBUG_LEVELNO_RECORD)
+        self.assertEqual(['pre msg'], lines)

setup.cfg

@@ -33,6 +33,8 @@ warnerrors = true
 [entry_points]
 oslo.config.opts =
     oslo.log = oslo_log._options:list_opts
+console_scripts =
+    convert-json = oslo_log.cmds.convert_json:main
 
 [build_sphinx]
 source-dir = doc/source