From 30f2074a372c345f493fd0188ce279ca9557dc17 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 May 2016 18:48:33 -0700 Subject: [PATCH] Add a json reformatter command It would save a lot of space to only write JSON logs, rather than also write formatted logs. This utility turns the former into the latter so we can do this without removing the ability of a sysadmin to read logs in a sensible format. You can either provide a filename or pipe via stdin. `tail -f` works as stdin. Each record must occupy precisely one line. Change-Id: I334e1e52d442f82bf68da9e581ce44bc3465208b Co-Authored-By: Alexis Lee --- oslo_log/cmds/__init__.py | 0 oslo_log/cmds/convert_json.py | 138 +++++++++++++++++++++++ oslo_log/tests/unit/test_convert_json.py | 70 ++++++++++++ setup.cfg | 2 + 4 files changed, 210 insertions(+) create mode 100644 oslo_log/cmds/__init__.py create mode 100644 oslo_log/cmds/convert_json.py create mode 100644 oslo_log/tests/unit/test_convert_json.py diff --git a/oslo_log/cmds/__init__.py b/oslo_log/cmds/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_log/cmds/convert_json.py b/oslo_log/cmds/convert_json.py new file mode 100644 index 00000000..5fcfe32d --- /dev/null +++ b/oslo_log/cmds/convert_json.py @@ -0,0 +1,138 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import argparse +import collections +import functools +import sys + +from oslo_serialization import jsonutils +from oslo_utils import importutils +import six + +termcolor = importutils.try_import('termcolor') + +from oslo_log import log + + +_USE_COLOR = False + + +def main(): + global _USE_COLOR + args = parse_args() + _USE_COLOR = args.color + formatter = functools.partial(console_format, args.prefix, args.locator) + for line in reformat_json(args.file, formatter): + print(line) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("file", + nargs='?', default=sys.stdin, + type=argparse.FileType(), + help="JSON log file to read from (if not provided" + " standard input is used instead)") + parser.add_argument("--prefix", + default='%(asctime)s.%(msecs)03d' + ' %(process)s %(levelname)s %(name)s', + help="Message prefixes") + parser.add_argument("--locator", + default='[%(funcname)s %(pathname)s:%(lineno)s]', + help="Locator to append to DEBUG records") + parser.add_argument("-c", "--color", + action='store_true', default=False, + help="Color log levels (requires `termcolor`)") + args = parser.parse_args() + if args.color and not termcolor: + raise ImportError("Coloring requested but `termcolor` is not" + " importable") + return args + + +def colorise(key, text=None): + if text is None: + text = key + if not _USE_COLOR: + return text + colors = { + 'exc': ('red', ['reverse', 'bold']), + 'FATAL': ('red', ['reverse', 'bold']), + 'ERROR': ('red', ['bold']), + 'WARNING': ('yellow', ['bold']), + 'WARN': ('yellow', ['bold']), + 'INFO': ('white', ['bold']), + } + color, attrs = colors.get(key, ('', [])) + if color: + return termcolor.colored(text, color=color, attrs=attrs) + return text + + +def warn(prefix, msg): + return "%s: %s" % (colorise('exc', prefix), msg) + + +def reformat_json(fh, formatter): + # using readline allows interactive stdin to respond to every line + while True: + line = fh.readline() + if not line: + break + line = line.strip() + if not line: + continue + try: + record = jsonutils.loads(line) + except ValueError: + yield warn("Not JSON", line) + continue + for out_line in formatter(record): + yield out_line + + +def console_format(prefix, locator, record): + # Provide an empty string to format-specifiers the record is + # missing, instead of failing. Doesn't work for non-string + # specifiers. + record = collections.defaultdict(str, record) + levelname = record.get('levelname') + if levelname: + record['levelname'] = colorise(levelname) + + try: + prefix = prefix % record + except TypeError: + # Thrown when a non-string format-specifier can't be filled in. + # Dict comprehension cleans up the output + yield warn('Missing non-string placeholder in record', + {str(k): str(v) if isinstance(v, six.string_types) else v + for k, v in six.iteritems(record)}) + return + + locator = '' + if (record.get('levelno', 100) <= log.DEBUG or levelname == 'DEBUG'): + locator = locator % record + + yield ' '.join(x for x in [prefix, record['message'], locator] if x) + + tb = record.get('traceback') + if tb: + for tb_line in tb.splitlines(): + yield ' '.join([prefix, tb_line]) + + +if __name__ == '__main__': + main() diff --git a/oslo_log/tests/unit/test_convert_json.py b/oslo_log/tests/unit/test_convert_json.py new file mode 100644 index 00000000..d349cd93 --- /dev/null +++ b/oslo_log/tests/unit/test_convert_json.py @@ -0,0 +1,70 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from oslo_log.cmds import convert_json +from oslo_serialization import jsonutils +from oslotest import base as test_base + + +TRIVIAL_RECORD = {'message': 'msg'} +DEBUG_LEVELNAME_RECORD = { + 'message': 'msg', + 'levelname': 'DEBUG', +} +DEBUG_LEVELNO_RECORD = { + 'message': 'msg', + 'levelno': 0, +} +TRACEBACK_RECORD = { + 'message': 'msg', + 'traceback': "abc\ndef", +} + + +class ConvertJsonTestCase(test_base.BaseTestCase): + def setUp(self): + super(ConvertJsonTestCase, self).setUp() + + def _reformat(self, text): + fh = six.StringIO(text) + return list(convert_json.reformat_json(fh, lambda x: [x])) + + def test_reformat_json_single(self): + text = jsonutils.dumps(TRIVIAL_RECORD) + self.assertEqual([TRIVIAL_RECORD], self._reformat(text)) + + def test_reformat_json_blanks(self): + text = jsonutils.dumps(TRIVIAL_RECORD) + self.assertEqual([TRIVIAL_RECORD], self._reformat(text + "\n\n")) + + def test_reformat_json_double(self): + text = jsonutils.dumps(TRIVIAL_RECORD) + self.assertEqual( + [TRIVIAL_RECORD, TRIVIAL_RECORD], + self._reformat("\n".join([text, text]))) + + def _lines(self, record, pre='pre', loc='loc'): + return list(convert_json.console_format(pre, loc, record)) + + def test_console_format_trivial(self): + lines = self._lines(TRIVIAL_RECORD) + self.assertEqual(['pre msg'], lines) + + def test_console_format_debug_levelname(self): + lines = self._lines(DEBUG_LEVELNAME_RECORD) + self.assertEqual(['pre msg'], lines) + + def test_console_format_debug_levelno(self): + lines = self._lines(DEBUG_LEVELNO_RECORD) + self.assertEqual(['pre msg'], lines) diff --git a/setup.cfg b/setup.cfg index a478375a..109b2a2a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,8 @@ warnerrors = true [entry_points] oslo.config.opts = oslo.log = oslo_log._options:list_opts +console_scripts = + convert-json = oslo_log.cmds.convert_json:main [build_sphinx] source-dir = doc/source