From 7951c30d9af94492e98538519ee67610ba60c24f Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 24 Sep 2013 16:28:37 -0400 Subject: [PATCH] add level parameter to htmlify-screen-log as an attempt to speed up both logstash, as well as user queries of the logs, allow for a level parameter, which is the minimum severity of a log line to display. This filter will work both with html and text lines, and mean a lot less traffic over the wire. Also provide links for making it easy to select log levels via the HTML version of the logs. Ensure we don't loose non tagged messages by having a "NONE" level and starting with that when we process logs. Change-Id: I14a2912d15d25a613c90ef350bcd3a2350456c61 --- .../files/logs/htmlify-screen-log.py | 74 ++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/modules/openstack_project/files/logs/htmlify-screen-log.py b/modules/openstack_project/files/logs/htmlify-screen-log.py index 8625f859f4..c59fcc946d 100755 --- a/modules/openstack_project/files/logs/htmlify-screen-log.py +++ b/modules/openstack_project/files/logs/htmlify-screen-log.py @@ -27,6 +27,16 @@ DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3})?' STATUSFMT = '(DEBUG|INFO|WARN|ERROR|TRACE|AUDIT)' LOGMATCH = '(?P%s)(?P \d+)? (?P%s)' % (DATEFMT, STATUSFMT) +SEVS = { + 'NONE': 0, + 'DEBUG': 1, + 'INFO': 2, + 'AUDIT': 3, + 'TRACE': 4, + 'WARN': 5, + 'ERROR': 6 + } + def _html_close(): return ("\n") @@ -44,17 +54,34 @@ a:hover {text-decoration: underline} .TRACE, .TRACE a {color: #c60} .WARN, .WARN a {color: #D89100; font-weight: bold} .INFO, .INFO a {color: #006; font-weight: bold} +.selector, .selector a {color: #888} +.selector a:hover {color: #c00} -
\n""")
+
+
+Display level: [
+ALL |
+DEBUG |
+INFO |
+AUDIT |
+TRACE |
+WARN |
+ERROR ]
+
+
""")
 
 
-def color_by_sev(line):
-    """Wrap a line in a span whose class matches it's severity."""
+def sev_of_line(line, oldsev="NONE"):
     m = re.match(LOGMATCH, line)
     if m:
-        return "%s" % (m.group('status'), line)
+        return m.group('status')
     else:
-        return line
+        return oldsev
+
+
+def color_by_sev(line, sev):
+    """Wrap a line in a span whose class matches it's severity."""
+    return "%s" % (sev, line)
 
 
 def escape_html(line):
@@ -80,8 +107,22 @@ def link_timestamp(line):
         return line
 
 
-def passthrough_filter(fname):
+def skip_line_by_sev(sev, minsev):
+    """should we skip this line?
+
+    If the line severity is less than our minimum severity,
+    yes we should"""
+    return SEVS.get(sev, 0) < SEVS.get(minsev, 0)
+
+
+def passthrough_filter(fname, minsev):
+    sev = "NONE"
     for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
+        sev = sev_of_line(line, sev)
+
+        if skip_line_by_sev(sev, minsev):
+            continue
+
         yield line
 
 
@@ -103,7 +144,7 @@ def does_file_exist(fname):
     f.close()
 
 
-def html_filter(fname):
+def html_filter(fname, minsev):
     """Generator to read logs and output html in a stream.
 
     This produces a stream of the htmlified logs which lets us return
@@ -111,9 +152,13 @@ def html_filter(fname):
     """
 
     yield _css_preamble()
+    sev = "NONE"
     for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
         newline = escape_html(line)
-        newline = color_by_sev(newline)
+        sev = sev_of_line(newline, sev)
+        if skip_line_by_sev(sev, minsev):
+            continue
+        newline = color_by_sev(newline, sev)
         newline = link_timestamp(newline)
         yield newline
     yield _html_close()
@@ -170,6 +215,14 @@ def should_be_html(environ):
     return accepts_html and not text_override
 
 
+def get_min_sev(environ):
+    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
+    if 'level' in parameters:
+        return cgi.escape(parameters['level'][0])
+    else:
+        return "NONE"
+
+
 def application(environ, start_response):
     status = '200 OK'
 
@@ -181,16 +234,17 @@ def application(environ, start_response):
         return ['Invalid file url']
 
     try:
+        minsev = get_min_sev(environ)
         if should_be_html(environ):
             response_headers = [('Content-type', 'text/html')]
             does_file_exist(logpath)
-            generator = html_filter(logpath)
+            generator = html_filter(logpath, minsev)
             start_response(status, response_headers)
             return generator
         else:
             response_headers = [('Content-type', 'text/plain')]
             does_file_exist(logpath)
-            generator = passthrough_filter(logpath)
+            generator = passthrough_filter(logpath, minsev)
             start_response(status, response_headers)
             return generator
     except IOError: