opendev/system-config
Code Issues Proposed changes

add level parameter to htmlify-screen-log

Browse Source 
as an attempt to speed up both logstash, as well as user queries
of the logs, allow for a level parameter, which is the minimum
severity of a log line to display. This filter will work both with
html and text lines, and mean a lot less traffic over the wire.

Also provide links for making it easy to select log levels via
the HTML version of the logs.

Ensure we don't loose non tagged messages by having a "NONE" level
and starting with that when we process logs.

Change-Id: I14a2912d15d25a613c90ef350bcd3a2350456c61
This commit is contained in:
Sean Dague 2013-09-24 16:28:37 -04:00
parent 43bad50bea
commit 7951c30d9a
1 changed files with 64 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
modules/openstack_project/files/logs/htmlify-screen-log.py
View File

@ -27,6 +27,16 @@ DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3})?'
STATUSFMT = '(DEBUG|INFO|WARN|ERROR|TRACE|AUDIT)'
LOGMATCH = '(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % (DATEFMT, STATUSFMT)
SEVS = {
'NONE': 0,
'DEBUG': 1,
'INFO': 2,
'AUDIT': 3,
'TRACE': 4,
'WARN': 5,
'ERROR': 6
}
def _html_close():
return ("</span></pre></body></html>\n")
@ -44,17 +54,34 @@ a:hover {text-decoration: underline}
.TRACE, .TRACE a {color: #c60}
.WARN, .WARN a {color: #D89100; font-weight: bold}
.INFO, .INFO a {color: #006; font-weight: bold}
.selector, .selector a {color: #888}
.selector a:hover {color: #c00}
</style>
<body><pre><span>\n""")
<body>
<span class='selector'>
Display level: [
<a href='?'>ALL</a> |
<a href='?level=DEBUG'>DEBUG</a> |
<a href='?level=INFO'>INFO</a> |
<a href='?level=AUDIT'>AUDIT</a> |
<a href='?level=TRACE'>TRACE</a> |
<a href='?level=WARN'>WARN</a> |
<a href='?level=ERROR'>ERROR</a> ]
</span>
<pre><span>""")
def color_by_sev(line):
"""Wrap a line in a span whose class matches it's severity."""
def sev_of_line(line, oldsev="NONE"):
m = re.match(LOGMATCH, line)
if m:
return "<span class='%s'>%s</span>" % (m.group('status'), line)
return m.group('status')
else:
return line
return oldsev
def color_by_sev(line, sev):
"""Wrap a line in a span whose class matches it's severity."""
return "<span class='%s'>%s</span>" % (sev, line)
def escape_html(line):
@ -80,8 +107,22 @@ def link_timestamp(line):
return line
def passthrough_filter(fname):
def skip_line_by_sev(sev, minsev):
"""should we skip this line?
If the line severity is less than our minimum severity,
yes we should"""
return SEVS.get(sev, 0) < SEVS.get(minsev, 0)
def passthrough_filter(fname, minsev):
sev = "NONE"
for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
sev = sev_of_line(line, sev)
if skip_line_by_sev(sev, minsev):
continue
yield line
@ -103,7 +144,7 @@ def does_file_exist(fname):
f.close()
def html_filter(fname):
def html_filter(fname, minsev):
"""Generator to read logs and output html in a stream.
This produces a stream of the htmlified logs which lets us return
@ -111,9 +152,13 @@ def html_filter(fname):
"""
yield _css_preamble()
sev = "NONE"
for line in fileinput.FileInput(fname, openhook=fileinput.hook_compressed):
newline = escape_html(line)
newline = color_by_sev(newline)
sev = sev_of_line(newline, sev)
if skip_line_by_sev(sev, minsev):
continue
newline = color_by_sev(newline, sev)
newline = link_timestamp(newline)
yield newline
yield _html_close()
@ -170,6 +215,14 @@ def should_be_html(environ):
return accepts_html and not text_override
def get_min_sev(environ):
parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
if 'level' in parameters:
return cgi.escape(parameters['level'][0])
else:
return "NONE"
def application(environ, start_response):
status = '200 OK'
@ -181,16 +234,17 @@ def application(environ, start_response):
return ['Invalid file url']
try:
minsev = get_min_sev(environ)
if should_be_html(environ):
response_headers = [('Content-type', 'text/html')]
does_file_exist(logpath)
generator = html_filter(logpath)
generator = html_filter(logpath, minsev)
start_response(status, response_headers)
return generator
else:
response_headers = [('Content-type', 'text/plain')]
does_file_exist(logpath)
generator = passthrough_filter(logpath)
generator = passthrough_filter(logpath, minsev)
start_response(status, response_headers)
return generator
except IOError: