Merge "add utility to htmlify screen logs for devstack runs"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
76c9f3f693
125
modules/openstack_project/files/logs/htmlify-screen-log.py
Executable file
125
modules/openstack_project/files/logs/htmlify-screen-log.py
Executable file
@ -0,0 +1,125 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
#
|
||||||
|
# Copyright (c) 2013 IBM Corp.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
import cgi
|
||||||
|
import fileinput
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import wsgiref.util
|
||||||
|
|
||||||
|
|
||||||
|
DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3})?'
|
||||||
|
STATUSFMT = '(DEBUG|INFO|WARN|ERROR|TRACE|AUDIT)'
|
||||||
|
LOGMATCH = '(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % (DATEFMT, STATUSFMT)
|
||||||
|
|
||||||
|
|
||||||
|
def _html_close():
|
||||||
|
return ("</pre></body></html>\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _css_preamble():
|
||||||
|
"""Write a valid html start with css that we need."""
|
||||||
|
return ("""<html>
|
||||||
|
<head>
|
||||||
|
<style>
|
||||||
|
a {color: #000; text-decoration: none}
|
||||||
|
a:hover {text-decoration: underline}
|
||||||
|
.DEBUG, .DEBUG a {color: #888}
|
||||||
|
.ERROR, .ERROR a {color: #c00; font-weight: bold}
|
||||||
|
.TRACE, .TRACE a {color: #c60}
|
||||||
|
.WARN, .WARN a {color: #D89100; font-weight: bold}
|
||||||
|
.INFO, .INFO a {color: #006; font-weight: bold}
|
||||||
|
</style>
|
||||||
|
<body><pre>\n""")
|
||||||
|
|
||||||
|
|
||||||
|
def color_by_sev(line):
|
||||||
|
"""Wrap a line in a span whose class matches it's severity."""
|
||||||
|
m = re.match(LOGMATCH, line)
|
||||||
|
if m:
|
||||||
|
return "<span class='%s'>%s</span>" % (m.group('status'), line)
|
||||||
|
else:
|
||||||
|
return line
|
||||||
|
|
||||||
|
|
||||||
|
def escape_html(line):
|
||||||
|
"""Escape the html in a line.
|
||||||
|
|
||||||
|
We need to do this because we dump xml into the logs, and if we don't
|
||||||
|
escape the xml we end up with invisible parts of the logs in turning it
|
||||||
|
into html.
|
||||||
|
"""
|
||||||
|
return cgi.escape(line)
|
||||||
|
|
||||||
|
|
||||||
|
def link_timestamp(line):
|
||||||
|
return re.sub('^(?P<span><span[^>]*>)?(?P<date>%s)' % DATEFMT,
|
||||||
|
('\g<span><a name="\g<date>" class="date" '
|
||||||
|
'href="#\g<date>">\g<date></a>'),
|
||||||
|
line)
|
||||||
|
|
||||||
|
|
||||||
|
def passthrough_filter(fname):
|
||||||
|
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||||
|
yield line
|
||||||
|
|
||||||
|
|
||||||
|
def html_filter(fname):
|
||||||
|
"""Generator to read logs and output html in a stream.
|
||||||
|
|
||||||
|
This produces a stream of the htmlified logs which lets us return
|
||||||
|
data quickly to the user, and use minimal memory in the process.
|
||||||
|
"""
|
||||||
|
yield _css_preamble()
|
||||||
|
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||||
|
newline = escape_html(line)
|
||||||
|
newline = color_by_sev(newline)
|
||||||
|
newline = link_timestamp(newline)
|
||||||
|
yield newline
|
||||||
|
yield _html_close()
|
||||||
|
|
||||||
|
|
||||||
|
def htmlify_stdin():
|
||||||
|
out = sys.stdout
|
||||||
|
out.write(_css_preamble())
|
||||||
|
for line in fileinput.input():
|
||||||
|
newline = escape_html(line)
|
||||||
|
newline = color_by_sev(newline)
|
||||||
|
newline = link_timestamp(newline)
|
||||||
|
out.write(newline)
|
||||||
|
out.write(_html_close())
|
||||||
|
|
||||||
|
|
||||||
|
def application(environ, start_response):
|
||||||
|
status = '200 OK'
|
||||||
|
path = wsgiref.util.request_uri(environ)
|
||||||
|
match = re.search('htmlify/(.*)', path)
|
||||||
|
# TODO(sdague): scrub all .. chars out of the path, for security reasons
|
||||||
|
fname = "/srv/static/logs/%s" % match.groups(1)[0]
|
||||||
|
if 'HTTP_ACCEPT' in environ and 'text/html' in environ['HTTP_ACCEPT']:
|
||||||
|
response_headers = [('Content-type', 'text/html')]
|
||||||
|
start_response(status, response_headers)
|
||||||
|
return html_filter(fname)
|
||||||
|
else:
|
||||||
|
response_headers = [('Content-type', 'text/plain')]
|
||||||
|
start_response(status, response_headers)
|
||||||
|
return passthrough_filter(fname)
|
||||||
|
|
||||||
|
|
||||||
|
# for development purposes, makes it easy to test the filter output
|
||||||
|
if __name__ == "__main__":
|
||||||
|
htmlify_stdin()
|
||||||
@ -21,6 +21,7 @@ class openstack_project::static (
|
|||||||
}
|
}
|
||||||
|
|
||||||
include apache
|
include apache
|
||||||
|
include apache::mod::wsgi
|
||||||
|
|
||||||
a2mod { 'rewrite':
|
a2mod { 'rewrite':
|
||||||
ensure => present,
|
ensure => present,
|
||||||
@ -81,6 +82,14 @@ class openstack_project::static (
|
|||||||
template => 'openstack_project/logs.vhost.erb',
|
template => 'openstack_project/logs.vhost.erb',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
apache::vhost { 'logs-dev.openstack.org':
|
||||||
|
port => 80,
|
||||||
|
priority => '51',
|
||||||
|
docroot => '/srv/static/logs',
|
||||||
|
require => File['/srv/static/logs'],
|
||||||
|
template => 'openstack_project/logs-dev.vhost.erb',
|
||||||
|
}
|
||||||
|
|
||||||
file { '/srv/static/logs':
|
file { '/srv/static/logs':
|
||||||
ensure => directory,
|
ensure => directory,
|
||||||
owner => 'jenkins',
|
owner => 'jenkins',
|
||||||
@ -97,6 +106,14 @@ class openstack_project::static (
|
|||||||
require => File['/srv/static/logs'],
|
require => File['/srv/static/logs'],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
file { '/usr/local/bin/htmlify-screen-log.py':
|
||||||
|
ensure => present,
|
||||||
|
owner => 'root',
|
||||||
|
group => 'root',
|
||||||
|
mode => '0755',
|
||||||
|
source => 'puppet:///modules/openstack_project/logs/htmlify-screen-log.py',
|
||||||
|
}
|
||||||
|
|
||||||
file { '/srv/static/logs/help':
|
file { '/srv/static/logs/help':
|
||||||
ensure => directory,
|
ensure => directory,
|
||||||
recurse => true,
|
recurse => true,
|
||||||
|
|||||||
47
modules/openstack_project/templates/logs-dev.vhost.erb
Normal file
47
modules/openstack_project/templates/logs-dev.vhost.erb
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
# -*- apache -*-
|
||||||
|
# ************************************
|
||||||
|
# Managed by Puppet
|
||||||
|
# ************************************
|
||||||
|
|
||||||
|
NameVirtualHost <%= vhost_name %>:<%= port %>
|
||||||
|
<VirtualHost <%= vhost_name %>:<%= port %>>
|
||||||
|
ServerName <%= srvname %>
|
||||||
|
<% if serveraliases.is_a? Array -%>
|
||||||
|
<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||||
|
<% elsif serveraliases != '' -%>
|
||||||
|
<%= " ServerAlias #{serveraliases}" %>
|
||||||
|
<% end -%>
|
||||||
|
DocumentRoot <%= docroot %>
|
||||||
|
|
||||||
|
RewriteEngine On
|
||||||
|
# rewrite all txt.gz files to map to our internal htmlify wsgi app
|
||||||
|
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT]
|
||||||
|
WSGIScriptAlias /htmlify /usr/local/bin/htmlify-screen-log.py
|
||||||
|
# use Apache to compress the results afterwards, to save on the wire
|
||||||
|
# it's approx 18x savings of wire traffic to compress. We need to
|
||||||
|
# compress by content types that htmlify can produce
|
||||||
|
AddOutputFilterByType DEFLATE text/plain text/html
|
||||||
|
|
||||||
|
<FilesMatch \.html\.gz$>
|
||||||
|
ForceType text/html
|
||||||
|
AddDefaultCharset UTF-8
|
||||||
|
AddEncoding x-gzip gz
|
||||||
|
</FilesMatch>
|
||||||
|
<Directory <%= docroot %>>
|
||||||
|
Options <%= options %>
|
||||||
|
AllowOverride None
|
||||||
|
Order allow,deny
|
||||||
|
allow from all
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*>
|
||||||
|
ReadmeName /help/tempest-overview.html
|
||||||
|
</Directory>
|
||||||
|
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||||
|
ReadmeName /help/tempest-logs.html
|
||||||
|
</Directory>
|
||||||
|
|
||||||
|
ErrorLog /var/log/apache2/<%= name %>_error.log
|
||||||
|
LogLevel warn
|
||||||
|
CustomLog /var/log/apache2/<%= name %>_access.log combined
|
||||||
|
ServerSignature Off
|
||||||
|
</VirtualHost>
|
||||||
Loading…
Reference in New Issue
Block a user