Merge "add utility to htmlify screen logs for devstack runs"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
76c9f3f693
125
modules/openstack_project/files/logs/htmlify-screen-log.py
Executable file
125
modules/openstack_project/files/logs/htmlify-screen-log.py
Executable file
@ -0,0 +1,125 @@
|
||||
#!/usr/bin/python
|
||||
#
|
||||
# Copyright (c) 2013 IBM Corp.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import cgi
|
||||
import fileinput
|
||||
import re
|
||||
import sys
|
||||
import wsgiref.util
|
||||
|
||||
|
||||
DATEFMT = '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(\.\d{3})?'
|
||||
STATUSFMT = '(DEBUG|INFO|WARN|ERROR|TRACE|AUDIT)'
|
||||
LOGMATCH = '(?P<date>%s)(?P<pid> \d+)? (?P<status>%s)' % (DATEFMT, STATUSFMT)
|
||||
|
||||
|
||||
def _html_close():
|
||||
return ("</pre></body></html>\n")
|
||||
|
||||
|
||||
def _css_preamble():
|
||||
"""Write a valid html start with css that we need."""
|
||||
return ("""<html>
|
||||
<head>
|
||||
<style>
|
||||
a {color: #000; text-decoration: none}
|
||||
a:hover {text-decoration: underline}
|
||||
.DEBUG, .DEBUG a {color: #888}
|
||||
.ERROR, .ERROR a {color: #c00; font-weight: bold}
|
||||
.TRACE, .TRACE a {color: #c60}
|
||||
.WARN, .WARN a {color: #D89100; font-weight: bold}
|
||||
.INFO, .INFO a {color: #006; font-weight: bold}
|
||||
</style>
|
||||
<body><pre>\n""")
|
||||
|
||||
|
||||
def color_by_sev(line):
|
||||
"""Wrap a line in a span whose class matches it's severity."""
|
||||
m = re.match(LOGMATCH, line)
|
||||
if m:
|
||||
return "<span class='%s'>%s</span>" % (m.group('status'), line)
|
||||
else:
|
||||
return line
|
||||
|
||||
|
||||
def escape_html(line):
|
||||
"""Escape the html in a line.
|
||||
|
||||
We need to do this because we dump xml into the logs, and if we don't
|
||||
escape the xml we end up with invisible parts of the logs in turning it
|
||||
into html.
|
||||
"""
|
||||
return cgi.escape(line)
|
||||
|
||||
|
||||
def link_timestamp(line):
|
||||
return re.sub('^(?P<span><span[^>]*>)?(?P<date>%s)' % DATEFMT,
|
||||
('\g<span><a name="\g<date>" class="date" '
|
||||
'href="#\g<date>">\g<date></a>'),
|
||||
line)
|
||||
|
||||
|
||||
def passthrough_filter(fname):
|
||||
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||
yield line
|
||||
|
||||
|
||||
def html_filter(fname):
|
||||
"""Generator to read logs and output html in a stream.
|
||||
|
||||
This produces a stream of the htmlified logs which lets us return
|
||||
data quickly to the user, and use minimal memory in the process.
|
||||
"""
|
||||
yield _css_preamble()
|
||||
for line in fileinput.input(fname, openhook=fileinput.hook_compressed):
|
||||
newline = escape_html(line)
|
||||
newline = color_by_sev(newline)
|
||||
newline = link_timestamp(newline)
|
||||
yield newline
|
||||
yield _html_close()
|
||||
|
||||
|
||||
def htmlify_stdin():
|
||||
out = sys.stdout
|
||||
out.write(_css_preamble())
|
||||
for line in fileinput.input():
|
||||
newline = escape_html(line)
|
||||
newline = color_by_sev(newline)
|
||||
newline = link_timestamp(newline)
|
||||
out.write(newline)
|
||||
out.write(_html_close())
|
||||
|
||||
|
||||
def application(environ, start_response):
|
||||
status = '200 OK'
|
||||
path = wsgiref.util.request_uri(environ)
|
||||
match = re.search('htmlify/(.*)', path)
|
||||
# TODO(sdague): scrub all .. chars out of the path, for security reasons
|
||||
fname = "/srv/static/logs/%s" % match.groups(1)[0]
|
||||
if 'HTTP_ACCEPT' in environ and 'text/html' in environ['HTTP_ACCEPT']:
|
||||
response_headers = [('Content-type', 'text/html')]
|
||||
start_response(status, response_headers)
|
||||
return html_filter(fname)
|
||||
else:
|
||||
response_headers = [('Content-type', 'text/plain')]
|
||||
start_response(status, response_headers)
|
||||
return passthrough_filter(fname)
|
||||
|
||||
|
||||
# for development purposes, makes it easy to test the filter output
|
||||
if __name__ == "__main__":
|
||||
htmlify_stdin()
|
||||
@ -21,6 +21,7 @@ class openstack_project::static (
|
||||
}
|
||||
|
||||
include apache
|
||||
include apache::mod::wsgi
|
||||
|
||||
a2mod { 'rewrite':
|
||||
ensure => present,
|
||||
@ -81,6 +82,14 @@ class openstack_project::static (
|
||||
template => 'openstack_project/logs.vhost.erb',
|
||||
}
|
||||
|
||||
apache::vhost { 'logs-dev.openstack.org':
|
||||
port => 80,
|
||||
priority => '51',
|
||||
docroot => '/srv/static/logs',
|
||||
require => File['/srv/static/logs'],
|
||||
template => 'openstack_project/logs-dev.vhost.erb',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs':
|
||||
ensure => directory,
|
||||
owner => 'jenkins',
|
||||
@ -97,6 +106,14 @@ class openstack_project::static (
|
||||
require => File['/srv/static/logs'],
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/htmlify-screen-log.py':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/logs/htmlify-screen-log.py',
|
||||
}
|
||||
|
||||
file { '/srv/static/logs/help':
|
||||
ensure => directory,
|
||||
recurse => true,
|
||||
|
||||
47
modules/openstack_project/templates/logs-dev.vhost.erb
Normal file
47
modules/openstack_project/templates/logs-dev.vhost.erb
Normal file
@ -0,0 +1,47 @@
|
||||
# -*- apache -*-
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= vhost_name %>:<%= port %>
|
||||
<VirtualHost <%= vhost_name %>:<%= port %>>
|
||||
ServerName <%= srvname %>
|
||||
<% if serveraliases.is_a? Array -%>
|
||||
<% serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||
<% elsif serveraliases != '' -%>
|
||||
<%= " ServerAlias #{serveraliases}" %>
|
||||
<% end -%>
|
||||
DocumentRoot <%= docroot %>
|
||||
|
||||
RewriteEngine On
|
||||
# rewrite all txt.gz files to map to our internal htmlify wsgi app
|
||||
RewriteRule ^/(.*\.txt\.gz)$ /htmlify/$1 [QSA,L,PT]
|
||||
WSGIScriptAlias /htmlify /usr/local/bin/htmlify-screen-log.py
|
||||
# use Apache to compress the results afterwards, to save on the wire
|
||||
# it's approx 18x savings of wire traffic to compress. We need to
|
||||
# compress by content types that htmlify can produce
|
||||
AddOutputFilterByType DEFLATE text/plain text/html
|
||||
|
||||
<FilesMatch \.html\.gz$>
|
||||
ForceType text/html
|
||||
AddDefaultCharset UTF-8
|
||||
AddEncoding x-gzip gz
|
||||
</FilesMatch>
|
||||
<Directory <%= docroot %>>
|
||||
Options <%= options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*>
|
||||
ReadmeName /help/tempest-overview.html
|
||||
</Directory>
|
||||
<Directory <%= docroot %>/*/*/*/gate-tempest-devstack*/*/logs/>
|
||||
ReadmeName /help/tempest-logs.html
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/apache2/<%= name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/<%= name %>_access.log combined
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
Loading…
Reference in New Issue
Block a user