diff --git a/modules/jenkins/files/slave_scripts/run-tox.sh b/modules/jenkins/files/slave_scripts/run-tox.sh index 14216027a1..28da76576f 100755 --- a/modules/jenkins/files/slave_scripts/run-tox.sh +++ b/modules/jenkins/files/slave_scripts/run-tox.sh @@ -38,6 +38,12 @@ echo "======================================================================" .tox/$venv/bin/pip freeze echo "======================================================================" +if [ -f ".testrepository/0" ] +then + cp .testrepository/0 ./subunit_log.txt + /usr/local/jenkins/slave_scripts/subunit2html.py ./subunit_log.txt testr_results.html +fi + sudo /usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh post sudoresult=$? diff --git a/modules/jenkins/files/slave_scripts/subunit2html.py b/modules/jenkins/files/slave_scripts/subunit2html.py new file mode 100644 index 0000000000..7e26ee24b0 --- /dev/null +++ b/modules/jenkins/files/slave_scripts/subunit2html.py @@ -0,0 +1,685 @@ +""" +Utility to convert a subunit stream to an html results file. +Code is adapted from the pyunit Html test runner at +http://tungwaiyip.info/software/HTMLTestRunner.html + +Takes two arguments. First argument is path to subunit log file, second +argument is path of desired output file. Second argument is optional, +defaults to 'results.html'. + +Original HTMLTestRunner License: +------------------------------------------------------------------------ +Copyright (c) 2004-2007, Wai Yip Tung +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name Wai Yip Tung nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +import datetime +import subunit +import sys +import traceback +import unittest +from xml.sax import saxutils + +__version__ = '0.1' + +class TemplateData(object): + """ + Define a HTML template for report customerization and generation. + + Overall structure of an HTML report + + HTML + +------------------------+ + | | + | | + | | + | STYLESHEET | + | +----------------+ | + | | | | + | +----------------+ | + | | + | | + | | + | | + | | + | HEADING | + | +----------------+ | + | | | | + | +----------------+ | + | | + | REPORT | + | +----------------+ | + | | | | + | +----------------+ | + | | + | ENDING | + | +----------------+ | + | | | | + | +----------------+ | + | | + | | + | | + +------------------------+ + """ + + STATUS = { + 0: 'pass', + 1: 'fail', + 2: 'error', + 3: 'skip', + } + + DEFAULT_TITLE = 'Unit Test Report' + DEFAULT_DESCRIPTION = '' + + # ------------------------------------------------------------------------ + # HTML Template + + HTML_TMPL = r""" + + + + %(title)s + + + %(stylesheet)s + + + + +%(heading)s +%(report)s +%(ending)s + + + +""" + # variables: (title, generator, stylesheet, heading, report, ending) + + + # ------------------------------------------------------------------------ + # Stylesheet + # + # alternatively use a for external style sheet, e.g. + # + + STYLESHEET_TMPL = """ + +""" + + + + # ------------------------------------------------------------------------ + # Heading + # + + HEADING_TMPL = """
+

%(title)s

+%(parameters)s +

%(description)s

+
+ +""" # variables: (title, parameters, description) + + HEADING_ATTRIBUTE_TMPL = """

%(name)s: %(value)s

+""" # variables: (name, value) + + + + # ------------------------------------------------------------------------ + # Report + # + + REPORT_TMPL = """ +

Show +Summary +Failed +All +

+ +++++++++ + + + + + + + + + +%(test_list)s + + + + + + + + + +
Test Group/Test caseCountPassFailErrorSkipView
Total%(count)s%(Pass)s%(fail)s%(error)s%(skip)s 
+""" # variables: (test_list, count, Pass, fail, error) + + REPORT_CLASS_TMPL = r""" + + %(desc)s + %(count)s + %(Pass)s + %(fail)s + %(error)s + %(skip)s + Detail + +""" # variables: (style, desc, count, Pass, fail, error, cid) + + + REPORT_TEST_WITH_OUTPUT_TMPL = r""" + +
%(desc)s
+ + + + + %(status)s + + + + + + +""" # variables: (tid, Class, style, desc, status) + + + REPORT_TEST_NO_OUTPUT_TMPL = r""" + +
%(desc)s
+ %(status)s + +""" # variables: (tid, Class, style, desc, status) + + + REPORT_TEST_OUTPUT_TMPL = r""" +%(id)s: %(output)s +""" # variables: (id, output) + + + + # ------------------------------------------------------------------------ + # ENDING + # + + ENDING_TMPL = """
 
""" + +# -------------------- The end of the Template class ------------------- + + +class ClassInfoWrapper(object): + def __init__(self, name, mod): + self.name = name + self.mod = mod + + def __repr__(self): + return "%s.%s" % (self.name, self.mod) + + +class HtmlOutput(unittest.TestResult): + """Output test results in html.""" + + def __init__(self, html_file='result.html'): + super(HtmlOutput, self).__init__() + self.success_count = 0 + self.failure_count = 0 + self.error_count = 0 + self.skip_count = 0 + self.result = [] + self.html_file = html_file + + def addSuccess(self, test): + self.success_count += 1 + output = test.shortDescription() + if output is None: + output = test.id() + self.result.append((0, test, output, '')) + + def addSkip(self, test, err): + output = test.shortDescription() + if output is None: + output = test.id() + self.skip_count += 1 + self.result.append((3, test, output, '')) + + def addError(self, test, err): + output = test.shortDescription() + if output is None: + output = test.id() + # Skipped tests are handled by SkipTest Exceptions. + #if err[0] == SkipTest: + # self.skip_count += 1 + # self.result.append((3, test, output, '')) + else: + self.error_count += 1 + _exc_str = self.formatErr(err) + self.result.append((2, test, output, _exc_str)) + + def addFailure(self, test, err): + print test + self.failure_count += 1 + _exc_str = self.formatErr(err) + output = test.shortDescription() + if output is None: + output = test.id() + self.result.append((1, test, output, _exc_str)) + + def formatErr(self, err): + exctype, value, tb = err + return ''.join(traceback.format_exception(exctype, value, tb)) + + def report(self): + self.stopTime = datetime.datetime.now() + report_attrs = self._getReportAttributes() + generator = 'subunit2html %s' % __version__ + heading = self._generate_heading(report_attrs) + report = self._generate_report() + ending = self._generate_ending() + output = TemplateData.HTML_TMPL % dict( + title = saxutils.escape(TemplateData.DEFAULT_TITLE), + generator = generator, + stylesheet = TemplateData.STYLESHEET_TMPL, + heading = heading, + report = report, + ending = ending, + ) + if self.html_file: + html_file = open(self.html_file, 'w') + html_file.write(output.encode('utf8')) + + def _getReportAttributes(self): + """Return report attributes as a list of (name, value).""" + status = [] + if self.success_count: + status.append('Pass %s' % self.success_count) + if self.failure_count: + status.append('Failure %s' % self.failure_count) + if self.error_count: + status.append('Error %s' % self.error_count) + if self.skip_count: + status.append('Skip %s' % self.skip_count) + if status: + status = ' '.join(status) + else: + status = 'none' + return [ + ('Status', status), + ] + + def _generate_heading(self, report_attrs): + a_lines = [] + for name, value in report_attrs: + line = TemplateData.HEADING_ATTRIBUTE_TMPL % dict( + name = saxutils.escape(name), + value = saxutils.escape(value), + ) + a_lines.append(line) + heading = TemplateData.HEADING_TMPL % dict( + title = saxutils.escape(TemplateData.DEFAULT_TITLE), + parameters = ''.join(a_lines), + description = saxutils.escape(TemplateData.DEFAULT_DESCRIPTION), + ) + return heading + + def _generate_report(self): + rows = [] + sortedResult = self._sortResult(self.result) + for cid, (cls, cls_results) in enumerate(sortedResult): + # subtotal for a class + np = nf = ne = ns = 0 + for n,t,o,e in cls_results: + if n == 0: np += 1 + elif n == 1: nf += 1 + elif n == 2: ne += 1 + else: ns += 1 + + # format class description + if cls.mod == "__main__": + name = cls.name + else: + name = "%s.%s" % (cls.mod, cls.name) + doc = cls.__doc__ and cls.__doc__.split("\n")[0] or "" + desc = doc and '%s: %s' % (name, doc) or name + + row = TemplateData.REPORT_CLASS_TMPL % dict( + style = ne > 0 and 'errorClass' or nf > 0 and 'failClass' or 'passClass', + desc = desc, + count = np + nf + ne + ns, + Pass = np, + fail = nf, + error = ne, + skip = ns, + cid = 'c%s' % (cid+1), + ) + rows.append(row) + + for tid, (n,t,o,e) in enumerate(cls_results): + self._generate_report_test(rows, cid, tid, n, t, o, e) + + report = TemplateData.REPORT_TMPL % dict( + test_list = ''.join(rows), + count = str(self.success_count + self.failure_count + + self.error_count + self.skip_count), + Pass = str(self.success_count), + fail = str(self.failure_count), + error = str(self.error_count), + skip = str(self.skip_count), + ) + return report + + def _sortResult(self, result_list): + # unittest does not seems to run in any particular order. + # Here at least we want to group them together by class. + rmap = {} + classes = [] + for n,t,o,e in result_list: + if hasattr(t, '_tests'): + for inner_test in t._tests: + self._add_cls(rmap, classes, inner_test, (n,inner_test,o,e)) + else: + self._add_cls(rmap, classes, t, (n,t,o,e)) + r = [(cls, rmap[str(cls)]) for cls in classes] + return r + + def _add_cls(self, rmap, classes, test, data_tuple): + if hasattr(test, 'test'): + test = test.test + if test.__class__ == subunit.RemotedTestCase: + #print test._RemotedTestCase__description.rsplit('.', 1)[0] + cl = test._RemotedTestCase__description.rsplit('.', 1)[0] + mod = cl.rsplit('.', 1)[0] + cls = ClassInfoWrapper(cl, mod) + else: + cls = ClassInfoWrapper(str(test.__class__), str(test.__module__)) + if not rmap.has_key(str(cls)): + rmap[str(cls)] = [] + classes.append(cls) + rmap[str(cls)].append(data_tuple) + + def _generate_report_test(self, rows, cid, tid, n, t, o, e): + # e.g. 'pt1.1', 'ft1.1', etc + # ptx.x for passed/skipped tests and ftx.x for failed/errored tests. + has_output = bool(o or e) + tid = ((n == 0 or n == 3) and 'p' or 'f') + 't%s.%s' % (cid+1,tid+1) + name = t.id().split('.')[-1] + doc = t.shortDescription() or "" + desc = doc and ('%s: %s' % (name, doc)) or name + tmpl = has_output and TemplateData.REPORT_TEST_WITH_OUTPUT_TMPL or TemplateData.REPORT_TEST_NO_OUTPUT_TMPL + + # Comments below from the original source project. + # TODO: clean this up within the context of a nose plugin. + # o and e should be byte string because they are collected from stdout and stderr? + if isinstance(o,str): + # TODO: some problem with 'string_escape': it escape \n and mess up formating + # uo = unicode(o.encode('string_escape')) + uo = o.decode('latin-1') + else: + uo = o + if isinstance(e,str): + # TODO: some problem with 'string_escape': it escape \n and mess up formating + # ue = unicode(e.encode('string_escape')) + ue = e.decode('latin-1') + else: + ue = e + + script = TemplateData.REPORT_TEST_OUTPUT_TMPL % dict( + id = tid, + output = saxutils.escape(uo+ue), + ) + + row = tmpl % dict( + tid = tid, + Class = ((n == 0 or n == 3) and 'hiddenRow' or 'none'), + style = n == 2 and 'errorCase' or (n == 1 and 'failCase' or 'none'), + desc = desc, + script = script, + status = TemplateData.STATUS[n], + ) + rows.append(row) + if not has_output: + return + + def _generate_ending(self): + return TemplateData.ENDING_TMPL + +def main(): + if len(sys.argv) < 2: + print "Need at least one argument: path to subunit log." + exit(1) + subunit_file = sys.argv[1] + if len(sys.argv) > 2: + html_file = sys.argv[2] + else: + html_file = 'results.html' + + stream = open(subunit_file, 'rb') + suite = subunit.ProtocolTestCase(stream) + result = HtmlOutput(html_file) + suite.run(result) + result.report() + + +if __name__ == '__main__': + main() diff --git a/modules/jenkins/manifests/slave.pp b/modules/jenkins/manifests/slave.pp index e19e629ac9..5239a306ec 100644 --- a/modules/jenkins/manifests/slave.pp +++ b/modules/jenkins/manifests/slave.pp @@ -45,6 +45,7 @@ class jenkins::slave( 'pandoc', #for docs, markdown->docbook, bug 924507 'pyflakes', 'python-libvirt', + 'python-subunit', # for subunit2html.py 'python-zmq', # zeromq unittests (not pip installable) 'python3-all-dev', 'rubygems', diff --git a/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml b/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml index d49c7b21c1..52aad9a0df 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/python-jobs.yaml @@ -73,6 +73,14 @@ source: '**/*nose_results.html' keep-hierarchy: false copy-after-failure: true + - target: 'logs/$ZUUL_CHANGE/$ZUUL_PATCHSET/$ZUUL_PIPELINE/$JOB_NAME/$BUILD_NUMBER' + source: '**/*testr_results.html' + keep-hierarchy: false + copy-after-failure: true + - target: 'logs/$ZUUL_CHANGE/$ZUUL_PATCHSET/$ZUUL_PIPELINE/$JOB_NAME/$BUILD_NUMBER' + source: '**/*subunit_log.txt' + keep-hierarchy: false + copy-after-failure: true - console-log # >= precise does not have python2.6 @@ -126,6 +134,14 @@ source: '**/*nose_results.html' keep-hierarchy: false copy-after-failure: true + - target: 'logs/$ZUUL_CHANGE/$ZUUL_PATCHSET/$ZUUL_PIPELINE/$JOB_NAME/$BUILD_NUMBER' + source: '**/*testr_results.html' + keep-hierarchy: false + copy-after-failure: true + - target: 'logs/$ZUUL_CHANGE/$ZUUL_PATCHSET/$ZUUL_PIPELINE/$JOB_NAME/$BUILD_NUMBER' + source: '**/*subunit_log.txt' + keep-hierarchy: false + copy-after-failure: true - console-log node: '{node}'