glib/gtester-report

   1 #! /usr/bin/env python
   2 # GLib Testing Framework Utility                        -*- Mode: python; -*-
   3 # Copyright (C) 2007 Imendio AB
   4 # Authors: Tim Janik
   5 #
   6 # This library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # This library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18 import datetime
  19 import optparse
  20 import sys, re, xml.dom.minidom
  21
  22 try:
  23     import subunit
  24     from subunit import iso8601
  25     from testtools.content import Content, ContentType
  26     mime_utf8 = ContentType('text', 'plain', {'charset': 'utf8'})
  27 except ImportError:
  28     subunit = None
  29
  30
  31 pkginstall_configvars = {
  32   #@PKGINSTALL_CONFIGVARS_IN24LINES@ # configvars are substituted upon script installation
  33 }
  34
  35 # xml utilities
  36 def find_child (node, child_name):
  37   for child in node.childNodes:
  38     if child.nodeName == child_name:
  39       return child
  40   return None
  41 def list_children (node, child_name):
  42   rlist = []
  43   for child in node.childNodes:
  44     if child.nodeName == child_name:
  45       rlist += [ child ]
  46   return rlist
  47 def find_node (node, name = None):
  48   if not node or node.nodeName == name or not name:
  49     return node
  50   for child in node.childNodes:
  51     c = find_node (child, name)
  52     if c:
  53       return c
  54   return None
  55 def node_as_text (node, name = None):
  56   if name:
  57     node = find_node (node, name)
  58   txt = ''
  59   if node:
  60     if node.nodeValue:
  61       txt += node.nodeValue
  62     for child in node.childNodes:
  63       txt += node_as_text (child)
  64   return txt
  65 def attribute_as_text (node, aname, node_name = None):
  66   node = find_node (node, node_name)
  67   if not node:
  68     return ''
  69   attr = node.attributes.get (aname, '')
  70   if hasattr (attr, 'value'):
  71     return attr.value
  72   return ''
  73
  74 # HTML utilities
  75 def html_indent_string (n):
  76   uncollapsible_space = ' &nbsp;' # HTML won't compress alternating sequences of ' ' and '&nbsp;'
  77   string = ''
  78   for i in range (0, int((n + 1) / 2)):
  79     string += uncollapsible_space
  80   return string
  81
  82 # TestBinary object, instantiated per test binary in the log file
  83 class TestBinary:
  84   def __init__ (self, name):
  85     self.name = name
  86     self.testcases = []
  87     self.duration = 0
  88     self.success_cases = 0
  89     self.skipped_cases = 0
  90     self.file = '???'
  91     self.random_seed = ''
  92
  93 # base class to handle processing/traversion of XML nodes
  94 class TreeProcess:
  95   def __init__ (self):
  96     self.nest_level = 0
  97   def trampoline (self, node):
  98     name = node.nodeName
  99     if name == '#text':
 100       self.handle_text (node)
 101     else:
 102       try:      method = getattr (self, 'handle_' + re.sub ('[^a-zA-Z0-9]', '_', name))
 103       except:   method = None
 104       if method:
 105         return method (node)
 106       else:
 107         return self.process_recursive (name, node)
 108   def process_recursive (self, node_name, node):
 109     self.process_children (node)
 110   def process_children (self, node):
 111     self.nest_level += 1
 112     for child in node.childNodes:
 113       self.trampoline (child)
 114     self.nest_level += 1
 115
 116 # test report reader, this class collects some statistics and merges duplicate test binary runs
 117 class ReportReader (TreeProcess):
 118   def __init__ (self):
 119     TreeProcess.__init__ (self)
 120     self.binary_names = []
 121     self.binaries = {}
 122     self.last_binary = None
 123     self.info = {}
 124   def binary_list (self):
 125     lst = []
 126     for name in self.binary_names:
 127       lst += [ self.binaries[name] ]
 128     return lst
 129   def get_info (self):
 130     return self.info
 131   def handle_info (self, node):
 132     dn = find_child (node, 'package')
 133     self.info['package'] = node_as_text (dn)
 134     dn = find_child (node, 'version')
 135     self.info['version'] = node_as_text (dn)
 136     dn = find_child (node, 'revision')
 137     if dn is not None:
 138         self.info['revision'] = node_as_text (dn)
 139   def handle_testcase (self, node):
 140     self.last_binary.testcases += [ node ]
 141     result = attribute_as_text (node, 'result', 'status')
 142     if result == 'success':
 143       self.last_binary.success_cases += 1
 144     if bool (int (attribute_as_text (node, 'skipped') + '0')):
 145       self.last_binary.skipped_cases += 1
 146   def handle_text (self, node):
 147     pass
 148   def handle_testbinary (self, node):
 149     path = node.attributes.get ('path', None).value
 150     if self.binaries.get (path, -1) == -1:
 151       self.binaries[path] = TestBinary (path)
 152       self.binary_names += [ path ]
 153     self.last_binary = self.binaries[path]
 154     dn = find_child (node, 'duration')
 155     dur = node_as_text (dn)
 156     try:        dur = float (dur)
 157     except:     dur = 0
 158     if dur:
 159       self.last_binary.duration += dur
 160     bin = find_child (node, 'binary')
 161     if bin:
 162       self.last_binary.file = attribute_as_text (bin, 'file')
 163     rseed = find_child (node, 'random-seed')
 164     if rseed:
 165       self.last_binary.random_seed = node_as_text (rseed)
 166     self.process_children (node)
 167
 168
 169 class ReportWriter(object):
 170     """Base class for reporting."""
 171
 172     def __init__(self, binary_list):
 173         self.binaries = binary_list
 174
 175     def _error_text(self, node):
 176         """Get a string representing the error children of node."""
 177         rlist = list_children(node, 'error')
 178         txt = ''
 179         for enode in rlist:
 180             txt += node_as_text (enode)
 181             if txt and txt[-1] != '\n':
 182                 txt += '\n'
 183         return txt
 184
 185
 186 class HTMLReportWriter(ReportWriter):
 187   # Javascript/CSS snippet to toggle element visibility
 188   cssjs = r'''
 189   <style type="text/css" media="screen">
 190     .VisibleSection { }
 191     .HiddenSection  { display: none; }
 192   </style>
 193   <script language="javascript" type="text/javascript"><!--
 194   function toggle_display (parentid, tagtype, idmatch, keymatch) {
 195     ptag = document.getElementById (parentid);
 196     tags = ptag.getElementsByTagName (tagtype);
 197     for (var i = 0; i < tags.length; i++) {
 198       tag = tags[i];
 199       var key = tag.getAttribute ("keywords");
 200       if (tag.id.indexOf (idmatch) == 0 && key && key.match (keymatch)) {
 201         if (tag.className.indexOf ("HiddenSection") >= 0)
 202           tag.className = "VisibleSection";
 203         else
 204           tag.className = "HiddenSection";
 205       }
 206     }
 207   }
 208   message_array = Array();
 209   function view_testlog (wname, file, random_seed, tcase, msgtitle, msgid) {
 210       txt = message_array[msgid];
 211       var w = window.open ("", // URI
 212                            wname,
 213                            "resizable,scrollbars,status,width=790,height=400");
 214       var doc = w.document;
 215       doc.write ("<h2>File: " + file + "</h2>\n");
 216       doc.write ("<h3>Case: " + tcase + "</h3>\n");
 217       doc.write ("<strong>Random Seed:</strong> <code>" + random_seed + "</code> <br /><br />\n");
 218       doc.write ("<strong>" + msgtitle + "</strong><br />\n");
 219       doc.write ("<pre>");
 220       doc.write (txt);
 221       doc.write ("</pre>\n");
 222       doc.write ("<a href=\'javascript:window.close()\'>Close Window</a>\n");
 223       doc.close();
 224   }
 225   --></script>
 226   '''
 227   def __init__ (self, info, binary_list):
 228     ReportWriter.__init__(self, binary_list)
 229     self.info = info
 230     self.bcounter = 0
 231     self.tcounter = 0
 232     self.total_tcounter = 0
 233     self.total_fcounter = 0
 234     self.total_duration = 0
 235     self.indent_depth = 0
 236     self.lastchar = ''
 237   def oprint (self, message):
 238     sys.stdout.write (message)
 239     if message:
 240       self.lastchar = message[-1]
 241   def handle_info (self):
 242     if 'package' in self.info and 'version' in self.info:
 243       self.oprint ('<h3>Package: %(package)s, version: %(version)s</h3>\n' % self.info)
 244       if 'revision' in self.info:
 245           self.oprint ('<h5>Report generated from: %(revision)s</h5>\n' % self.info)
 246   def handle_text (self, node):
 247     self.oprint (node.nodeValue)
 248   def handle_testcase (self, node, binary):
 249     skipped = bool (int (attribute_as_text (node, 'skipped') + '0'))
 250     if skipped:
 251       return            # skipped tests are uninteresting for HTML reports
 252     path = attribute_as_text (node, 'path')
 253     duration = node_as_text (node, 'duration')
 254     result = attribute_as_text (node, 'result', 'status')
 255     rcolor = {
 256       'success': 'bgcolor="lightgreen"',
 257       'failed':  'bgcolor="red"',
 258     }.get (result, '')
 259     if result != 'success':
 260       duration = '-'    # ignore bogus durations
 261     self.oprint ('<tr id="b%u_t%u_" keywords="%s all" class="HiddenSection">\n' % (self.bcounter, self.tcounter, result))
 262     self.oprint ('<td>%s %s</td> <td align="right">%s</td> \n' % (html_indent_string (4), path, duration))
 263     perflist = list_children (node, 'performance')
 264     if result != 'success':
 265       txt = self._error_text(node)
 266       txt = re.sub (r'"', r'\\"', txt)
 267       txt = re.sub (r'\n', r'\\n', txt)
 268       txt = re.sub (r'&', r'&amp;', txt)
 269       txt = re.sub (r'<', r'&lt;', txt)
 270       self.oprint ('<script language="javascript" type="text/javascript">message_array["b%u_t%u_"] = "%s";</script>\n' % (self.bcounter, self.tcounter, txt))
 271       self.oprint ('<td align="center"><a href="javascript:view_testlog (\'%s\', \'%s\', \'%s\', \'%s\', \'Output:\', \'b%u_t%u_\')">Details</a></td>\n' %
 272                    ('TestResultWindow', binary.file, binary.random_seed, path, self.bcounter, self.tcounter))
 273     elif perflist:
 274       presults = []
 275       for perf in perflist:
 276         pmin = bool (int (attribute_as_text (perf, 'minimize')))
 277         pmax = bool (int (attribute_as_text (perf, 'maximize')))
 278         pval = float (attribute_as_text (perf, 'value'))
 279         txt = node_as_text (perf)
 280         txt = re.sub (r'&', r'&amp;', txt)
 281         txt = re.sub (r'<', r'&gt;', txt)
 282         txt = '<strong>Performance(' + (pmin and '<em>minimized</em>' or '<em>maximized</em>') + '):</strong> ' + txt.strip() + '<br />\n'
 283         txt = re.sub (r'"', r'\\"', txt)
 284         txt = re.sub (r'\n', r'\\n', txt)
 285         presults += [ (pval, txt) ]
 286       presults.sort()
 287       ptxt = ''.join ([e[1] for e in presults])
 288       self.oprint ('<script language="javascript" type="text/javascript">message_array["b%u_t%u_"] = "%s";</script>\n' % (self.bcounter, self.tcounter, ptxt))
 289       self.oprint ('<td align="center"><a href="javascript:view_testlog (\'%s\', \'%s\', \'%s\', \'%s\', \'Test Results:\', \'b%u_t%u_\')">Details</a></td>\n' %
 290                    ('TestResultWindow', binary.file, binary.random_seed, path, self.bcounter, self.tcounter))
 291     else:
 292       self.oprint ('<td align="center">-</td>\n')
 293     self.oprint ('<td align="right" %s>%s</td>\n' % (rcolor, result))
 294     self.oprint ('</tr>\n')
 295     self.tcounter += 1
 296     self.total_tcounter += 1
 297     self.total_fcounter += result != 'success'
 298   def handle_binary (self, binary):
 299     self.tcounter = 1
 300     self.bcounter += 1
 301     self.total_duration += binary.duration
 302     self.oprint ('<tr><td><strong>%s</strong></td><td align="right">%f</td> <td align="center">\n' % (binary.name, binary.duration))
 303     erlink, oklink = ('', '')
 304     real_cases = len (binary.testcases) - binary.skipped_cases
 305     if binary.success_cases < real_cases:
 306       erlink = 'href="javascript:toggle_display (\'ResultTable\', \'tr\', \'b%u_\', \'failed\')"' % self.bcounter
 307     if binary.success_cases:
 308       oklink = 'href="javascript:toggle_display (\'ResultTable\', \'tr\', \'b%u_\', \'success\')"' % self.bcounter
 309     if real_cases != 0:
 310         self.oprint ('<a %s>ER</a>\n' % erlink)
 311         self.oprint ('<a %s>OK</a>\n' % oklink)
 312         self.oprint ('</td>\n')
 313         perc = binary.success_cases * 100.0 / real_cases
 314         pcolor = {
 315           100 : 'bgcolor="lightgreen"',
 316           0   : 'bgcolor="red"',
 317         }.get (int (perc), 'bgcolor="yellow"')
 318         self.oprint ('<td align="right" %s>%.2f%%</td>\n' % (pcolor, perc))
 319         self.oprint ('</tr>\n')
 320     else:
 321         self.oprint ('Empty\n')
 322         self.oprint ('</td>\n')
 323         self.oprint ('</tr>\n')
 324     for tc in binary.testcases:
 325       self.handle_testcase (tc, binary)
 326   def handle_totals (self):
 327     self.oprint ('<tr>')
 328     self.oprint ('<td><strong>Totals:</strong> %u Binaries, %u Tests, %u Failed, %u Succeeded</td>' %
 329                  (self.bcounter, self.total_tcounter, self.total_fcounter, self.total_tcounter - self.total_fcounter))
 330     self.oprint ('<td align="right">%f</td>\n' % self.total_duration)
 331     self.oprint ('<td align="center">-</td>\n')
 332     if self.total_tcounter != 0:
 333         perc = (self.total_tcounter - self.total_fcounter) * 100.0 / self.total_tcounter
 334     else:
 335         perc = 0.0
 336     pcolor = {
 337       100 : 'bgcolor="lightgreen"',
 338       0   : 'bgcolor="red"',
 339     }.get (int (perc), 'bgcolor="yellow"')
 340     self.oprint ('<td align="right" %s>%.2f%%</td>\n' % (pcolor, perc))
 341     self.oprint ('</tr>\n')
 342   def printout (self):
 343     self.oprint ('<html><head>\n')
 344     self.oprint ('<title>GTester Unit Test Report</title>\n')
 345     self.oprint (self.cssjs)
 346     self.oprint ('</head>\n')
 347     self.oprint ('<body>\n')
 348     self.oprint ('<h2>GTester Unit Test Report</h2>\n')
 349     self.handle_info ()
 350     self.oprint ('<table id="ResultTable" width="100%" border="1">\n<tr>\n')
 351     self.oprint ('<th>Program / Testcase </th>\n')
 352     self.oprint ('<th style="width:8em">Duration (sec)</th>\n')
 353     self.oprint ('<th style="width:5em">View</th>\n')
 354     self.oprint ('<th style="width:5em">Result</th>\n')
 355     self.oprint ('</tr>\n')
 356     for tb in self.binaries:
 357       self.handle_binary (tb)
 358     self.handle_totals()
 359     self.oprint ('</table>\n')
 360     self.oprint ('</body>\n')
 361     self.oprint ('</html>\n')
 362
 363
 364 class SubunitWriter(ReportWriter):
 365     """Reporter to output a subunit stream."""
 366
 367     def printout(self):
 368         reporter = subunit.TestProtocolClient(sys.stdout)
 369         for binary in self.binaries:
 370             for tc in binary.testcases:
 371                 test = GTestCase(tc, binary)
 372                 test.run(reporter)
 373
 374
 375 class GTestCase(object):
 376     """A representation of a gtester test result as a pyunit TestCase."""
 377
 378     def __init__(self, case, binary):
 379         """Create a GTestCase for case 'case' from binary program 'binary'."""
 380         self._case = case
 381         self._binary = binary
 382         # the name of the case - e.g. /dbusmenu/glib/objects/menuitem/props_boolstr
 383         self._path = attribute_as_text(self._case, 'path')
 384
 385     def id(self):
 386         """What test is this? Returns the gtester path for the testcase."""
 387         return self._path
 388
 389     def _get_details(self):
 390         """Calculate a details dict for the test - attachments etc."""
 391         details = {}
 392         result = attribute_as_text(self._case, 'result', 'status')
 393         details['filename'] = Content(mime_utf8, lambda:[self._binary.file])
 394         details['random_seed'] = Content(mime_utf8,
 395             lambda:[self._binary.random_seed])
 396         if self._get_outcome() == 'addFailure':
 397             # Extract the error details. Skips have no details because its not
 398             # skip like unittest does, instead the runner just bypasses N test.
 399             txt = self._error_text(self._case)
 400             details['error'] = Content(mime_utf8, lambda:[txt])
 401         if self._get_outcome() == 'addSuccess':
 402             # Sucessful tests may have performance metrics.
 403             perflist = list_children(self._case, 'performance')
 404             if perflist:
 405                 presults = []
 406                 for perf in perflist:
 407                     pmin = bool (int (attribute_as_text (perf, 'minimize')))
 408                     pmax = bool (int (attribute_as_text (perf, 'maximize')))
 409                     pval = float (attribute_as_text (perf, 'value'))
 410                     txt = node_as_text (perf)
 411                     txt = 'Performance(' + (pmin and 'minimized' or 'maximized'
 412                         ) + '): ' + txt.strip() + '\n'
 413                     presults += [(pval, txt)]
 414                 presults.sort()
 415                 perf_details = [e[1] for e in presults]
 416                 details['performance'] = Content(mime_utf8, lambda:perf_details)
 417         return details
 418
 419     def _get_outcome(self):
 420         if int(attribute_as_text(self._case, 'skipped') + '0'):
 421             return 'addSkip'
 422         outcome = attribute_as_text(self._case, 'result', 'status')
 423         if outcome == 'success':
 424             return 'addSuccess'
 425         else:
 426             return 'addFailure'
 427
 428     def run(self, result):
 429         time = datetime.datetime.utcnow().replace(tzinfo=iso8601.Utc())
 430         result.time(time)
 431         result.startTest(self)
 432         try:
 433             outcome = self._get_outcome()
 434             details = self._get_details()
 435             # Only provide a duration IFF outcome == 'addSuccess' - the main
 436             # parser claims bogus results otherwise: in that case emit time as
 437             # zero perhaps.
 438             if outcome == 'addSuccess':
 439                 duration = float(node_as_text(self._case, 'duration'))
 440                 duration = duration * 1000000
 441                 timedelta = datetime.timedelta(0, 0, duration)
 442                 time = time + timedelta
 443                 result.time(time)
 444             getattr(result, outcome)(self, details=details)
 445         finally:
 446             result.stopTest(self)
 447
 448
 449
 450 # main program handling
 451 def parse_opts():
 452     """Parse program options.
 453
 454     :return: An options object and the program arguments.
 455     """
 456     parser = optparse.OptionParser()
 457     parser.version =  pkginstall_configvars.get ('glib-version', '0.0-uninstalled')
 458     parser.usage = "%prog [OPTIONS] <gtester-log.xml>"
 459     parser.description = "Generate HTML reports from the XML log files generated by gtester."
 460     parser.epilog = "gtester-report (GLib utils) version %s."% (parser.version,)
 461     parser.add_option("-v", "--version", action="store_true", dest="version", default=False,
 462         help="Show program version.")
 463     parser.add_option("-s", "--subunit", action="store_true", dest="subunit", default=False,
 464         help="Output subunit [See https://launchpad.net/subunit/"
 465             " Needs python-subunit]")
 466     options, files = parser.parse_args()
 467     if options.version:
 468         print(parser.epilog)
 469         return None, None
 470     if len(files) != 1:
 471         parser.error("Must supply a log file to parse.")
 472     if options.subunit and subunit is None:
 473         parser.error("python-subunit is not installed.")
 474     return options, files
 475
 476
 477 def main():
 478   options, files = parse_opts()
 479   if options is None:
 480     return 0
 481   xd = xml.dom.minidom.parse (files[0])
 482   rr = ReportReader()
 483   rr.trampoline (xd)
 484   if not options.subunit:
 485       HTMLReportWriter(rr.get_info(), rr.binary_list()).printout()
 486   else:
 487       SubunitWriter(rr.get_info(), rr.binary_list()).printout()
 488
 489
 490 if __name__ == '__main__':
 491   main()