0test

   1 #!/usr/bin/env python
   2 # Copyright (C) 2008, Thomas Leonard
   3 # Visit http://0install.net for details.
   4
   5 import os, traceback, sys
   6 from optparse import OptionParser
   7
   8 sys.path.insert(0, os.environ["0TEST_ZEROINSTALL"])
   9
  10 from zeroinstall.injector import autopolicy, iface_cache, model, run, handler, arch
  11
  12 version = '0.2'
  13
  14 parser = OptionParser(usage="usage: %prog [INTERFACE VERSION ...] ...\n\n"
  15                 "For example, to test three versions of 0compile against two versions of Zero Install:\n\n"
  16                 "0test http://0install.net/2006/interfaces/0compile.xml 0.10 0.12 0.12-post \\\n"
  17                 "      http://0install.net/2007/interfaces/ZeroInstall.xml 0.31 0.36")
  18
  19 parser.add_option("", "--html", help="write results as HTML", action='store', metavar='OUTPUT')
  20 parser.add_option("-t", "--test-command", help="specify a custom test command", action='store', metavar='CMD')
  21 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  22 parser.add_option("-V", "--version", help="display version information", action='store_true')
  23
  24 (options, args) = parser.parse_args()
  25
  26 if options.version:
  27         print "0test (zero-install) " + version
  28         print "Copyright (C) 2008 Thomas Leonard"
  29         print "This program comes with ABSOLUTELY NO WARRANTY,"
  30         print "to the extent permitted by law."
  31         print "You may redistribute copies of this program"
  32         print "under the terms of the GNU General Public License."
  33         print "For more information about these matters, see the file named COPYING."
  34         sys.exit(0)
  35
  36 if not args:
  37         parser.print_help()
  38         sys.exit(1)
  39
  40 if options.verbose:
  41         import logging
  42         logger = logging.getLogger()
  43         if options.verbose == 1:
  44                 logger.setLevel(logging.INFO)
  45         else:
  46                 logger.setLevel(logging.DEBUG)
  47
  48 test_ifaces = []
  49 test_wrapper = options.test_command
  50 test_matrix = {}
  51
  52 if test_wrapper:
  53         test_wrapper += ' #'
  54
  55 iface = None
  56 for x in args:
  57         if x[0].isdigit() and iface:
  58                 test_matrix[iface].append(x)
  59         else:
  60                 assert x not in test_matrix, "Interface %s given twice!" % x
  61                 iface = model.canonical_iface_uri(x)
  62                 test_matrix[iface] = []
  63                 test_ifaces.append(iface)
  64
  65 # We don't need to specify the version of the interface under test.
  66 test_iface = test_ifaces[0]
  67 if not test_matrix[test_iface]:
  68         del test_matrix[test_iface]
  69         del test_ifaces[0]
  70
  71 if options.html and len(test_ifaces) < 2:
  72         print >>sys.stderr, "Need versions for at least two interfaces for --html mode."
  73         possible_deps = set()
  74         for impl in iface_cache.iface_cache.get_feed(test_iface).implementations.values():
  75                 for dep in impl.requires:
  76                         if dep.interface not in test_matrix:
  77                                 possible_deps.add(dep.interface)
  78         if possible_deps:
  79                 print "Suggestions:"
  80                 for url in possible_deps:
  81                         print "-", url
  82
  83         sys.exit(1)
  84
  85 # We do need a version for all the others (else what was the point of listing them?)
  86 for iface in test_ifaces[1:]:
  87         if not test_matrix[iface]:
  88                 raise Exception("No versions given for interface %s" % iface)
  89
  90 class VersionRestriction(model.Restriction):
  91         def __init__(self, version):
  92                 self.version = version
  93
  94         def meets_restriction(self, impl):
  95                 return impl.get_version() == self.version
  96
  97         def __repr__(self):
  98                 return "version = %s" % self.version
  99
 100 # Yield a sequence of set((uri, version)), one for each combination to test
 101 def get_combos(ifaces):
 102         if not ifaces:
 103                 yield {}
 104                 return
 105         for version in test_matrix[ifaces[0]]:
 106                 for combo in get_combos(ifaces[1:]):
 107                         combo[ifaces[0]] = version
 108                         yield combo.copy()
 109
 110 def all_combinations():
 111         return get_combos(test_ifaces)
 112
 113 class TestingArchitecture(arch.Architecture):
 114         use = frozenset([None, "testing"])
 115
 116         def __init__(self, child_arch):
 117                 arch.Architecture.__init__(self, child_arch.os_ranks, child_arch.machine_ranks)
 118                 self.child_arch = child_arch
 119
 120 ap = autopolicy.AutoPolicy(test_iface)
 121 ap.target_arch = TestingArchitecture(ap.target_arch)
 122
 123 if os.isatty(1):
 124         ap.handler = handler.ConsoleHandler()
 125
 126 # Explore all combinations...
 127
 128 tested_iface = iface_cache.iface_cache.get_interface(test_iface)
 129
 130 def _get_implementation_path(impl):
 131         return impl.local_path or iface_cache.iface_cache.stores.lookup_any(impl.digests)
 132
 133 def format_combo(selections):
 134         this_combo = []
 135         for x in selections:
 136                 impl = selections[x]
 137                 if impl:
 138                         version = impl.get_version()
 139                 else:
 140                         version = '(none)'
 141                 this_combo.append("%s v%s" % (x.get_name(), version))
 142         return ', '.join(this_combo)
 143
 144 def run_tests(ap):
 145         root_impl = ap.get_implementation(tested_iface)
 146
 147         print format_combo(ap.solver.selections)
 148
 149         if test_wrapper:
 150                 tests_dir = None
 151                 test_main = None
 152         else:
 153                 test_main = root_impl.metadata.get("self-test", None)
 154                 if not test_main:
 155                         print >>sys.stderr, "No self-test for version %s" % root_impl.get_version()
 156                         return "skipped"
 157                 main_abs = os.path.join(_get_implementation_path(root_impl), test_main)
 158                 if not os.path.exists(main_abs):
 159                         print >>sys.stderr, "Test executable does not exist:", main_abs
 160                         return "skipped"
 161
 162                 tests_dir = os.path.dirname(main_abs)
 163                 test_main = '/' + test_main
 164
 165         child = os.fork()
 166         if child:
 167                 # We are the parent
 168                 pid, status = os.waitpid(child, 0)
 169                 assert pid == child
 170                 print "Status:", hex(status)
 171                 if status == 0:
 172                         return "passed"
 173                 else:
 174                         return "failed"
 175         else:
 176                 # We are the child
 177                 try:
 178                         try:
 179                                 if test_wrapper is None:
 180                                         os.chdir(tests_dir)
 181                                 run.execute(ap, [], main = test_main, wrapper = test_wrapper)
 182                                 os._exit(0)
 183                         except model.SafeException, ex:
 184                                 print >>sys.stderr, str(ex)
 185                         except:
 186                                 traceback.print_exc()
 187                 finally:
 188                         sys.stdout.flush()
 189                         sys.stderr.flush()
 190                         os._exit(1)
 191
 192 if 'DISPLAY' in os.environ and not test_wrapper:
 193         del os.environ['DISPLAY']
 194
 195 results_by_combo = {}           # { set((uri, version)) : status }
 196 results_by_status = {           # status -> [ selections ]
 197         'passed': [],
 198         'skipped': [],
 199         'failed': [],
 200 }
 201
 202 for combo in all_combinations():
 203         key = set()
 204         restrictions = {}
 205         for (uri, version) in combo.iteritems():
 206                 restrictions[iface_cache.iface_cache.get_interface(uri)] = [VersionRestriction(version)]
 207                 key.add((uri, version))
 208
 209         ap.solver.extra_restrictions = restrictions
 210         solve = ap.solve_with_downloads()
 211         ap.handler.wait_for_blocker(solve)
 212         if not ap.ready:
 213                 result = 'skipped'
 214         else:
 215                 download = ap.download_uncached_implementations()
 216                 if download:
 217                         ap.handler.wait_for_blocker(download)
 218
 219                 tested_impl = ap.implementation[tested_iface]
 220
 221                 result = run_tests(ap)
 222
 223         selections = ap.solver.selections.copy()
 224         results_by_status[result].append(selections)
 225         results_by_combo[frozenset(key)] = (result, selections)
 226
 227 print "\nSUMMARY:\n"
 228
 229 for label in ["passed", "skipped", "failed"]:
 230         results = results_by_status[label]
 231         if not results:
 232                 print "None", label
 233         else:
 234                 print label.capitalize()
 235                 for combo in results:
 236                         print " - " + format_combo(combo)
 237
 238 html_file = options.html
 239
 240 if html_file:
 241         from xml.dom import minidom
 242         impl = minidom.getDOMImplementation()
 243         doctype = impl.createDocumentType("html",
 244                         "-//W3C//DTD XHTML 1.0 Strict//EN",
 245                         "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd")
 246         XMLNS_XHTML = "http://www.w3.org/1999/xhtml"
 247         doc = impl.createDocument(XMLNS_XHTML, "html", doctype)
 248         root = doc.documentElement
 249
 250         head = doc.createElement('head')
 251         root.appendChild(head)
 252         title = doc.createElement('title')
 253         title.appendChild(doc.createTextNode('0test results for ' + test_iface))
 254         head.appendChild(title)
 255         style = doc.createElement('style')
 256         head.appendChild(style)
 257         style.setAttribute('type', "text/css")
 258         style.appendChild(doc.createTextNode("""
 259                 table.testresults {
 260                         border: 1px solid black;
 261                         background: white;
 262                         color: white;
 263                 }
 264
 265                 table.testresults th {
 266                         text-align: left;
 267                         background: #888;
 268                 }
 269
 270                 table.testresults td.passed {
 271                         white-space: pre;
 272                 }
 273
 274                 table.testresults td.passed {
 275                         background: green;
 276                 }
 277
 278                 table.testresults td.skipped {
 279                         background: yellow;
 280                         color: #888;
 281                 }
 282
 283                 table.testresults td.failed {
 284                         background: red;
 285                 }
 286         """))
 287
 288         body = doc.createElement('body')
 289         root.appendChild(body)
 290
 291         for outer_combo in get_combos(test_ifaces[:-2]):
 292                 outer_key = frozenset(outer_combo.items())
 293                 outers = [(iface_cache.iface_cache.get_feed(uri).get_name() + " " + version) for (uri, version) in outer_combo.iteritems()]
 294
 295                 heading = doc.createElement('h1')
 296                 heading.appendChild(doc.createTextNode(', '.join(outers) or 'Results'))
 297                 body.appendChild(heading)
 298
 299                 table = doc.createElement('table')
 300                 table.setAttribute('class', 'testresults')
 301                 body.appendChild(table)
 302
 303                 col_iface_uri = test_ifaces[-1]
 304                 row_iface_uri = test_ifaces[-2]
 305
 306                 col_iface = iface_cache.iface_cache.get_interface(col_iface_uri)
 307                 row_iface = iface_cache.iface_cache.get_interface(row_iface_uri)
 308
 309                 test_columns = list(get_combos([test_ifaces[-1]]))
 310
 311                 row = doc.createElement('tr')
 312                 table.appendChild(row)
 313                 th = doc.createElement('th')
 314                 row.appendChild(th)
 315                 th = doc.createElement('th')
 316                 th.setAttribute("colspan", str(len(test_columns)))
 317                 row.appendChild(th)
 318                 th.appendChild(doc.createTextNode(col_iface.get_name()))
 319
 320                 row = doc.createElement('tr')
 321                 table.appendChild(row)
 322                 th = doc.createElement('th')
 323                 row.appendChild(th)
 324                 th.appendChild(doc.createTextNode(row_iface.get_name()))
 325                 for col_iface_version in test_matrix[col_iface_uri]:
 326                         th = doc.createElement('th')
 327                         row.appendChild(th)
 328                         th.appendChild(doc.createTextNode(col_iface_version))
 329
 330                 for row_iface_version in test_matrix[row_iface_uri]:
 331                         table.appendChild(doc.createTextNode('\n'))
 332                         row = doc.createElement('tr')
 333                         table.appendChild(row)
 334                         th = doc.createElement('th')
 335                         row.appendChild(th)
 336                         th.appendChild(doc.createTextNode(row_iface_version))
 337                         for col_iface in test_columns:
 338                                 td = doc.createElement('td')
 339                                 row.appendChild(td)
 340                                 key = frozenset(outer_key | set([(row_iface_uri, row_iface_version), (col_iface_uri, col_iface_version)]))
 341
 342                                 result, selections = results_by_combo[key]
 343                                 td.setAttribute('class', result)
 344
 345                                 other_ifaces = []
 346                                 combo_ifaces = set(uri for (uri, version) in key)
 347                                 for iface, impl in selections.iteritems():
 348                                         if iface.uri not in combo_ifaces:
 349                                                 other_ifaces.append((iface.get_name(), impl.get_version()))
 350                                 if other_ifaces:
 351                                         td.appendChild(doc.createTextNode('\n'.join('%s %s' % (uri, version) for uri, version in other_ifaces)))
 352                                 else:
 353                                         td.appendChild(doc.createTextNode(result))
 354                 table.appendChild(doc.createTextNode('\n'))
 355
 356         stream = open(html_file, 'w')
 357         doc.writexml(stream, encoding = 'utf8')
 358         stream.close()
 359
 360 if results_by_status['failed']:
 361         sys.exit(1)