testing/mozbase/test.py

   1 #!/usr/bin/env python
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 """
   8 run mozbase tests from a manifest,
   9 by default https://github.com/mozilla/mozbase/blob/master/test-manifest.ini
  10 """
  11
  12 import imp
  13 import manifestparser
  14 import mozinfo
  15 import optparse
  16 import os
  17 import sys
  18 import unittest
  19
  20 from moztest.results import TestResultCollection
  21
  22
  23 here = os.path.dirname(os.path.abspath(__file__))
  24
  25
  26 class TBPLTextTestResult(unittest.TextTestResult):
  27     """
  28     Format the failure outputs according to TBPL. See:
  29     https://wiki.mozilla.org/Sheriffing/Job_Visibility_Policy#6.29_Outputs_failures_in_a_TBPL-starrable_format
  30     """
  31
  32     def addFailure(self, test, err):
  33         super(unittest.TextTestResult, self).addFailure(test, err)
  34         self.stream.writeln()
  35         self.stream.writeln('TEST-UNEXPECTED-FAIL | %s | %s' % (test.id(), err[1]))
  36
  37     def addUnexpectedSuccess(self, test):
  38         super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
  39         self.stream.writeln()
  40         self.stream.writeln('TEST-UNEXPECTED-PASS | %s | Unexpected pass' % test.id())
  41
  42 def unittests(path):
  43     """return the unittests in a .py file"""
  44
  45     path = os.path.abspath(path)
  46     unittests = []
  47     assert os.path.exists(path)
  48     directory = os.path.dirname(path)
  49     sys.path.insert(0, directory) # insert directory into path for top-level imports
  50     modname = os.path.splitext(os.path.basename(path))[0]
  51     module = imp.load_source(modname, path)
  52     sys.path.pop(0) # remove directory from global path
  53     loader = unittest.TestLoader()
  54     suite = loader.loadTestsFromModule(module)
  55     for test in suite:
  56         unittests.append(test)
  57     return unittests
  58
  59 def main(args=sys.argv[1:]):
  60
  61     # parse command line options
  62     usage = '%prog [options] manifest.ini <manifest.ini> <...>'
  63     parser = optparse.OptionParser(usage=usage, description=__doc__)
  64     parser.add_option('-b', "--binary",
  65                   dest="binary", help="Binary path",
  66                   metavar=None, default=None)
  67     parser.add_option('--list', dest='list_tests',
  68                       action='store_true', default=False,
  69                       help="list paths of tests to be run")
  70     options, args = parser.parse_args(args)
  71
  72     # read the manifest
  73     if args:
  74         manifests = args
  75     else:
  76         manifests = [os.path.join(here, 'test-manifest.ini')]
  77     missing = []
  78     for manifest in manifests:
  79         # ensure manifests exist
  80         if not os.path.exists(manifest):
  81             missing.append(manifest)
  82     assert not missing, 'manifest(s) not found: %s' % ', '.join(missing)
  83     manifest = manifestparser.TestManifest(manifests=manifests)
  84
  85     if options.binary:
  86         # A specified binary should override the environment variable
  87         os.environ['BROWSER_PATH'] = options.binary
  88
  89     # gather the tests
  90     tests = manifest.active_tests(disabled=False, **mozinfo.info)
  91     tests = [test['path'] for test in tests]
  92     if options.list_tests:
  93         # print test paths
  94         print '\n'.join(tests)
  95         sys.exit(0)
  96
  97     # create unittests
  98     unittestlist = []
  99     for test in tests:
 100         unittestlist.extend(unittests(test))
 101
 102     # run the tests
 103     suite = unittest.TestSuite(unittestlist)
 104     runner = unittest.TextTestRunner(verbosity=2, # default=1 does not show success of unittests
 105                                      resultclass=TBPLTextTestResult)
 106     unittest_results = runner.run(suite)
 107     results = TestResultCollection.from_unittest_results(None, unittest_results)
 108
 109     # exit according to results
 110     sys.exit(1 if results.num_failures else 0)
 111
 112 if __name__ == '__main__':
 113     main()