indico/tests/runners.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 ##
   4 ## This file is part of Indico.
   5 ## Copyright (C) 2002 - 2012 European Organization for Nuclear Research (CERN).
   6 ##
   7 ## Indico is free software; you can redistribute it and/or
   8 ## modify it under the terms of the GNU General Public License as
   9 ## published by the Free Software Foundation; either version 3 of the
  10 ## License, or (at your option) any later version.
  11 ##
  12 ## Indico is distributed in the hope that it will be useful, but
  13 ## WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ## General Public License for more details.
  16 ##
  17 ## You should have received a copy of the GNU General Public License
  18 ## along with Indico;if not, see <http://www.gnu.org/licenses/>.
  19
  20 """
  21 This module defines the TestRunners that are included by default by indico.tests:
  22
  23  * UnitTestRunner
  24  * FunctionalTestRunner
  25  * GridTestRunner
  26  * PylintTestRunner
  27  * JSLintTestRunner
  28  * JSUnitTestRunner
  29
  30 """
  31
  32 # System modules
  33 import commands, os, socket, subprocess, tempfile, threading, multiprocessing
  34
  35 # Python stdlib
  36 import time, urllib2
  37
  38 # Test modules
  39 import figleaf
  40 import figleaf.annotate_html
  41 from selenium import webdriver
  42 import nose
  43
  44 from indico.tests.config import TestConfig
  45 from indico.tests.base import BaseTestRunner, Option
  46 from indico.tests.util import openBrowser, relpathto
  47 from indico.tests import default_actions
  48
  49 # legacy indico modules
  50 from MaKaC.common import Config, DBMgr
  51
  52
  53 __all__ = [
  54     'UnitTestRunner',
  55     'FunctionalTestRunner',
  56     'PylintTestRunner',
  57     'JSLintTestRunner',
  58     'JSUnitTestRunner'
  59     ]
  60
  61 JSTEST_CFG_FILE = "builtConf.conf"
  62
  63
  64 class CoverageBaseTestOption(Option):
  65     """
  66     This class can be used in order to add an
  67     optional code coverage analysis
  68     """
  69
  70     def __init__(self, value):
  71         Option.__init__(self, value)
  72         self.coverageDir = None
  73
  74     def final_message(self, __):
  75         """
  76         just a short message
  77         """
  78         self._info("Code coverage report generated at "
  79                    "%s/index.html\n" % self.coverageDir)
  80
  81     def shouldExecute(self):
  82         return self.value
  83
  84
  85 class CoveragePythonTestOption(CoverageBaseTestOption):
  86     """
  87     Python Coverage Tests
  88     """
  89
  90     def __init__(self, value):
  91         CoverageBaseTestOption.__init__(self, value)
  92
  93     def pre_run(self, __):
  94         """
  95         starts figleaf
  96         """
  97
  98         figleaf.start()
  99
 100     def post_run(self, runner):
 101         """
 102         stops figleaf and returns a report
 103         """
 104
 105         figleaf.stop()
 106         coverageOutput = figleaf.get_data().gather_files()
 107         self.coverageDir = os.path.join(runner.setupDir, 'report', 'pycoverage')
 108
 109         # check if there's a dir first
 110         if not os.path.exists(self.coverageDir):
 111             os.mkdir(self.coverageDir)
 112
 113         figleaf.annotate_html.report_as_html(coverageOutput,
 114                                              self.coverageDir, [], {})
 115
 116         # open a browser window with the report
 117         openBrowser(runner.config.getBrowserPath(), os.path.join(
 118             self.coverageDir, "index.html"))
 119
 120
 121
 122 class LogToConsoleTestOption(Option):
 123     """
 124     Python Coverage Tests
 125     """
 126
 127     def __init__(self, value):
 128         super(LogToConsoleTestOption, self).__init__(value)
 129
 130     def pre_run(self, __):
 131         """
 132         sets up logging
 133         """
 134
 135         import logging
 136         h = logging.StreamHandler()
 137         logger = logging.getLogger('')
 138
 139         logger.setLevel(getattr(logging, self.value))
 140
 141         # add multiprocessing info
 142         multiprocessing.get_logger().addHandler(h)
 143
 144         logger.addHandler(h)
 145         formatter = logging.Formatter("[%(process)d:%(threadName)s] %(asctime)s - %(name)s - %(levelname)s - %(message)s")
 146         h.setFormatter(formatter)
 147
 148     def shouldExecute(self):
 149         # execute it as long as it is specified
 150         return not not self.value
 151
 152
 153 class XMLOutputOption(Option):
 154
 155     def use_xml_output(self, obj, fname, args):
 156
 157         if self.value:
 158             if not os.path.exists('build'):
 159                 os.makedirs('build')
 160             args += ['--with-xunit', '--xunit-file=build/%s-results.xml' % fname]
 161         else:
 162             args += ['-v']
 163
 164
 165 class NoseTestRunner(BaseTestRunner):
 166
 167     def _buildArgs(self):
 168         args = ['nose', '--nologcapture', '--logging-clear-handlers', \
 169                 '--with-id', '-s']
 170
 171         # will set args
 172         self._callOptions('use_xml_output', 'unit', args)
 173
 174         specific = self.options.valueOf('specify')
 175
 176         if specific:
 177             args.append(specific)
 178         else:
 179             args.append(os.path.join(self.setupDir, self._defaultPath))
 180
 181         return args
 182
 183
 184 class UnitTestRunner(NoseTestRunner):
 185     """
 186     Python Unit Tests
 187
 188     Using nosetest
 189     """
 190
 191     _defaultPath = os.path.join('python', 'unit')
 192     _runnerOptions = {'silent': Option,
 193                       'coverage': CoveragePythonTestOption,
 194                       'specify': Option,
 195                       'log': LogToConsoleTestOption,
 196                       'xml': XMLOutputOption }
 197
 198     def _buildArgs(self):
 199         args = NoseTestRunner._buildArgs(self)
 200
 201         if not self.options.valueOf('specify'):
 202             #TODO: Make more general for functional test.
 203             # add plugins
 204             args += BaseTestRunner.findPlugins()
 205         return args
 206
 207     def _run(self):
 208         args = self._buildArgs()
 209         return nose.run(argv = args)
 210
 211
 212 class FunctionalTestRunner(NoseTestRunner):
 213     """
 214     Functional Tests
 215
 216     Using selenium
 217     """
 218
 219     _defaultPath = os.path.join('python', 'functional')
 220     _runnerOptions = {'silent': Option,
 221                       'record': Option,
 222                       'browser': Option,
 223                       'mode': Option,
 224                       'server_url': Option,
 225                       'specify': Option,
 226                       'xml': XMLOutputOption}
 227
 228
 229     def __init__(self, **kwargs):
 230         BaseTestRunner.__init__(self, **kwargs)
 231         self.child = None
 232
 233     def _runSeleniumCycle(self):
 234         """
 235         Run selenium over the existing test suite (or a specific test)
 236         """
 237         test_config = TestConfig.getInstance()
 238
 239         mode = self.options.valueOf('mode', test_config.getRunMode())
 240
 241         browser = self.options.valueOf('browser')
 242         if browser:
 243             browsers = [browser]
 244         elif mode == 'local':
 245             browsers = [test_config.getStandaloneBrowser()]
 246         else:
 247             browsers = test_config.getGridBrowsers()
 248
 249         args = self._buildArgs()
 250
 251         # Execute the tests
 252         result = True
 253
 254         os.environ['INDICO_TEST_MODE'] = mode or ''
 255         os.environ['INDICO_TEST_URL'] = self.options.valueOf('server_url') or ''
 256
 257         for browser in browsers:
 258             os.environ['INDICO_TEST_BROWSER'] = browser
 259             testResult = nose.run(argv=args)
 260             result = result and testResult
 261             self._info("%s: %s\n" % \
 262                        (browser, testResult and 'OK' or 'Error'))
 263
 264         return result
 265
 266     def _run(self):
 267         if self.options.valueOf('record'):
 268             dbi = DBMgr.getInstance()
 269
 270             dbi.startRequest()
 271             conn = dbi.getDBConnection()
 272
 273             default_actions.initialize_new_db(conn.root())
 274             default_actions.create_dummy_user()
 275             dbi.endRequest()
 276
 277             raw_input("Press [ENTER] to finish recording... ")
 278             result = False
 279
 280         else:
 281             result = self._runSeleniumCycle()
 282
 283         return result
 284
 285     def walkThroughPluginsFolders(self):
 286         """
 287         Goes throught the plugin directories, and adds
 288         existing functional test dirs
 289         """
 290         rootPluginsPath = os.path.join(self.setupDir, '..', 'MaKaC', 'plugins')
 291         foldersArray = []
 292
 293         for root, __, ___ in os.walk(rootPluginsPath):
 294             if root.endswith("/tests/python/functional") > 0:
 295                 foldersArray.append(root)
 296
 297         return foldersArray
 298
 299
 300 class HTMLOption(Option):
 301     """
 302     Represents the option that allows HTML reports to be generated
 303     instead of console output
 304     """
 305
 306     def __init__(self, value):
 307
 308         Option.__init__(self, value)
 309         self.tmpFile = None
 310
 311     def prepare_outstream(self, runner):
 312         """
 313         Forward the output to either a file or the process output
 314         """
 315
 316         if self.value:
 317             __, filePath = tempfile.mkstemp()
 318             self.tmpFile = open(filePath, 'w+b')
 319             runner.outStream = self.tmpFile
 320         else:
 321             # for regular text, just use a pipe
 322             runner.outStream = subprocess.PIPE
 323
 324     def write_report(self, runner, fileName, content):
 325         """
 326         Open the browser or write an actual report, depending on the state
 327         of the option
 328         """
 329         if self.value:
 330             self.tmpFile.close()
 331             # open a browser window with the report
 332             openBrowser(runner.config.getBrowserPath(), self.tmpFile.name)
 333         else:
 334             # for non-html output, use the normal mechanisms
 335             runner.writeNormalReport(fileName, content)
 336
 337
 338 class PylintTestRunner(BaseTestRunner):
 339     """
 340     Pylint
 341     """
 342
 343     _runnerOptions = {'silent': Option,
 344                       'html': HTMLOption}
 345
 346     def __init__(self, **kwargs):
 347
 348         BaseTestRunner.__init__(self, **kwargs)
 349         self.outStream = None
 350
 351     def _run(self):
 352
 353         fileList = self.config.getPylintFiles()
 354
 355         try:
 356             # while we have MaKaC, we have to keep this extra path
 357             extraPath = os.path.join(self.setupDir, '..')
 358             os.environ['PYTHONPATH'] = "%s:%s" % (extraPath,
 359                                                   os.environ.get('PYTHONPATH',''))
 360
 361             # Prepare the args for Pylint
 362             args = ["pylint", "--rcfile=%s" %
 363                     os.path.join(self.setupDir,
 364                                  'python',
 365                                  'pylint',
 366                                  'pylint.conf')
 367                     ] + fileList
 368
 369             if self.options.valueOf('html'):
 370                 args += ['-f', 'html']
 371
 372             # will set self.outStream
 373             self._callOptions('prepare_outstream')
 374
 375             pylintProcess = subprocess.Popen(
 376                 args,
 377                 stdout = self.outStream,
 378                 stderr = subprocess.PIPE)
 379
 380             # for regular aoutput, redirect the out pipe to stdout
 381             if not self.options.valueOf('html'):
 382                 self._redirectPipeToStdout(pylintProcess.stdout)
 383
 384             # stderr always goes to the same place
 385             self._redirectPipeToStdout(pylintProcess.stderr)
 386
 387             # wait pylint to finish
 388             pylintProcess.wait()
 389
 390         except OSError, e:
 391             self._error("[ERR] Could not start Source Analysis - "
 392                         "command \"pylint\" needs to be in your PATH. (%s)\n" % e)
 393             return False
 394
 395         return True
 396
 397     def _writeReport(self, fileName, content):
 398
 399         # overloaded just to handle the case of HTML reports
 400         self._callOptions('write_report', fileName, content)
 401
 402     def writeNormalReport(self, fileName, content):
 403         """
 404         Just call the parent report writing method
 405         (used from HTMLOption)
 406         """
 407         BaseTestRunner._writeReport(self, fileName, content)
 408
 409
 410 class CoverageJSTestOption(CoverageBaseTestOption):
 411     """
 412     Python Coverage Tests
 413     """
 414
 415     def __init__(self, value):
 416         CoverageBaseTestOption.__init__(self, value)
 417
 418     def post_run(self, runner):
 419         """
 420         Creates a coverage report in HTML
 421         """
 422
 423         #generate html for coverage
 424
 425         reportPath = os.path.join(runner.setupDir, 'report', 'jscoverage')
 426         genOutput = commands.getstatusoutput(
 427             "genhtml -o %s %s" % (reportPath,
 428                                   os.path.join(reportPath,
 429                                                '%s-coverage.dat' %
 430                                                JSTEST_CFG_FILE)))
 431
 432         self.coverageDir = reportPath
 433
 434         if genOutput[1].find("genhtml") > -1:
 435             BaseTestRunner._error("JS Unit Tests - html coverage "
 436                                   "generation failed, genhtml needs to be "
 437                                   "in your PATH. (%s)\n" % genOutput[1])
 438         else:
 439             BaseTestRunner._info("JS Coverage - report generated\n")
 440             # open a browser window with the report
 441             openBrowser(runner.config.getBrowserPath(), os.path.join(
 442                 reportPath, "index.html"))
 443
 444
 445 class JSUnitTestRunner(BaseTestRunner):
 446     """
 447     JS Unit Tests
 448
 449     Based on JSUnit
 450     """
 451
 452     _runnerOptions = {'silent': Option,
 453                       'coverage': CoverageJSTestOption,
 454                       'specify': Option}
 455
 456     def __init__(self, **kwargs):
 457         BaseTestRunner.__init__(self, **kwargs)
 458         self.coverage = self.options.valueOf('coverage')
 459         self.specify = self.options.valueOf('specify')
 460
 461     def _run(self):
 462
 463         #conf file used at run time
 464
 465         try:
 466             #Starting js-test-driver server
 467             server = subprocess.Popen(["java", "-jar",
 468                                        os.path.join(self.setupDir,
 469                                                     'javascript',
 470                                                     'unit',
 471                                                     TestConfig.getInstance().
 472                                                     getJSUnitFilename()),
 473                                         "--port",
 474                                         "9876",
 475                                         "--browser",
 476                                         "firefox"],
 477                                         stdout=subprocess.PIPE,
 478                                         stderr=subprocess.PIPE)
 479             time.sleep(2)
 480
 481             #constructing conf file depending on installed plugins and
 482             #coverage activation
 483
 484             success = self.buildConfFile(JSTEST_CFG_FILE, self.coverage)
 485
 486             if success != "":
 487                 self._error(success)
 488                 return False
 489
 490             #switching directory to run the tests
 491             os.chdir(os.path.join(self.setupDir, 'javascript', 'unit'))
 492
 493             #check if server is ready
 494             for i in range(5):
 495                 jsDryRun = commands.getstatusoutput(("java -jar "
 496                                              "%s"
 497                                              " --config "
 498                                              "%s"
 499                                              " --tests Fake.dryRun") %\
 500                                              (TestConfig.getInstance().
 501                                               getJSUnitFilename(),
 502                                               JSTEST_CFG_FILE))
 503
 504                 # Not very nice error checking, but how to do it nicely?
 505                 if "browsers" in jsDryRun[1] or \
 506                     "Connection refused" in jsDryRun[1]:
 507                     print "Js-test-driver server has not started yet. " \
 508                           "Attempt #%s\n" % (i+1)
 509                     time.sleep(5)
 510                 else:
 511                     #server is ready
 512                     break
 513             else:
 514                 raise Exception('Could not start js unit tests because '
 515                                 'js-test-driver server cannot be started.\n')
 516
 517             #setting tests to run
 518             toTest = ""
 519             if self.specify:
 520                 toTest = self.specify
 521             else:
 522                 toTest = "all"
 523
 524             command = ("java -jar %s "
 525                             "--config %s "
 526                             "--verbose "
 527                             "--tests %s ") % \
 528                             (TestConfig.getInstance().getJSUnitFilename(),
 529                              JSTEST_CFG_FILE,
 530                              toTest)
 531
 532             if self.coverage:
 533                 # path relative to the jar file
 534                 command += "--testOutput %s" % \
 535                            os.path.join('..', '..', 'report', 'jscoverage')
 536
 537             #running tests
 538             jsTest = commands.getoutput(command)
 539
 540
 541             #delete built conf file
 542             os.unlink(JSTEST_CFG_FILE)
 543
 544             #restoring directory
 545             os.chdir(self.setupDir)
 546
 547         except OSError, e:
 548             self._error("[ERR] Could not start js-test-driver server - command "
 549                         "\"java\" needs to be in your PATH. (%s)\n" % e)
 550         except KeyError:
 551             self._error("[ERR] Please specify a JSUnitFilename in tests.conf\n")
 552         except Exception, e:
 553             self._error(e)
 554         finally:
 555             # stopping the server
 556             server.kill()
 557
 558         return True
 559
 560     def buildConfFile(self, confFilePath, coverage):
 561         """
 562         Builds a driver config file
 563         """
 564         confTemplateDir = os.path.join(self.setupDir,
 565                                         'javascript',
 566                                         'unit')
 567         confTemplatePath = os.path.join(confTemplateDir, 'confTemplate.conf')
 568
 569         absoluteTestsDir = os.path.join(self.setupDir,
 570                                            "javascript",
 571                                            "unit",
 572                                            "tests")
 573
 574         absolutePluginDir = os.path.join(self.setupDir,
 575                                             "..",
 576                                             "..",
 577                                             "indico",
 578                                             "MaKaC",
 579                                             "plugins")
 580
 581         try:
 582             #lines needed to activate coverage plugin
 583             coverageConf = """\nplugin:
 584   - name: \"coverage\"
 585     jar: \"plugins/%s\"
 586     module: \"com.google.jstestdriver.coverage.CoverageModule\"""" % \
 587         TestConfig.getInstance().getJSCoverageFilename()
 588         except KeyError:
 589             return "Please, specify a JSCoverageFilename in tests.conf\n"
 590
 591
 592         try:
 593             #retrieve and store the template file
 594             f = open(confTemplatePath)
 595             confTemplate = f.read()
 596             f.close()
 597
 598             #adding tests files from tests folder
 599             for root, __, files in os.walk(absoluteTestsDir):
 600                 for name in files:
 601                     if name.endswith(".js"):
 602                         absoluteFilePath = os.path.join(root, name)
 603                         relativeFilePath = relpathto(confTemplateDir,
 604                                                      absoluteFilePath)
 605
 606                         confTemplate += "\n  - %s" % os.path.join(relativeFilePath)
 607
 608
 609             #adding plugins test files
 610             for root, __, files in os.walk(absolutePluginDir):
 611                 for name in files:
 612                     if name.endswith(".js") and \
 613                                           root.find("/tests/javascript/unit") > 0:
 614                         absoluteFilePath = os.path.join(root, name)
 615                         relativeFilePath = relpathto(confTemplateDir,
 616                                                      absoluteFilePath)
 617
 618                         confTemplate += "\n  - %s" % os.path.join('..',
 619                                                                   '..',
 620                                                                   relativeFilePath)
 621
 622             #addind coverage if necessary
 623             if coverage:
 624                 confTemplate += coverageConf
 625
 626             #writing the complete configuration in a file
 627             confFile = open(os.path.join(self.setupDir, 'javascript', 'unit',
 628                                          confFilePath), 'w')
 629             confFile.write(confTemplate)
 630             confFile.close()
 631
 632             return ""
 633         except IOError, e:
 634             return "JS Unit Tests - Could not open a file. (%s)" % e
 635
 636 class JSLintTestRunner(BaseTestRunner):
 637     """
 638     JSLint
 639     """
 640
 641     def _run(self):
 642
 643         # Folders which are not going to be scanned.
 644         # Files are going to be find recursively in the other folders
 645         blackList = set(['pack', 'Loader.js', 'Common', 'i18n'])
 646
 647         #checking if rhino is accessible
 648         statusOutput = commands.getstatusoutput("rhino -?")
 649         if statusOutput[1].find("rhino: not found") > -1:
 650             return ("[ERR] Could not start JS Source Analysis - command "
 651                     "\"rhino\" needs to be in your PATH. (%s)\n" % statusOutput[1])
 652
 653         #constructing a list of folders to scan
 654         folderNames = []
 655
 656         indicoDir = os.path.join(self.setupDir, '..', '..', 'indico')
 657
 658         fileList  = os.listdir(os.path.join(indicoDir,
 659                                             'htdocs',
 660                                             'js',
 661                                             'indico'))
 662         for name in fileList:
 663             if not (name in blackList):
 664                 folderNames.append(name)
 665
 666         #Scanning Indico core
 667         for folderName in folderNames:
 668             self.runJSLint(
 669                 os.path.join(indicoDir, 'htdocs', 'js', 'indico'),
 670                 folderName)
 671
 672         #Scanning plugins js files
 673         return self.runJSLint(
 674             os.path.join(indicoDir, 'MaKaC', 'plugins'))
 675
 676
 677     def runJSLint(self, path, folderRestriction=''):
 678         """
 679         runs the actual JSLint command
 680         """
 681
 682         for root, __, files in os.walk(os.path.join(self.setupDir,
 683                                                     path,
 684                                                     folderRestriction)):
 685             for name in files:
 686                 if name.endswith(".js"):
 687                     filename = os.path.join(root, name)
 688                     self._info("Scanning %s" % filename)
 689                     output = commands.getstatusoutput("rhino %s %s" %
 690                                                       (os.path.join(
 691                                                                 self.setupDir,
 692                                                                 'javascript',
 693                                                                 'jslint',
 694                                                                 'jslint.js'),
 695                                                        filename))
 696                     print output[1]
 697
 698                     if output[0] != 0:
 699                         return False
 700         return True
 701