test/acceptance/runtests.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python; indent-tabs-mode: nil; tab-width: 4 -*-
   3 # vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5)
   4 #
   5 # ***** BEGIN LICENSE BLOCK *****
   6 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   7 #
   8 # The contents of this file are subject to the Mozilla Public License Version
   9 # 1.1 (the "License"); you may not use this file except in compliance with
  10 # the License. You may obtain a copy of the License at
  11 # http://www.mozilla.org/MPL/
  12 #
  13 # Software distributed under the License is distributed on an "AS IS" basis,
  14 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  15 # for the specific language governing rights and limitations under the
  16 # License.
  17 #
  18 # The Original Code is [Open Source Virtual Machine.].
  19 #
  20 # The Initial Developer of the Original Code is
  21 # Adobe System Incorporated.
  22 # Portions created by the Initial Developer are Copyright (C) 2005-2009
  23 # the Initial Developer. All Rights Reserved.
  24 #
  25 # Contributor(s):
  26 #   christine@netscape.com
  27 #   Adobe AS3 Team
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK ***** */
  42 #
  43
  44 import os, sys, getopt, datetime, pipes, glob, itertools, tempfile, string
  45 import re, platform, threading, time
  46 from os.path import *
  47 from os import getcwd,environ,walk
  48 from datetime import datetime
  49 from glob import glob
  50 from sys import argv, exit
  51 from getopt import getopt
  52 from itertools import count
  53
  54
  55 # add parent dir to python module search path
  56 sys.path.append('..')
  57 from util.runtestBase import RuntestBase
  58 # runtestUtils must be imported after "from os.path import *" as walk is overridden
  59 from util.runtestUtils import *
  60
  61 class AcceptanceRuntest(RuntestBase):
  62     runESC = False
  63     escbin = '../../esc/bin/'
  64     androidthreads = False
  65     androiddevices = []
  66     verifyonly = False
  67     swfversions = [9,10,11,12]
  68
  69     def __init__(self):
  70         # Set threads to # of available cpus/cores
  71         self.threads = detectCPUs()
  72         RuntestBase.__init__(self)
  73
  74     def setEnvironVars(self):
  75         RuntestBase.setEnvironVars(self)
  76         if 'ESCBIN' in environ:
  77             self.escbin = environ['ESCBIN'].strip()
  78
  79     def usage(self, c):
  80         RuntestBase.usage(self, c)
  81         print('    --esc           run esc instead of avm')
  82         print('    --escbin        location of esc/bin directory - defaults to ../../esc/bin')
  83         print('    --eval          use run-time compiler')
  84         print('    --ext           set the testfile extension (defaults to .as)')
  85         print('    --ats           generate ats swfs instead of running tests')
  86         print('    --atsdir        base output directory for ats swfs - defaults to ATS_SWFS')
  87         print('    --threads       number of threads to run (default=# of cpu/cores), set to 1 to have tests finish sequentially')
  88         print('    --androidthreads    assign a thread for each android device connected.')
  89         print('    --verify        run a verify pass instead of running abcs')
  90         print('    --verifyonly    run a -Dverifyonly pass: only checks test exitcode')
  91         print('    --remoteip      IP/DNS address of the machine to run the tests on.')
  92         print('    --remoteuser    user name to use to connect to the remote machine.')
  93         exit(c)
  94
  95     def setOptions(self):
  96         RuntestBase.setOptions(self)
  97         self.longOptions.extend(['ext=','esc','escbin=','eval','threads=','ats',
  98                                  'atsdir=','verify','verifyonly','androidthreads'])
  99
 100     def parseOptions(self):
 101         opts = RuntestBase.parseOptions(self)
 102         for o, v in opts:
 103             if o in ('--ext',):
 104                 self.sourceExt = v
 105             elif o in ('--esc',):
 106                 self.runESC = True
 107             elif o in ('--escbin',):
 108                 self.escbin = v
 109             elif o in ('--eval',):
 110                 self.eval = True
 111             elif o in ('--androidthreads',):
 112                 self.androidthreads=True
 113                 self.threads=1
 114             elif o in ('--threads',):
 115                 try:
 116                     self.threads=int(v)
 117                 except ValueError:
 118                     print('Incorrect threads value: %s\n' % v)
 119                     self.usage(2)
 120             elif o in ('--ats',):
 121                 self.genAtsSwfs = True
 122                 self.rebuildtests = True
 123                 # Need to run single threaded since we create a temp file for
 124                 # every test and this file can collide when using multiple threads
 125                 self.threads = 1
 126             elif o in ('--atsdir',):
 127                 self.atsDir = v
 128             elif o in ('--verify',):
 129                 self.verify = True
 130                 self.vmargs = '-Dverifyall -Dverbose=verify'
 131             elif o in ('--verifyonly',):
 132                 self.verifyonly = True
 133                 if '-Dverifyonly' not in self.vmargs:
 134                     self.vmargs += ' -Dverifyonly'
 135             elif o in ('--remoteip',):
 136                 self.remoteip = v
 137             elif o in ('--remoteuser',):
 138                 self.remoteuser = v
 139
 140     def run(self):
 141         self.setEnvironVars()
 142         self.loadPropertiesFile()
 143         self.setOptions()
 144         self.parseOptions()
 145         self.setTimestamp()
 146         self.checkPath()
 147         if not self.config:
 148             self.determineConfig()
 149         if self.verifyonly and 'debugger' not in self.vmtype:
 150             exit('You must be running a debugger build in order to use the -Dverifyonly option')
 151         if self.rebuildtests==False and (re.search('arm-winmobile-emulator',self.config)!=None or self.osName=='winmobile'):
 152             if re.search('^arm-winmobile-emulator',self.config)==None:
 153                 print('ERROR: to use windows mobile build set --config arm-winmobile-emulator-tvm-release or install cygwin utility /usr/bin/file.exe')
 154                 sys.exit(1)
 155             self.setupCEEmulators()
 156         if self.htmlOutput and not self.rebuildtests:
 157             self.createOutputFile()
 158         # extension lists must be tuples
 159         self.otherTestExtensions = (self.abcasmExt,)
 160         self.executableExtensions = (self.abcOnlyExt,)
 161         # test configuration is contained in two files: failconfig & testconfig)
 162         self.settings = self.parseTestConfig(self.testconfig) & self.parsTestConfig(self.failconfig)
 163         self.tests = self.getTestsList(self.args)
 164         # Load root .asc_args and .java_args files
 165         self.parseRootConfigFiles()
 166         self.preProcessTests()
 167         if self.rebuildtests:
 168             self.rebuildTests()
 169         else:
 170             self.runTests(self.tests)
 171         self.cleanup()
 172
 173     def preProcessTests(self):
 174         RuntestBase.preProcessTests(self)
 175         # Are we running esc - depends on a valid avm
 176         if self.runESC:
 177             self.runSource = True
 178             # generate the executable cmd for esc
 179             #escAbcs = [f for f in os.listdir(self.escbin) if f.endswith('.abc')] #not all abcs are used for esc
 180             escAbcs = ['debug','util','bytes-tamarin','util-tamarin','lex-char','lex-token',
 181                        'lex-scan','ast','ast-decode','parse','asm','abc','emit','cogen',
 182                        'cogen-stmt','cogen-expr','esc-core','eval-support','esc-env','main']
 183             if not self.escbin.endswith('/'):
 184                 self.escbin += '/'
 185             for f in escAbcs:
 186                 self.avm += ' %s%s.es.abc' % (self.escbin, f)
 187             self.avm += ' -- '
 188             self.avm += ' %s../test/spidermonkey-prefix.es' % self.escbin  #needed to run shell harness
 189         if self.androidthreads:
 190             p=subprocess.Popen('adb devices',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 191             (out,err)=p.communicate()
 192             for line in out.split('\n'):
 193                 items=line.split()
 194                 if len(items)==2 and items[1]=='device':
 195                     for i in range(self.threads):
 196                         self.androiddevices.append(items[0])
 197             if len(self.androiddevices)==0:
 198                 print("error: adb did not detect any attached devices")
 199                 print("adb devices stdout: %s stderr: %s" % (out,err))
 200                 sys.exit(1)
 201             print("detected %d android devices" % (len(self.androiddevices)/self.threads))
 202             self.threads=len(self.androiddevices)
 203
 204     def runTestPrep(self, testAndNum):
 205         ast = testAndNum[0]
 206         testnum = testAndNum[1]
 207         outputCalls = [] #queue all output calls so that output is written in a block
 208         extraVmArgs = ''
 209         abcargs = ''
 210
 211         dir = os.path.split(ast)[0]
 212         root,ext = splitext(ast)
 213         if self.runSource or self.eval or ext in self.executableExtensions:
 214             testName = ast
 215         else:
 216             testName = root + '.abc'
 217
 218         includes = self.includes #list
 219
 220         settings = self.getLocalSettings(root)
 221
 222         # skip tests that can't be verified
 223         if self.verify and '.*' in settings and 'verify_skip' in settings['.*']:
 224             outputCalls.append((self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 225             outputCalls.append((self.js_print,('  skipping... reason: %s' % settings['.*']['verify_skip'],)))
 226             self.allskips += 1
 227             return outputCalls
 228
 229         # skip entire test if specified
 230         if '.*' in settings and 'skip' in settings['.*'] and not 'include' in settings['.*']:
 231             outputCalls.append((self.js_print,('  skipping... reason: %s' % settings['.*']['skip'],)))
 232             self.allskips += 1
 233             outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 234             return outputCalls
 235
 236         # Check for a timezone file
 237         if isfile('%s.tz' % ast):
 238             if not self.valid_time_zone(ast, testnum, outputCalls):
 239                 return outputCalls
 240
 241         # delete abc if forcerebuild
 242         if self.forcerebuild and isfile(testName) and ext not in self.executableExtensions:
 243             os.unlink(testName)
 244         if isfile(testName) and getmtime(ast)>getmtime(testName) and self.timestampcheck:
 245             outputCalls.append((self.verbose_print, ("%s has been modified, recompiling" % ast,)))
 246             os.unlink(testName)
 247
 248         # process support dir
 249         if exists(root+self.supportFolderExt):
 250             self.compile_support_files(root+self.supportFolderExt, outputCalls)
 251
 252         # compile file if needed
 253         if not isfile(testName):
 254             compileOutput = self.compile_test(ast, outputCalls=outputCalls)
 255             if not isfile(testName):
 256                 if ast.endswith(self.abcasmExt):
 257                     # file didn't compile, compare compile output
 258                     flines = self.compareAbcAsmOutput(ast, compileOutput)
 259                     if flines:
 260                         self.allfails += 1
 261                         outputCalls.append((self.fail,(testName, 'FAILED! :\nExpected:\n'+''.join(flines)+'\nGOT:\n'+''.join(compileOutput), self.failmsgs)))
 262                         outputCalls.append((self.js_print, ('   FAILED passes: 0 fails: 1 unexpected passes: 0 expected failures: 0', '', '<br/>')))
 263                     else:
 264                         self.allpasses += 1
 265                         outputCalls.append((self.verbose_print, ('   PASSED passes: 1 fails: 0 unexpected passes: 0 expected failures: 0', '', '<br/>')))
 266                     outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 267                     return outputCalls
 268                 else:
 269                     self.allfails += 1
 270                     outputCalls.append((self.js_print, ('%s' % '\n'.join(compileOutput),)))
 271                     outputCalls.append((self.fail,(testName, 'FAILED! file did not compile: %s' %
 272                                                    testName, self.failmsgs)))
 273                     return outputCalls
 274
 275         if self.runSource or self.eval:
 276             incfiles=self.build_incfiles(testName)
 277             incfiles.append("shell" + self.sourceExt)
 278             for incfile in incfiles:
 279                 testName=incfile+" "+testName
 280
 281         # read any extra avm arguments, each line will execute the avm with those args
 282         if isfile('%s.avm_args' % ast):
 283             avm_args_file = open('%s.avm_args' % ast,'r')
 284             index = 0
 285             uses_swfversion = re.compile('uses_swfversion', re.IGNORECASE)
 286             for line in avm_args_file:
 287                 line = line.strip()
 288                 if line.startswith('#'):
 289                     continue
 290                 index += 1
 291                 # uses_swfversion
 292                 if uses_swfversion.search(line):
 293                     # run avm with all bugcompat versions
 294                     for swf_ver in self.swfversions:
 295                         line = uses_swfversion.sub('', line)
 296                         line, extraVmArgs, abcargs = self.process_avm_args_line(line, dir)
 297                         extraVmArgs += ' -swfversion %s ' % swf_ver
 298                         outputCalls.extend(self.runTest(
 299                             ast, root, testName, '%s.%s' % (testnum, index),
 300                             settings, extraVmArgs, abcargs))
 301                         index += 1
 302                     continue
 303                 line, extraVmArgs, abcargs = self.process_avm_args_line(line, dir)
 304                 outputCalls.extend(self.runTest(ast, root, testName, '%s.%s' % (testnum, index), settings, extraVmArgs, abcargs))
 305         else:
 306             outputCalls.extend(self.runTest(ast, root, testName, testnum, settings))
 307
 308         return outputCalls
 309
 310     def valid_time_zone(self, testname, testnum, outputCalls):
 311         '''Read in the associated time zone (testname.as.tz) file and
 312             check if we match that timezone.  Returns a boolean indicating
 313             whether we are in the right timezone'''
 314         tz_file = open('%s.tz' % testname)
 315         valid_timezones = []
 316         for line in tz_file:
 317             line = line.strip()
 318             if not line or line.startswith('#'):
 319                 continue
 320             valid_timezones.append(line)
 321
 322         # windows python time.tzname returns time zone strings instead of
 323         # abbreviations.  Take the string and condense it so that it matches
 324         # the unix tzname values.  Not ideal, but works for most time zones.
 325         # e.g.: computer set to Pacific Standard Time
 326         # windows tzname = ('Pacific Standard Time', 'Pacific Daylight Time')
 327         # cygwin  tzname = ('PST', 'PDT')
 328         if self.osName == 'win' and not self.cygwin:
 329             # extract only the upper case letters from the windows time zone
 330             tzname = tuple([''.join(re.findall('[A-Z]',tz)) for tz in time.tzname])
 331         else:
 332             tzname = time.tzname
 333         # check if current tz is in the list
 334         if str(tzname) not in valid_timezones:
 335             outputCalls.append(
 336                     (self.js_print,
 337                     ('  skipping... reason: Current timezone: %s not in %s.tz timezones:\n' \
 338                      '                      %s'
 339                     % (tzname, testname, ','.join(valid_timezones)),)))
 340             outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, testname), '<b>', '</b><br/>')));
 341             self.allskips += 1
 342             return False
 343         return True
 344
 345     def compile_support_files(self, support_dir, outputCalls):
 346         for p, dirs, files in walk(support_dir):
 347             for f in files:
 348                 if f.endswith(self.sourceExt):
 349                     f = join(p,f)
 350                     binFile = splitext(f)[0]+'.abc'
 351                     if exists(binFile) and (self.forcerebuild or (self.timestampcheck and getmtime(f)>getmtime(binFile))):
 352                         os.unlink(binFile)
 353                     if not isfile(binFile):
 354                         compileOutput = self.compile_test(f, outputCalls=outputCalls)
 355                         if not isfile(binFile):
 356                             outputCalls.append((self.js_print,('  Error compiling support file: %s' % f,)))
 357                             outputCalls.append((self.verbose_print, ('   compile output: %s' % compileOutput,)))
 358
 359     def process_avm_args_line(self, line, dir):
 360         abcargs = ''
 361         line = line.replace("$DIR", dir)
 362         if line.find('--') != -1:
 363             (extraVmArgs, abcargs) = line.split('--')
 364         else:
 365             extraVmArgs = line
 366         return line, extraVmArgs, abcargs
 367
 368     def runTest(self, ast, root, testName, testnum, settings, extraVmArgs='', abcargs=''):
 369         if self.androidthreads:
 370             try:
 371                 if threading.currentThread().getName()=='MainThread':
 372                     n=0
 373                 else:
 374                     n=int(threading.currentThread().getName()[7:])-1
 375                 if n<len(self.androiddevices):
 376                    extraVmArgs+=" --threadid=%d --androidid=%s %s" % (n,self.androiddevices[n],extraVmArgs)
 377             except:
 378                 print(sys.exc_info())
 379         outputCalls = []
 380         lpass = 0
 381         lfail = 0
 382         lexpfail = 0
 383         lunpass = 0
 384         ltimeout = 0
 385         lassert = 0
 386         starttime=time.time()
 387
 388         if self.aotsdk and self.aotout:
 389             avm_args = ""
 390             if isfile("%s.avm_args" % ast):
 391                 avm_args = open("%s.avm_args" % ast).readline()
 392                 if avm_args.find("mops") >= 0:
 393                     avm_args = ""
 394             progname = testName.replace(".abc", "")
 395             progname = progname.replace("/", ".")
 396             progpath = os.path.join(self.aotout, progname)
 397             if self.remoteip:
 398                 retryCount = 5
 399                 while retryCount > 0:
 400                     # copy file to remote device
 401                     cmd = "scp %s %s@%s:" % (progpath, self.remoteuser, self.remoteip)
 402                     self.run_pipe(cmd, outputCalls=outputCalls)
 403                     # run app
 404                     cmd = "ssh %s@%s ./%s %s" % (self.remoteuser, self.remoteip, progname, avm_args)
 405                     (f,err,exitcode) = self.run_pipe(cmd, outputCalls=outputCalls)
 406                     if exitcode != 0:
 407                         sleep(5)
 408                         retryCount -= 1
 409                     else:
 410                         break
 411                 # delete app
 412                 cmd = "ssh %s@%s rm %s" % (self.remoteuser, self.remoteip, progname)
 413                 self.run_pipe(cmd, outputCalls=outputCalls)
 414             else:
 415                 cmd = "%s %s" % (progpath, avm_args)
 416                 # print("about to execute: " + cmd)
 417                 (f,err,exitcode) = self.run_pipe(cmd, outputCalls=outputCalls)
 418         elif ast.endswith(self.abcasmExt):
 419             # make sure util file has been compiled
 420             if not exists(self.abcasmShell+'.abc'):  # compile abcasmShell with no additional args
 421                 self.run_pipe('"%s" -jar %s %s' % (self.java, self.asc, self.abcasmShell+'.as'), outputCalls=outputCalls)
 422             (f,err,exitcode) = self.run_pipe('%s %s %s %s %s' % (self.avm, self.vmargs, extraVmArgs, self.abcasmShell+'.abc', testName), outputCalls=outputCalls)
 423         elif self.verify:
 424             # get the abcdump for the file
 425             (f,err,exitcode) = self.run_pipe('%s %s -- %s' % (self.avm, self.abcdump+'.abc', testName), outputCalls=outputCalls)
 426             abcdumpFunctions = [line.strip() for line in f if line.startswith('var function')]
 427             if self.verbose:
 428                 outputCalls.append((self.js_print,(abcdumpFunctions,)))
 429             # get -Dverifyall -Dverbose=verify output
 430             (f,err,exitcode) = self.run_pipe('%s %s %s' % (self.avm, self.vmargs, testName), outputCalls=outputCalls)
 431             verifyFunctions = [line.strip() for line in f if line.startswith('verify Function/')]
 432             if self.verbose:
 433                 outputCalls.append((self.js_print,(verifyFunctions,)))
 434
 435             # we can't compare actual function names since abcdump treats function names and var names the same
 436             # we instead just compare that the # of functions verified == the # of functions listed out by abcdump
 437             if len(abcdumpFunctions) != len(verifyFunctions):
 438                 lfail += 1
 439                 outputCalls.append((self.fail,(testName, 'FAILED! :\nExpected (from abcdump): %s functions verified\nGOT (-Dverifyall -Dverbose=verify): %s functions verified' %
 440                                                (len(abcdumpFunctions),len(verifyFunctions)), self.failmsgs)))
 441             else:
 442                 lpass += 1
 443         else:
 444             if abcargs:
 445                 (f,err,exitcode) = self.run_pipe('%s %s %s %s -- %s' % (self.avm, self.vmargs, extraVmArgs, testName, abcargs), outputCalls=outputCalls)
 446             else:
 447                 (f,err,exitcode) = self.run_pipe('%s %s %s %s' % (self.avm, self.vmargs, extraVmArgs, testName), outputCalls=outputCalls)
 448
 449         # Test has been run, handle output
 450         if self.verifyonly:
 451             # only check the exit code when running a verifyonly pass
 452             ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
 453             if ec_lfail or ec_lexpfail:
 454                 lfail += ec_lfail
 455                 lexpfail += ec_lexpfail
 456             else:
 457                 lpass += 1
 458         elif not self.verify:
 459             try:
 460                 outputLines = []
 461                 if isfile(root+'.out'):
 462                     # override standard runtests behavior, just compare the .out file with stdout+stderr
 463                     actual = [line.strip() for line in f+err if line.strip()]
 464                     try:
 465                         outfile = open(root+'.out', 'r')
 466                         expectedOut = [line.strip() for line in outfile.readlines() if line.strip()]
 467                         outfile.close()
 468                         outputCalls.append((self.verbose_print,('%s.out file (expected):' % root,)))
 469                         outputCalls.append((self.verbose_print,(expectedOut,)))
 470                         outputCalls.append((self.verbose_print,('\nactual output:',)))
 471                         outputCalls.append((self.verbose_print,(actual,)))
 472                         # check settings if this should be an expected failure
 473                         expectedfail = dict_match(settings,'','expectedfail')
 474                         # .out files can contain regex but must be prefaced with REGEXP as the first line in the file
 475                         try:
 476                             if expectedOut[0] == 'REGEXP':
 477                                 expectedOut = expectedOut[1:]
 478                                 if len(actual) < len(expectedOut):
 479                                     # pad actual output w/ empty lines
 480                                     for i in range(len(expectedOut)-len(actual)):
 481                                         actual.append('')
 482                                 for i in range(len(actual)):
 483                                     if not re.search(expectedOut[i], actual[i]):
 484                                         raise IndexError
 485                             else:
 486                                 if actual != expectedOut:
 487                                     raise IndexError
 488                             # test passed - check to make sure its not an expected failure
 489                             if expectedfail:
 490                                 outputCalls.append((self.fail,(
 491                                     testName, 'unexpected .out file pass. ' +
 492                                     ' reason: '+expectedfail, self.unpassmsgs)))
 493                                 lunpass += 1
 494                             else:
 495                                 lpass += 1
 496                         except IndexError:
 497                             # test failed
 498                             if expectedfail:
 499                                 outputCalls.append((self.fail,(
 500                                     testName,
 501                                     'expected failure: .out file does not match stdout+stderr. ' +
 502                                     ' reason: '+expectedfail, self.expfailmsgs)))
 503                                 lexpfail += 1
 504                             else:
 505                                 outputCalls.append((self.fail,(testName,
 506                                     '.out file does not match output:\n%s.out file (expected):\n%s\nactual output:\n%s' % (root, expectedOut,actual),
 507                                     self.failmsgs)))
 508                                 lfail += 1
 509                     except IOError:
 510                         outputLines.append((self.js_print,('Error opening %s.out' % root,)))
 511                         lfail += 1
 512                 else:
 513                     for line in f+err:
 514                         outputLines.append(line)
 515                         outputCalls.append((self.verbose_print,('   %s' % line.strip(),)))
 516                         if 'Assertion failed:' in line:
 517                             lassert += 1
 518                             outputCalls.append((self.fail,(testName+extraVmArgs, line, self.assertmsgs)))
 519                         testcase=''
 520                         if len(line)>9:
 521                             testcase=line.strip()
 522                         skipTestDesc = dict_match(settings,testcase,'skip')
 523                         includeTestDesc = dict_match(settings, testcase, 'include')
 524                         if skipTestDesc and not includeTestDesc:
 525                             outputCalls.append((self.js_print,('  skipping "%s" ... reason: %s' % (line.strip(),skipTestDesc),)))
 526                             self.allskips+=1
 527                             continue
 528                         if 'PASSED!' in line:
 529                             res=dict_match(settings,testcase,'expectedfail')
 530                             if res:
 531                                 outputCalls.append((self.fail,(testName, 'unexpected pass: ' + line.strip() + ' reason: '+res, self.unpassmsgs)))
 532                                 lunpass += 1
 533                             else:
 534                                 lpass += 1
 535                         if 'FAILED!' in line:
 536                             res=dict_match(settings,testcase,'expectedfail')
 537                             if res:
 538                                 outputCalls.append((self.fail,(testName, 'expected failure: ' + line.strip() + ' reason: '+res, self.expfailmsgs)))
 539                                 lexpfail += 1
 540                             else:
 541                                 lfail += 1
 542                                 outputCalls.append((self.fail,(testName+extraVmArgs, line, self.failmsgs)))
 543             except:
 544                 print('exception running avm')
 545                 raise
 546
 547             # exitcode check
 548             ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
 549             if ec_lfail or ec_lexpfail:
 550                 lfail += ec_lfail
 551                 lexpfail += ec_lexpfail
 552             elif err:
 553                 # TODO: When needed, add support for expected stderr output - see https://bugzilla.mozilla.org/show_bug.cgi?id=561892
 554                 outputCalls.append((self.fail,(testName, "unexpected stderr expected:'%s' actual:'%s'" % ('',err), self.failmsgs)))
 555                 lfail += 1  # any messages to stderr automatically fail the test
 556             elif lpass == 0 and lfail == 0 and lunpass==0 and lexpfail==0:
 557                 res=dict_match(settings,'*','expectedfail')
 558                 if res:
 559                     outputCalls.append((self.fail,(testName, 'expected failure: FAILED contained no testcase messages reason: %s' % res,self.expfailmsgs)))
 560                     lexpfail += 1
 561                 else:
 562                     # test may not have any output but is still passing if a non-standard exitcode was expected (e.g. VerifyErrors)
 563                     if expectedExitcode != 0 and exitcode==expectedExitcode:
 564                         #outputCalls.append((self.js_print,('  Expected Exit Code: %s Actual Exit Code: %s PASSED' % (expectedExitcode, exitcode),)))
 565                         lpass = 1
 566                     else:
 567                         lfail = 1
 568                         outputmsg=''
 569                         for l in outputLines:
 570                             outputmsg+=l.strip()+'|'
 571                         outputCalls.append((self.fail,(testName, '   FAILED contained no testcase messages - reason: %s' % outputmsg, self.failmsgs)))
 572
 573         self.allfails += lfail
 574         self.allpasses += lpass
 575         self.allexpfails += lexpfail
 576         self.allunpass += lunpass
 577         self.alltimeouts += ltimeout
 578         self.allasserts += lassert
 579         if lfail or lunpass:
 580             outputCalls.append((self.js_print, ('   FAILED passes:%d fails:%d unexpected passes: %d expected failures: %d\n' % (lpass,lfail,lunpass,lexpfail), '', '<br/>')))
 581         else:
 582             outputCalls.append((self.verbose_print, ('   PASSED passes:%d fails:%d unexpected passes: %d expected failures: %d\n' % (lpass,lfail,lunpass,lexpfail), '', '<br/>')))
 583         if self.show_time:
 584             outputCalls.insert(0,(self.js_print,('%s running %s %s %s time %.1f' % (testnum, ast, extraVmArgs, abcargs, time.time()-starttime), '<b>', '</b><br/>')));
 585         else:
 586             outputCalls.insert(0,(self.js_print,('%s running %s %s %s' % (testnum, ast, extraVmArgs, abcargs), '<b>', '</b><br/>')));
 587
 588
 589         return outputCalls
 590
 591     def checkExitCode(self, exitcode, root, testName, f, err, settings, outputCalls):
 592         '''Check the exitcode for a test against any expected non-zero exitcodes
 593             Return the fail and the expected exitcode if non-zero
 594         '''
 595         lfail = 0
 596         lexpfail = 0
 597         expectedExitcode=0
 598         if isfile(root+'.err'):
 599             # .err file holds both the expected (non-catchable) error (usually a VerifyError) and the expected exitcode
 600             expectedErr,expectedExitcode = self.loadExpectedErr(root+'.err')
 601             # check the expectedErr - error is always the last (non-empty) line of output
 602             actualErr = ''
 603             for line in reversed(f):
 604                 # When running in --verifyonly mode, output will be VERIFY FAILED instead of VerifyError
 605                 line = line.replace('VERIFY FAILED', 'VerifyError')
 606                 # make sure we have an error
 607                 if re.search('.*Error:.*', line):
 608                     actualErr = self.getError(line.strip())
 609                     break
 610             if actualErr != expectedErr:
 611                 outputCalls.append((self.fail,(testName, 'unexpected error message. expected: %s actual: %s'
 612                                                % (expectedErr, actualErr), self.failmsgs)))
 613                 lfail += 1
 614         elif isfile(root+".exitcode"):
 615             try:
 616                 expectedExitcode=int(open(root+".exitcode").read())
 617             except:
 618                 print("ERROR: reading exit code file '%s' should contain an integer")
 619         res=dict_match(settings,'exitcode','expectedfail')
 620
 621         if exitcode!=expectedExitcode:
 622             res2=dict_match(settings,'exitcode','skip')
 623             if res2==None and res:
 624                 outputCalls.append((self.js_print,(testName, 'expected failure: exitcode reason: %s'
 625                                                    % res,self.expfailmsgs)))
 626                 lexpfail += 1
 627             elif res2==None:
 628                 outputCalls.append((self.fail,(testName, 'unexpected exit code expected:%d actual:%d Signal Name: %s FAILED!'
 629                                                % (expectedExitcode,exitcode,getSignalName(abs(exitcode))),
 630                                                self.failmsgs)))
 631                 outputmsg=''
 632                 for l in f+err:
 633                     outputmsg+=l.strip()+'|'
 634                 outputCalls.append((self.fail,(testName, 'captured output: %s'
 635                                                % outputmsg,self.failmsgs)))
 636                 lfail+= 1
 637         return lfail, lexpfail, expectedExitcode
 638
 639     def loadExpectedErr(self, file):
 640         try:
 641             f = open(file, 'r')
 642             expectedErr = ''
 643             expectedExitcode = 0
 644             for line in f:
 645                 line = line.strip()
 646                 if line and not line.startswith('#'):
 647                     if line.lower().startswith('exitcode:'):
 648                         try:
 649                             expectedExitcode = int(line.split(':')[1].strip())
 650                         except ValueError:
 651                             pass
 652                     else:
 653                         # check to see if line is a number - if so set expectedExitcode
 654                         try:
 655                             expectedExitcode = int(line)
 656                         except ValueError:
 657                             expectedErr = line
 658             return expectedErr, expectedExitcode
 659         except:
 660             return 'Error reading .err file: %s' % file, 0
 661
 662     def getError(self, line):
 663         # Parse out the error type and #, but strip the description
 664         try:
 665             return ':'.join(line.split(':')[0:2])
 666         except:
 667             return line
 668
 669 runtest = AcceptanceRuntest()