test/acceptance/runtests.py

   1 #!/usr/bin/env python
   2 # -*- Mode: Python; indent-tabs-mode: nil; tab-width: 4 -*-
   3 # vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5)
   4 #
   5 # ***** BEGIN LICENSE BLOCK *****
   6 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   7 #
   8 # The contents of this file are subject to the Mozilla Public License Version
   9 # 1.1 (the "License"); you may not use this file except in compliance with
  10 # the License. You may obtain a copy of the License at
  11 # http://www.mozilla.org/MPL/
  12 #
  13 # Software distributed under the License is distributed on an "AS IS" basis,
  14 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  15 # for the specific language governing rights and limitations under the
  16 # License.
  17 #
  18 # The Original Code is [Open Source Virtual Machine.].
  19 #
  20 # The Initial Developer of the Original Code is
  21 # Adobe System Incorporated.
  22 # Portions created by the Initial Developer are Copyright (C) 2005-2009
  23 # the Initial Developer. All Rights Reserved.
  24 #
  25 # Contributor(s):
  26 #   christine@netscape.com
  27 #   Adobe AS3 Team
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK ***** */
  42 #
  43
  44 import os, sys, getopt, datetime, pipes, glob, itertools, tempfile, string
  45 import re, platform, threading, time
  46 from os.path import *
  47 from os import getcwd,environ,walk
  48 from datetime import datetime
  49 from glob import glob
  50 from sys import argv, exit
  51 from getopt import getopt
  52 from itertools import count
  53
  54
  55 # add parent dir to python module search path
  56 sys.path.append('..')
  57 from util.runtestBase import RuntestBase
  58 # runtestUtils must be imported after "from os.path import *" as walk is overridden
  59 from util.runtestUtils import *
  60
  61 class AcceptanceRuntest(RuntestBase):
  62     runESC = False
  63     escbin = '../../esc/bin/'
  64     androidthreads = False
  65     androiddevices = []
  66     verifyonly = False
  67     swfversions = [9,10,11,12]
  68
  69     def __init__(self):
  70         # Set threads to # of available cpus/cores
  71         self.threads = detectCPUs()
  72         RuntestBase.__init__(self)
  73
  74     def setEnvironVars(self):
  75         RuntestBase.setEnvironVars(self)
  76         if 'ESCBIN' in environ:
  77             self.escbin = environ['ESCBIN'].strip()
  78
  79     def usage(self, c):
  80         RuntestBase.usage(self, c)
  81         print('    --esc           run esc instead of avm')
  82         print('    --escbin        location of esc/bin directory - defaults to ../../esc/bin')
  83         print('    --eval          use run-time compiler')
  84         print('    --ext           set the testfile extension (defaults to .as)')
  85         print('    --ats           generate ats swfs instead of running tests')
  86         print('    --atsdir        base output directory for ats swfs - defaults to ATS_SWFS')
  87         print('    --threads       number of threads to run (default=# of cpu/cores), set to 1 to have tests finish sequentially')
  88         print('    --androidthreads    assign a thread for each android device connected.')
  89         print('    --verify        run a verify pass instead of running abcs')
  90         print('    --verifyonly    run a -Dverifyonly pass: only checks test exitcode')
  91         print('    --remoteip      IP/DNS address of the machine to run the tests on.')
  92         print('    --remoteuser    user name to use to connect to the remote machine.')
  93         exit(c)
  94
  95     def setOptions(self):
  96         RuntestBase.setOptions(self)
  97         self.longOptions.extend(['ext=','esc','escbin=','eval','threads=','ats',
  98                                  'atsdir=','verify','verifyonly','androidthreads'])
  99
 100     def parseOptions(self):
 101         opts = RuntestBase.parseOptions(self)
 102         for o, v in opts:
 103             if o in ('--ext',):
 104                 self.sourceExt = v
 105             elif o in ('--esc',):
 106                 self.runESC = True
 107             elif o in ('--escbin',):
 108                 self.escbin = v
 109             elif o in ('--eval',):
 110                 self.eval = True
 111             elif o in ('--androidthreads',):
 112                 self.androidthreads=True
 113                 self.threads=1
 114             elif o in ('--threads',):
 115                 try:
 116                     self.threads=int(v)
 117                 except ValueError:
 118                     print('Incorrect threads value: %s\n' % v)
 119                     self.usage(2)
 120             elif o in ('--ats',):
 121                 self.genAtsSwfs = True
 122                 self.rebuildtests = True
 123                 # Need to run single threaded since we create a temp file for
 124                 # every test and this file can collide when using multiple threads
 125                 self.threads = 1
 126             elif o in ('--atsdir',):
 127                 self.atsDir = v
 128             elif o in ('--verify',):
 129                 self.verify = True
 130                 self.vmargs = '-Dverifyall -Dverbose=verify'
 131             elif o in ('--verifyonly',):
 132                 self.verifyonly = True
 133                 if '-Dverifyonly' not in self.vmargs:
 134                     self.vmargs += ' -Dverifyonly'
 135             elif o in ('--remoteip',):
 136                 self.remoteip = v
 137             elif o in ('--remoteuser',):
 138                 self.remoteuser = v
 139
 140     def run(self):
 141         self.setEnvironVars()
 142         self.loadPropertiesFile()
 143         self.setOptions()
 144         self.parseOptions()
 145         self.setTimestamp()
 146         self.checkPath()
 147         if not self.config:
 148             self.determineConfig()
 149         if self.verifyonly and 'debugger' not in self.vmtype:
 150             exit('You must be running a debugger build in order to use the -Dverifyonly option')
 151         if self.rebuildtests==False and (re.search('arm-winmobile-emulator',self.config)!=None or self.osName=='winmobile'):
 152             if re.search('^arm-winmobile-emulator',self.config)==None:
 153                 print('ERROR: to use windows mobile build set --config arm-winmobile-emulator-tvm-release or install cygwin utility /usr/bin/file.exe')
 154                 sys.exit(1)
 155             self.setupCEEmulators()
 156         if self.htmlOutput and not self.rebuildtests:
 157             self.createOutputFile()
 158         # extension lists must be tuples
 159         self.otherTestExtensions = (self.abcasmExt,)
 160         self.executableExtensions = (self.abcOnlyExt,)
 161         # test configuration is contained in two files: failconfig & testconfig)
 162         self.settings = self.parseTestConfig(self.testconfig) & self.parsTestConfig(self.failconfig)
 163         self.tests = self.getTestsList(self.args)
 164         # Load root .asc_args and .java_args files
 165         self.parseRootConfigFiles()
 166         self.preProcessTests()
 167         if self.rebuildtests:
 168             self.rebuildTests()
 169         else:
 170             self.runTests(self.tests)
 171         self.cleanup()
 172
 173     def preProcessTests(self):
 174         RuntestBase.preProcessTests(self)
 175         # Are we running esc - depends on a valid avm
 176         if self.runESC:
 177             self.runSource = True
 178             # generate the executable cmd for esc
 179             #escAbcs = [f for f in os.listdir(self.escbin) if f.endswith('.abc')] #not all abcs are used for esc
 180             escAbcs = ['debug','util','bytes-tamarin','util-tamarin','lex-char','lex-token',
 181                        'lex-scan','ast','ast-decode','parse','asm','abc','emit','cogen',
 182                        'cogen-stmt','cogen-expr','esc-core','eval-support','esc-env','main']
 183             if not self.escbin.endswith('/'):
 184                 self.escbin += '/'
 185             for f in escAbcs:
 186                 self.avm += ' %s%s.es.abc' % (self.escbin, f)
 187             self.avm += ' -- '
 188             self.avm += ' %s../test/spidermonkey-prefix.es' % self.escbin  #needed to run shell harness
 189         if self.androidthreads:
 190             p=subprocess.Popen('adb devices',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 191             (out,err)=p.communicate()
 192             for line in out.split('\n'):
 193                 items=line.split()
 194                 if len(items)==2 and items[1]=='device':
 195                     for i in range(self.threads):
 196                         self.androiddevices.append(items[0])
 197             if len(self.androiddevices)==0:
 198                 print("error: adb did not detect any attached devices")
 199                 print("adb devices stdout: %s stderr: %s" % (out,err))
 200                 sys.exit(1)
 201             print("detected %d android devices" % (len(self.androiddevices)/self.threads))
 202             self.threads=len(self.androiddevices)
 203
 204     def runTestPrep(self, testAndNum):
 205         ast = testAndNum[0]
 206         testnum = testAndNum[1]
 207         outputCalls = [] #queue all output calls so that output is written in a block
 208         extraVmArgs = ''
 209         abcargs = ''
 210
 211         dir = os.path.split(ast)[0]
 212         root,ext = splitext(ast)
 213         if self.runSource or self.eval or ext in self.executableExtensions:
 214             testName = ast
 215         else:
 216             testName = root + '.abc'
 217
 218         includes = self.includes #list
 219
 220         settings = self.getLocalSettings(root)
 221
 222         # skip tests that can't be verified
 223         if self.verify and '.*' in settings and 'verify_skip' in settings['.*']:
 224             outputCalls.append((self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 225             outputCalls.append((self.js_print,('  skipping... reason: %s' % settings['.*']['verify_skip'],)))
 226             self.allskips += 1
 227             return outputCalls
 228
 229         # skip entire test if specified
 230         if '.*' in settings and 'skip' in settings['.*'] and not 'include' in settings['.*']:
 231             outputCalls.append((self.js_print,('  skipping... reason: %s' % settings['.*']['skip'],)))
 232             self.allskips += 1
 233             outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 234             return outputCalls
 235
 236         # Check for a timezone file
 237         if isfile('%s.tz' % ast):
 238             if not self.valid_time_zone(ast, testnum, outputCalls):
 239                 return outputCalls
 240
 241         # delete abc if forcerebuild
 242         if self.forcerebuild and isfile(testName) and ext not in self.executableExtensions:
 243             os.unlink(testName)
 244         if isfile(testName) and getmtime(ast)>getmtime(testName) and self.timestampcheck:
 245             outputCalls.append((self.verbose_print, ("%s has been modified, recompiling" % ast,)))
 246             os.unlink(testName)
 247
 248         # process support dir
 249         if exists(root+self.supportFolderExt):
 250             self.compile_support_files(root+self.supportFolderExt, outputCalls)
 251
 252         # compile file if needed
 253         if not isfile(testName):
 254             compileOutput = self.compile_test(ast, outputCalls=outputCalls)
 255             if not isfile(testName):
 256                 if ast.endswith(self.abcasmExt):
 257                     # file didn't compile, compare compile output
 258                     flines = self.compareAbcAsmOutput(ast, compileOutput)
 259                     if flines:
 260                         self.allfails += 1
 261                         outputCalls.append((self.fail,(testName, 'FAILED! :\nExpected:\n'+''.join(flines)+'\nGOT:\n'+''.join(compileOutput), self.failmsgs)))
 262                         outputCalls.append((self.js_print, ('   FAILED passes: 0 fails: 1 unexpected passes: 0 expected failures: 0', '', '<br/>')))
 263                     else:
 264                         self.allpasses += 1
 265                         outputCalls.append((self.verbose_print, ('   PASSED passes: 1 fails: 0 unexpected passes: 0 expected failures: 0', '', '<br/>')))
 266                     outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, ast), '<b>', '</b><br/>')));
 267                     return outputCalls
 268                 else:
 269                     self.allfails += 1
 270                     outputCalls.append((self.js_print, ('%s' % '\n'.join(compileOutput),)))
 271                     outputCalls.append((self.fail,(testName, 'FAILED! file did not compile: %s' %
 272                                                    testName, self.failmsgs)))
 273                     return outputCalls
 274
 275         if self.runSource or self.eval:
 276             incfiles=self.build_incfiles(testName)
 277             incfiles.append("shell" + self.sourceExt)
 278             for incfile in incfiles:
 279                 testName=incfile+" "+testName
 280
 281         # read any extra avm arguments, each line will execute the avm with those args
 282         if isfile('%s.avm_args' % ast):
 283             avm_args_file = open('%s.avm_args' % ast,'r')
 284             index = 0
 285             uses_swfversion = re.compile('uses_swfversion', re.IGNORECASE)
 286             for line in avm_args_file:
 287                 line = line.strip()
 288                 if line.startswith('#'):
 289                     continue
 290                 index += 1
 291                 # uses_swfversion
 292                 if uses_swfversion.search(line):
 293                     # run avm with all bugcompat versions
 294                     for swf_ver in self.swfversions:
 295                         line = uses_swfversion.sub('', line)
 296                         line, extraVmArgs, abcargs = self.process_avm_args_line(line, dir)
 297                         extraVmArgs += ' -swfversion %s ' % swf_ver
 298                         outputCalls.extend(self.runTest(
 299                             ast, root, testName, '%s.%s' % (testnum, index),
 300                             settings, extraVmArgs, abcargs))
 301                         index += 1
 302                     continue
 303                 line, extraVmArgs, abcargs = self.process_avm_args_line(line, dir)
 304                 outputCalls.extend(self.runTest(ast, root, testName, '%s.%s' % (testnum, index), settings, extraVmArgs, abcargs))
 305         else:
 306             outputCalls.extend(self.runTest(ast, root, testName, testnum, settings))
 307
 308         return outputCalls
 309
 310     def valid_time_zone(self, testname, testnum, outputCalls):
 311         '''Read in the associated time zone (testname.as.tz) file and
 312             check if we match that timezone.  Returns a boolean indicating
 313             whether we are in the right timezone'''
 314         tz_file = open('%s.tz' % testname)
 315         valid_timezones = []
 316         for line in tz_file:
 317             line = line.strip()
 318             if not line or line.startswith('#'):
 319                 continue
 320             valid_timezones.append(line)
 321
 322         # windows python time.tzname returns time zone strings instead of
 323         # abbreviations.  Take the string and condense it so that it matches
 324         # the unix tzname values.  Not ideal, but works for most time zones.
 325         # e.g.: computer set to Pacific Standard Time
 326         # windows tzname = ('Pacific Standard Time', 'Pacific Daylight Time')
 327         # cygwin  tzname = ('PST', 'PDT')
 328         if self.osName == 'win' and not self.cygwin:
 329             print('win tz: ', time.tzname)
 330             # extract only the upper case letters from the windows time zone
 331             tzname = tuple([''.join(re.findall('[A-Z]',tz)) for tz in time.tzname])
 332         else:
 333             tzname = time.tzname
 334         print('tzname = ', tzname)
 335         # check if current tz is in the list
 336         if str(tzname) not in valid_timezones:
 337             outputCalls.append(
 338                     (self.js_print,
 339                     ('  skipping... reason: Current timezone: %s not in %s.tz timezones:\n' \
 340                      '                      %s'
 341                     % (tzname, testname, ','.join(valid_timezones)),)))
 342             outputCalls.insert(0,(self.js_print,('%d running %s' % (testnum, testname), '<b>', '</b><br/>')));
 343             self.allskips += 1
 344             return False
 345         return True
 346
 347     def compile_support_files(self, support_dir, outputCalls):
 348         for p, dirs, files in walk(support_dir):
 349             for f in files:
 350                 if f.endswith(self.sourceExt):
 351                     f = join(p,f)
 352                     binFile = splitext(f)[0]+'.abc'
 353                     if exists(binFile) and (self.forcerebuild or (self.timestampcheck and getmtime(f)>getmtime(binFile))):
 354                         os.unlink(binFile)
 355                     if not isfile(binFile):
 356                         compileOutput = self.compile_test(f, outputCalls=outputCalls)
 357                         if not isfile(binFile):
 358                             outputCalls.append((self.js_print,('  Error compiling support file: %s' % f,)))
 359                             outputCalls.append((self.verbose_print, ('   compile output: %s' % compileOutput,)))
 360
 361     def process_avm_args_line(self, line, dir):
 362         abcargs = ''
 363         line = line.replace("$DIR", dir)
 364         if line.find('--') != -1:
 365             (extraVmArgs, abcargs) = line.split('--')
 366         else:
 367             extraVmArgs = line
 368         return line, extraVmArgs, abcargs
 369
 370     def runTest(self, ast, root, testName, testnum, settings, extraVmArgs='', abcargs=''):
 371         if self.androidthreads:
 372             try:
 373                 if threading.currentThread().getName()=='MainThread':
 374                     n=0
 375                 else:
 376                     n=int(threading.currentThread().getName()[7:])-1
 377                 if n<len(self.androiddevices):
 378                    extraVmArgs+=" --threadid=%d --androidid=%s %s" % (n,self.androiddevices[n],extraVmArgs)
 379             except:
 380                 print(sys.exc_info())
 381         outputCalls = []
 382         lpass = 0
 383         lfail = 0
 384         lexpfail = 0
 385         lunpass = 0
 386         ltimeout = 0
 387         lassert = 0
 388         starttime=time.time()
 389
 390         if self.aotsdk and self.aotout:
 391             avm_args = ""
 392             if isfile("%s.avm_args" % ast):
 393                 avm_args = open("%s.avm_args" % ast).readline()
 394                 if avm_args.find("mops") >= 0:
 395                     avm_args = ""
 396             progname = testName.replace(".abc", "")
 397             progname = progname.replace("/", ".")
 398             progpath = os.path.join(self.aotout, progname)
 399             if self.remoteip:
 400                 retryCount = 5
 401                 while retryCount > 0:
 402                     # copy file to remote device
 403                     cmd = "scp %s %s@%s:" % (progpath, self.remoteuser, self.remoteip)
 404                     self.run_pipe(cmd, outputCalls=outputCalls)
 405                     # run app
 406                     cmd = "ssh %s@%s ./%s %s" % (self.remoteuser, self.remoteip, progname, avm_args)
 407                     (f,err,exitcode) = self.run_pipe(cmd, outputCalls=outputCalls)
 408                     if exitcode != 0:
 409                         sleep(5)
 410                         retryCount -= 1
 411                     else:
 412                         break
 413                 # delete app
 414                 cmd = "ssh %s@%s rm %s" % (self.remoteuser, self.remoteip, progname)
 415                 self.run_pipe(cmd, outputCalls=outputCalls)
 416             else:
 417                 cmd = "%s %s" % (progpath, avm_args)
 418                 # print("about to execute: " + cmd)
 419                 (f,err,exitcode) = self.run_pipe(cmd, outputCalls=outputCalls)
 420         elif ast.endswith(self.abcasmExt):
 421             # make sure util file has been compiled
 422             if not exists(self.abcasmShell+'.abc'):  # compile abcasmShell with no additional args
 423                 self.run_pipe('"%s" -jar %s %s' % (self.java, self.asc, self.abcasmShell+'.as'), outputCalls=outputCalls)
 424             (f,err,exitcode) = self.run_pipe('%s %s %s %s %s' % (self.avm, self.vmargs, extraVmArgs, self.abcasmShell+'.abc', testName), outputCalls=outputCalls)
 425         elif self.verify:
 426             # get the abcdump for the file
 427             (f,err,exitcode) = self.run_pipe('%s %s -- %s' % (self.avm, self.abcdump+'.abc', testName), outputCalls=outputCalls)
 428             abcdumpFunctions = [line.strip() for line in f if line.startswith('var function')]
 429             if self.verbose:
 430                 outputCalls.append((self.js_print,(abcdumpFunctions,)))
 431             # get -Dverifyall -Dverbose=verify output
 432             (f,err,exitcode) = self.run_pipe('%s %s %s' % (self.avm, self.vmargs, testName), outputCalls=outputCalls)
 433             verifyFunctions = [line.strip() for line in f if line.startswith('verify Function/')]
 434             if self.verbose:
 435                 outputCalls.append((self.js_print,(verifyFunctions,)))
 436
 437             # we can't compare actual function names since abcdump treats function names and var names the same
 438             # we instead just compare that the # of functions verified == the # of functions listed out by abcdump
 439             if len(abcdumpFunctions) != len(verifyFunctions):
 440                 lfail += 1
 441                 outputCalls.append((self.fail,(testName, 'FAILED! :\nExpected (from abcdump): %s functions verified\nGOT (-Dverifyall -Dverbose=verify): %s functions verified' %
 442                                                (len(abcdumpFunctions),len(verifyFunctions)), self.failmsgs)))
 443             else:
 444                 lpass += 1
 445         else:
 446             if abcargs:
 447                 (f,err,exitcode) = self.run_pipe('%s %s %s %s -- %s' % (self.avm, self.vmargs, extraVmArgs, testName, abcargs), outputCalls=outputCalls)
 448             else:
 449                 (f,err,exitcode) = self.run_pipe('%s %s %s %s' % (self.avm, self.vmargs, extraVmArgs, testName), outputCalls=outputCalls)
 450
 451         # Test has been run, handle output
 452         if self.verifyonly:
 453             # only check the exit code when running a verifyonly pass
 454             ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
 455             if ec_lfail or ec_lexpfail:
 456                 lfail += ec_lfail
 457                 lexpfail += ec_lexpfail
 458             else:
 459                 lpass += 1
 460         elif not self.verify:
 461             try:
 462                 outputLines = []
 463                 if isfile(root+'.out'):
 464                     # override standard runtests behavior, just compare the .out file with stdout+stderr
 465                     actual = [line.strip() for line in f+err if line.strip()]
 466                     try:
 467                         outfile = open(root+'.out', 'r')
 468                         expectedOut = [line.strip() for line in outfile.readlines() if line.strip()]
 469                         outfile.close()
 470                         outputCalls.append((self.verbose_print,('%s.out file (expected):' % root,)))
 471                         outputCalls.append((self.verbose_print,(expectedOut,)))
 472                         outputCalls.append((self.verbose_print,('\nactual output:',)))
 473                         outputCalls.append((self.verbose_print,(actual,)))
 474                         # check settings if this should be an expected failure
 475                         expectedfail = dict_match(settings,'','expectedfail')
 476                         # .out files can contain regex but must be prefaced with REGEXP as the first line in the file
 477                         try:
 478                             if expectedOut[0] == 'REGEXP':
 479                                 expectedOut = expectedOut[1:]
 480                                 if len(actual) < len(expectedOut):
 481                                     # pad actual output w/ empty lines
 482                                     for i in range(len(expectedOut)-len(actual)):
 483                                         actual.append('')
 484                                 for i in range(len(actual)):
 485                                     if not re.search(expectedOut[i], actual[i]):
 486                                         raise IndexError
 487                             else:
 488                                 if actual != expectedOut:
 489                                     raise IndexError
 490                             # test passed - check to make sure its not an expected failure
 491                             if expectedfail:
 492                                 outputCalls.append((self.fail,(
 493                                     testName, 'unexpected .out file pass. ' +
 494                                     ' reason: '+expectedfail, self.unpassmsgs)))
 495                                 lunpass += 1
 496                             else:
 497                                 lpass += 1
 498                         except IndexError:
 499                             # test failed
 500                             if expectedfail:
 501                                 outputCalls.append((self.fail,(
 502                                     testName,
 503                                     'expected failure: .out file does not match stdout+stderr. ' +
 504                                     ' reason: '+expectedfail, self.expfailmsgs)))
 505                                 lexpfail += 1
 506                             else:
 507                                 outputCalls.append((self.fail,(testName,
 508                                     '.out file does not match output:\n%s.out file (expected):\n%s\nactual output:\n%s' % (root, expectedOut,actual),
 509                                     self.failmsgs)))
 510                                 lfail += 1
 511                     except IOError:
 512                         outputLines.append((self.js_print,('Error opening %s.out' % root,)))
 513                         lfail += 1
 514                 else:
 515                     for line in f+err:
 516                         outputLines.append(line)
 517                         outputCalls.append((self.verbose_print,('   %s' % line.strip(),)))
 518                         if 'Assertion failed:' in line:
 519                             lassert += 1
 520                             outputCalls.append((self.fail,(testName+extraVmArgs, line, self.assertmsgs)))
 521                         testcase=''
 522                         if len(line)>9:
 523                             testcase=line.strip()
 524                         skipTestDesc = dict_match(settings,testcase,'skip')
 525                         includeTestDesc = dict_match(settings, testcase, 'include')
 526                         if skipTestDesc and not includeTestDesc:
 527                             outputCalls.append((self.js_print,('  skipping "%s" ... reason: %s' % (line.strip(),skipTestDesc),)))
 528                             self.allskips+=1
 529                             continue
 530                         if 'PASSED!' in line:
 531                             res=dict_match(settings,testcase,'expectedfail')
 532                             if res:
 533                                 outputCalls.append((self.fail,(testName, 'unexpected pass: ' + line.strip() + ' reason: '+res, self.unpassmsgs)))
 534                                 lunpass += 1
 535                             else:
 536                                 lpass += 1
 537                         if 'FAILED!' in line:
 538                             res=dict_match(settings,testcase,'expectedfail')
 539                             if res:
 540                                 outputCalls.append((self.fail,(testName, 'expected failure: ' + line.strip() + ' reason: '+res, self.expfailmsgs)))
 541                                 lexpfail += 1
 542                             else:
 543                                 lfail += 1
 544                                 outputCalls.append((self.fail,(testName+extraVmArgs, line, self.failmsgs)))
 545             except:
 546                 print('exception running avm')
 547                 raise
 548
 549             # exitcode check
 550             ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
 551             if ec_lfail or ec_lexpfail:
 552                 lfail += ec_lfail
 553                 lexpfail += ec_lexpfail
 554             elif err:
 555                 # TODO: When needed, add support for expected stderr output - see https://bugzilla.mozilla.org/show_bug.cgi?id=561892
 556                 outputCalls.append((self.fail,(testName, "unexpected stderr expected:'%s' actual:'%s'" % ('',err), self.failmsgs)))
 557                 lfail += 1  # any messages to stderr automatically fail the test
 558             elif lpass == 0 and lfail == 0 and lunpass==0 and lexpfail==0:
 559                 res=dict_match(settings,'*','expectedfail')
 560                 if res:
 561                     outputCalls.append((self.fail,(testName, 'expected failure: FAILED contained no testcase messages reason: %s' % res,self.expfailmsgs)))
 562                     lexpfail += 1
 563                 else:
 564                     # test may not have any output but is still passing if a non-standard exitcode was expected (e.g. VerifyErrors)
 565                     if expectedExitcode != 0 and exitcode==expectedExitcode:
 566                         #outputCalls.append((self.js_print,('  Expected Exit Code: %s Actual Exit Code: %s PASSED' % (expectedExitcode, exitcode),)))
 567                         lpass = 1
 568                     else:
 569                         lfail = 1
 570                         outputmsg=''
 571                         for l in outputLines:
 572                             outputmsg+=l.strip()+'|'
 573                         outputCalls.append((self.fail,(testName, '   FAILED contained no testcase messages - reason: %s' % outputmsg, self.failmsgs)))
 574
 575         self.allfails += lfail
 576         self.allpasses += lpass
 577         self.allexpfails += lexpfail
 578         self.allunpass += lunpass
 579         self.alltimeouts += ltimeout
 580         self.allasserts += lassert
 581         if lfail or lunpass:
 582             outputCalls.append((self.js_print, ('   FAILED passes:%d fails:%d unexpected passes: %d expected failures: %d\n' % (lpass,lfail,lunpass,lexpfail), '', '<br/>')))
 583         else:
 584             outputCalls.append((self.verbose_print, ('   PASSED passes:%d fails:%d unexpected passes: %d expected failures: %d\n' % (lpass,lfail,lunpass,lexpfail), '', '<br/>')))
 585         if self.show_time:
 586             outputCalls.insert(0,(self.js_print,('%s running %s %s %s time %.1f' % (testnum, ast, extraVmArgs, abcargs, time.time()-starttime), '<b>', '</b><br/>')));
 587         else:
 588             outputCalls.insert(0,(self.js_print,('%s running %s %s %s' % (testnum, ast, extraVmArgs, abcargs), '<b>', '</b><br/>')));
 589
 590
 591         return outputCalls
 592
 593     def checkExitCode(self, exitcode, root, testName, f, err, settings, outputCalls):
 594         '''Check the exitcode for a test against any expected non-zero exitcodes
 595             Return the fail and the expected exitcode if non-zero
 596         '''
 597         lfail = 0
 598         lexpfail = 0
 599         expectedExitcode=0
 600         if isfile(root+'.err'):
 601             # .err file holds both the expected (non-catchable) error (usually a VerifyError) and the expected exitcode
 602             expectedErr,expectedExitcode = self.loadExpectedErr(root+'.err')
 603             # check the expectedErr - error is always the last (non-empty) line of output
 604             actualErr = ''
 605             for line in reversed(f):
 606                 # When running in --verifyonly mode, output will be VERIFY FAILED instead of VerifyError
 607                 line = line.replace('VERIFY FAILED', 'VerifyError')
 608                 # make sure we have an error
 609                 if re.search('.*Error:.*', line):
 610                     actualErr = self.getError(line.strip())
 611                     break
 612             if actualErr != expectedErr:
 613                 outputCalls.append((self.fail,(testName, 'unexpected error message. expected: %s actual: %s'
 614                                                % (expectedErr, actualErr), self.failmsgs)))
 615                 lfail += 1
 616         elif isfile(root+".exitcode"):
 617             try:
 618                 expectedExitcode=int(open(root+".exitcode").read())
 619             except:
 620                 print("ERROR: reading exit code file '%s' should contain an integer")
 621         res=dict_match(settings,'exitcode','expectedfail')
 622
 623         if exitcode!=expectedExitcode:
 624             res2=dict_match(settings,'exitcode','skip')
 625             if res2==None and res:
 626                 outputCalls.append((self.js_print,(testName, 'expected failure: exitcode reason: %s'
 627                                                    % res,self.expfailmsgs)))
 628                 lexpfail += 1
 629             elif res2==None:
 630                 outputCalls.append((self.fail,(testName, 'unexpected exit code expected:%d actual:%d Signal Name: %s FAILED!'
 631                                                % (expectedExitcode,exitcode,getSignalName(abs(exitcode))),
 632                                                self.failmsgs)))
 633                 outputmsg=''
 634                 for l in f+err:
 635                     outputmsg+=l.strip()+'|'
 636                 outputCalls.append((self.fail,(testName, 'captured output: %s'
 637                                                % outputmsg,self.failmsgs)))
 638                 lfail+= 1
 639         return lfail, lexpfail, expectedExitcode
 640
 641     def loadExpectedErr(self, file):
 642         try:
 643             f = open(file, 'r')
 644             expectedErr = ''
 645             expectedExitcode = 0
 646             for line in f:
 647                 line = line.strip()
 648                 if line and not line.startswith('#'):
 649                     if line.lower().startswith('exitcode:'):
 650                         try:
 651                             expectedExitcode = int(line.split(':')[1].strip())
 652                         except ValueError:
 653                             pass
 654                     else:
 655                         # check to see if line is a number - if so set expectedExitcode
 656                         try:
 657                             expectedExitcode = int(line)
 658                         except ValueError:
 659                             expectedErr = line
 660             return expectedErr, expectedExitcode
 661         except:
 662             return 'Error reading .err file: %s' % file, 0
 663
 664     def getError(self, line):
 665         # Parse out the error type and #, but strip the description
 666         try:
 667             return ':'.join(line.split(':')[0:2])
 668         except:
 669             return line
 670
 671 runtest = AcceptanceRuntest()