selftest/selftest.py

   1 #!/usr/bin/python -u
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2012 Jelmer Vernooij <jelmer@samba.org>
   4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import atexit
  20 from cStringIO import StringIO
  21 import datetime
  22 import iso8601
  23 import os
  24 import sys
  25 import signal
  26 import shutil
  27 import subprocess
  28 import subunit
  29 import tempfile
  30 import traceback
  31 import warnings
  32
  33 import optparse
  34
  35 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
  36
  37 from selftest import (
  38     socket_wrapper,
  39     subunithelper,
  40     testlist,
  41     )
  42 from selftest.target import (
  43     EnvironmentManager,
  44     NoneTarget,
  45     UnsupportedEnvironment,
  46     )
  47
  48 includes = ()
  49 excludes = ()
  50
  51 def now():
  52     return datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.Utc())
  53
  54 def read_excludes(fn):
  55     excludes.extend(testlist.read_test_regexes(fn))
  56
  57 def read_includes(fn):
  58     includes.extend(testlist.read_test_regexes(fn))
  59
  60 parser = optparse.OptionParser("TEST-REGEXES")
  61 parser.add_option("--target", type="choice", choices=["samba", "samba3", "none"], default="samba", help="Samba version to target")
  62 parser.add_option("--quick", help="run quick overall test", action="store_true", default=False)
  63 parser.add_option("--list", help="list available tests", action="store_true", default=False)
  64 parser.add_option("--socket-wrapper", help="enable socket wrapper", action="store_true", default=False)
  65 parser.add_option("--socket-wrapper-pcap", help="save traffic to pcap directories", type="str")
  66 parser.add_option("--socket-wrapper-keep-pcap", help="keep all pcap files, not just those for tests that failed", action="store_true", default=False)
  67 parser.add_option("--one", help="abort when the first test fails", action="store_true", default=False)
  68 parser.add_option("--exclude", action="callback", help="Add file to exclude files", callback=read_excludes)
  69 parser.add_option("--include", action="callback", help="Add file to include files", callback=read_includes)
  70 parser.add_option("--testenv", help="run a shell in the requested test environment", action="store_true", default=False)
  71 parser.add_option("--resetup-environment", help="Re-setup environment", action="store_true", default=False)
  72 parser.add_option("--binary-mapping", help="Map binaries to use", type=str)
  73 parser.add_option("--load-list", help="Load list of tests to load from a file", type=str)
  74 parser.add_option("--prefix", help="prefix to run tests in", type=str, default="./st")
  75 parser.add_option("--srcdir", type=str, default=".", help="source directory")
  76 parser.add_option("--bindir", type=str, default="./bin", help="binaries directory")
  77 parser.add_option("--testlist", type=str, action="append", help="file to read available tests from")
  78 parser.add_option("--ldap", help="back samba onto specified ldap server", choices=["openldap", "fedora-ds"], type="choice")
  79
  80 opts, args = parser.parse_args()
  81
  82 subunit_ops = subunithelper.SubunitOps(sys.stdout)
  83
  84 def handle_signal(sig, frame):
  85     sys.stderr.write("Exiting early because of signal %s.\n" % sig)
  86     sys.exit(1)
  87
  88 for sig in (signal.SIGINT, signal.SIGQUIT, signal.SIGTERm, signal.SIGPIPE):
  89     signal.signal(sig, handle_signal)
  90
  91 def skip(name):
  92     return testlist.find_in_list(excludes, name)
  93
  94 def setup_pcap(name):
  95     if (not opts.socket_wrapper_pcap or
  96         not os.environ.get("SOCKET_WRAPPER_PCAP_DIR")):
  97         return
  98
  99     fname = "".join([x for x in name if x.isalnum() or x == '-'])
 100
 101     pcap_file = os.path.join(
 102         os.environ["SOCKET_WRAPPER_PCAP_DIR"], "%s.pcap" % fname)
 103
 104     socket_wrapper.setup_pcap(pcap_file)
 105     return pcap_file
 106
 107
 108 def cleanup_pcap(pcap_file, exit_code):
 109     if not opts.socket_wrapper_pcap:
 110         return
 111     if opts.socket_wrapper_keep_pcap:
 112         return
 113     if exitcode == 0:
 114         return
 115     if pcap_file is None:
 116         return
 117
 118     os.unlink(pcap_file)
 119
 120
 121 # expand strings from %ENV
 122 def expand_environment_strings(s):
 123     # we use a reverse sort so we do the longer ones first
 124     for k in sorted(os.environ.keys(), reverse=True):
 125         v = os.environ[k]
 126         s = s.replace("$%s" % k, v)
 127     return s
 128
 129
 130 def run_testsuite(envname, name, cmd):
 131     """Run a single testsuite.
 132
 133     :param envname: Name of the environment to ru nin
 134     :param name: Name of the testsuite
 135     :param cmd: Name of the (fully expanded) command to run
 136     :return: exitcode of the command
 137     """
 138     pcap_file = setup_pcap(name)
 139
 140     subunit_ops.start_testsuite(name)
 141     subunit_ops.progress(None, subunit.PROGRESS_PUSH)
 142     subunit_ops.time(now())
 143     try:
 144         exitcode = subprocess.call(cmd, shell=True)
 145     except Exception, e:
 146         subunit_ops.time(now())
 147         subunit_ops.progress(None, subunit.PROGRESS_POP)
 148         subunit_ops.end_testsuite(name, "error", "Unable to run %r: %s" % (cmd, e))
 149         sys.exit(1)
 150
 151     subunit_ops.time(now())
 152     subunit_ops.progress(None, subunit.PROGRESS_POP)
 153
 154     envlog = env_manager.getlog_env(envname)
 155     if envlog != "":
 156         sys.stdout.write("envlog: %s\n" % envlog)
 157
 158     sys.stdout.write("command: %s\n" % cmd)
 159     sys.stdout.write("expanded command: %s\n" % expand_environment_strings(cmd))
 160
 161     if exitcode == 0:
 162         subunit_ops.end_testsuite(name, "success")
 163     else:
 164         subunit_ops.end_testsuite(name, "failure", "Exit code was %d" % exitcode)
 165
 166     cleanup_pcap(pcap_file, exitcode)
 167
 168     if not opts.socket_wrapper_keep_pcap and pcap_file is not None:
 169         sys.stdout.write("PCAP FILE: %s\n" % pcap_file)
 170
 171     if exitcode != 0 and opts.one:
 172         sys.exit(1)
 173
 174     return exitcode
 175
 176 if opts.list and opts.testenv:
 177     sys.stderr.write("--list and --testenv are mutually exclusive\n")
 178     sys.exit(1)
 179
 180 tests = args
 181
 182 # quick hack to disable rpc validation when using valgrind - it is way too slow
 183 if not os.environ.get("VALGRIND"):
 184     os.environ["VALIDATE"] = "validate"
 185     os.environ["MALLOC_CHECK_"] = "2"
 186
 187 # make all our python scripts unbuffered
 188 os.environ["PYTHONUNBUFFERED"] = "1"
 189
 190 bindir_abs = os.path.abspath(opts.bindir)
 191
 192 # Backwards compatibility:
 193 if os.environ.get("TEST_LDAP") == "yes":
 194     if os.environ.get("FEDORA_DS_ROOT"):
 195         ldap = "fedora-ds"
 196     else:
 197         ldap = "openldap"
 198
 199 torture_maxtime = int(os.getenv("TORTURE_MAXTIME", "1200"))
 200 if opts.ldap:
 201     # LDAP is slow
 202     torture_maxtime *= 2
 203
 204 prefix = os.path.normpath(opts.prefix)
 205
 206 if prefix == "":
 207     raise Exception("using an empty prefix isn't allowed")
 208
 209 # Ensure we have the test prefix around.
 210 #
 211 # We need restrictive permissions on this as some subdirectories in this tree
 212 # will have wider permissions (ie 0777) and this would allow other users on the
 213 # host to subvert the test process.
 214 if not os.path.isdir(prefix):
 215     os.mkdir(prefix, 0700)
 216 else:
 217     os.chmod(prefix, 0700)
 218
 219 prefix_abs = os.path.abspath(prefix)
 220 tmpdir_abs = os.path.abspath(os.path.join(prefix_abs, "tmp"))
 221 if not os.path.isdir(tmpdir_abs):
 222     os.mkdir(tmpdir_abs, 0777)
 223
 224 srcdir_abs = os.path.abspath(opts.srcdir)
 225
 226 if prefix_abs == "":
 227     raise Exception("using an empty absolute prefix isn't allowed")
 228 if prefix_abs == "/":
 229     raise Exception("using '/' as absolute prefix isn't allowed")
 230
 231 os.environ["PREFIX"] = prefix
 232 os.environ["KRB5CCNAME"] = os.path.join(prefix, "krb5ticket")
 233 os.environ["PREFIX_ABS"] = prefix_abs
 234 os.environ["SRCDIR"] = opts.srcdir
 235 os.environ["SRCDIR_ABS"] = srcdir_abs
 236 os.environ["BINDIR"] = bindir_abs
 237
 238 tls_enabled = not opts.quick
 239 if tls_enabled:
 240     os.environ["TLS_ENABLED"] = "yes"
 241 else:
 242     os.environ["TLS_ENABLED"] = "no"
 243
 244 def prefix_pathvar(name, newpath):
 245     if name in os.environ:
 246         os.environ[name] = "%s:%s" % (newpath, os.environ[name])
 247     else:
 248         os.environ[name] = newpath
 249 prefix_pathvar("PKG_CONFIG_PATH", os.path.join(bindir_abs, "pkgconfig"))
 250 prefix_pathvar("PYTHONPATH", os.path.join(bindir_abs, "python"))
 251
 252 if opts.socket_wrapper_keep_pcap:
 253     # Socket wrapper keep pcap implies socket wrapper pcap
 254     opts.socket_wrapper_pcap = True
 255
 256 if opts.socket_wrapper_pcap:
 257     # Socket wrapper pcap implies socket wrapper
 258     opts.socket_wrapper = True
 259
 260 if opts.socket_wrapper:
 261     socket_wrapper_dir = socket_wrapper.setup_dir(os.path.join(prefix_abs, "w"), opts.socket_wrapper_pcap)
 262     sys.stdout.write("SOCKET_WRAPPER_DIR=%s\n" % socket_wrapper_dir)
 263 elif not opts.list:
 264     if os.getuid() != 0:
 265         warnings.warn("not using socket wrapper, but also not running as root. Will not be able to listen on proper ports")
 266
 267 testenv_default = "none"
 268
 269 if opts.binary_mapping:
 270     binary_mapping = dict([l.split(":") for l in opts.binary_mapping.split(",")])
 271     os.environ["BINARY_MAPPING"] = opts.binary_mapping
 272 else:
 273     binary_mapping = {}
 274     os.environ["BINARY_MAPPING"] = ""
 275
 276 # After this many seconds, the server will self-terminate.  All tests
 277 # must terminate in this time, and testenv will only stay alive this
 278 # long
 279
 280 server_maxtime = 7500
 281 if os.environ.get("SMBD_MAXTIME", ""):
 282     server_maxtime = int(os.environ["SMBD_MAXTIME"])
 283
 284
 285 def has_socket_wrapper(bindir):
 286     """Check if Samba has been built with socket wrapper support.
 287     """
 288     f = StringIO()
 289     subprocess.check_call([os.path.join(bindir, "smbd"), "-b"], stdout=f)
 290     for l in f.readlines():
 291         if "SOCKET_WRAPPER" in l:
 292             return True
 293     return False
 294
 295
 296 if not opts.list:
 297     if opts.target == "samba":
 298         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 299             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 300             sys.exit(1)
 301         testenv_default = "dc"
 302         from selftest.target.samba import Samba
 303         target = Samba(opts.bindir, binary_mapping, ldap, opts.srcdir, server_maxtime)
 304     elif opts.target == "samba3":
 305         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 306             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 307             sys.exit(1)
 308         testenv_default = "member"
 309         from selftest.target.samba3 import Samba3
 310         target = Samba3(opts.bindir, binary_mapping, srcdir_abs, server_maxtime)
 311     elif opts.target == "none":
 312         testenv_default = "none"
 313         target = NoneTarget()
 314
 315     env_manager = EnvironmentManager(target)
 316     atexit.register(env_manager.teardown_all)
 317
 318 interfaces = ",".join([
 319     "127.0.0.11/8",
 320     "127.0.0.12/8",
 321     "127.0.0.13/8",
 322     "127.0.0.14/8",
 323     "127.0.0.15/8",
 324     "127.0.0.16/8"])
 325
 326 clientdir = os.path.join(prefix_abs, "client")
 327
 328 conffile = os.path.join(clientdir, "client.conf")
 329 os.environ["SMB_CONF_PATH"] = conffile
 330
 331 def write_clientconf(conffile, clientdir, vars):
 332     if not os.path.isdir(clientdir):
 333         os.mkdir(clientdir, 0777)
 334
 335     for n in ["private", "lockdir", "statedir", "cachedir"]:
 336         p = os.path.join(clientdir, n)
 337         if os.path.isdir(p):
 338             shutil.rmtree(p)
 339         os.mkdir(p, 0777)
 340
 341     # this is ugly, but the ncalrpcdir needs exactly 0755
 342     # otherwise tests fail.
 343     mask = os.umask(0022)
 344
 345     for n in ["ncalrpcdir", "ncalrpcdir/np"]:
 346         p = os.path.join(clientdir, n)
 347         if os.path.isdir(p):
 348             shutil.rmtree(p)
 349         os.mkdir(p, 0777)
 350     os.umask(mask)
 351
 352     settings = {
 353         "netbios name": "client",
 354         "private dir": os.path.join(clientdir, "private"),
 355         "lock dir": os.path.join(clientdir, "lockdir"),
 356         "state directory": os.path.join(clientdir, "statedir"),
 357         "cache directory": os.path.join(clientdir, "cachedir"),
 358         "ncalrpc dir": os.path.join(clientdir, "ncalrpcdir"),
 359         "name resolve order": "file bcast",
 360         "panic action": os.path.join(os.path.dirname(__file__), "gdb_backtrace \%d"),
 361         "max xmit": "32K",
 362         "notify:inotify": "false",
 363         "ldb:nosync": "true",
 364         "system:anonymous": "true",
 365         "client lanman auth": "Yes",
 366         "log level": "1",
 367         "torture:basedir": clientdir,
 368         # We don't want to pass our self-tests if the PAC code is wrong
 369         "gensec:require_pac": "true",
 370         "resolv:host file": os.path.join(prefix_abs, "dns_host_file"),
 371         # We don't want to run 'speed' tests for very long
 372         "torture:timelimit": "1",
 373         }
 374
 375     if "DOMAIN" in vars:
 376         settings["workgroup"] = vars["DOMAIN"]
 377     if "REALM" in vars:
 378         settings["realm"] = vars["REALM"]
 379     if opts.socket_wrapper:
 380         settings["interfaces"] = interfaces
 381
 382     f = open(conffile, 'w')
 383     try:
 384         f.write("[global]\n")
 385         for item in settings.iteritems():
 386             f.write("\t%s = %s\n" % item)
 387     finally:
 388         f.close()
 389
 390 todo = []
 391
 392 if not opts.testlist:
 393     sys.stderr.write("No testlists specified\n")
 394     sys.exit(1)
 395
 396 os.environ["SELFTEST_PREFIX"] = prefix_abs
 397 os.environ["SELFTEST_TMPDIR"] = tmpdir_abs
 398 os.environ["TEST_DATA_PREFIX"] = tmpdir_abs
 399 if opts.socket_wrapper:
 400     os.environ["SELFTEST_INTERFACES"] = interfaces
 401 else:
 402     os.environ["SELFTEST_INTERFACES"] = ""
 403 if opts.quick:
 404     os.environ["SELFTEST_QUICK"] = "1"
 405 else:
 406     os.environ["SELFTEST_QUICK"] = ""
 407 os.environ["SELFTEST_MAXTIME"] = str(torture_maxtime)
 408
 409
 410 def open_file_or_pipe(path, mode):
 411     if path.endswith("|"):
 412         return os.popen(path[:-1], mode)
 413     return open(path, mode)
 414
 415 available = []
 416 for fn in opts.testlist:
 417     inf = open_file_or_pipe(fn, 'r')
 418     try:
 419         for testsuite in testlist.read_testlist(inf, sys.stdout):
 420             if not testlist.should_run_test(tests, testsuite):
 421                 continue
 422             name = testsuite[0]
 423             if includes is not None and testlist.find_in_list(includes, name) is not None:
 424                 continue
 425             available.append(testsuite)
 426     finally:
 427         inf.close()
 428
 429 if opts.load_list:
 430     restricted_mgr = testlist.RestrictedTestManager.from_path(opts.load_list)
 431 else:
 432     restricted_mgr = None
 433
 434
 435 for testsuite in available:
 436     name = testsuite[0]
 437     skipreason = skip(name)
 438     if restricted_mgr is not None:
 439         match = restricted_mgr.should_run_testsuite(name)
 440         if match == []:
 441             continue
 442     else:
 443         match = None
 444     if skipreason is not None:
 445         if not opts.list:
 446             subunit_ops.skip_testsuite(name, skipreason)
 447     else:
 448         todo.append(testsuite + (match,))
 449
 450 if restricted_mgr is not None:
 451     for name in restricted_mgr.iter_unused():
 452         sys.stdout.write("No test or testsuite found matching %s\n" % name)
 453 if todo == []:
 454     sys.stderr.write("No tests to run\n")
 455     sys.exit(1)
 456
 457 suitestotal = len(todo)
 458
 459 if not opts.list:
 460     subunit_ops.progress(suitestotal, subunit.PROGRESS_SET)
 461     subunit_ops.time(now())
 462
 463 exported_envvars = [
 464     # domain stuff
 465     "DOMAIN",
 466     "REALM",
 467
 468     # domain controller stuff
 469     "DC_SERVER",
 470     "DC_SERVER_IP",
 471     "DC_NETBIOSNAME",
 472     "DC_NETBIOSALIAS",
 473
 474     # domain member
 475     "MEMBER_SERVER",
 476     "MEMBER_SERVER_IP",
 477     "MEMBER_NETBIOSNAME",
 478     "MEMBER_NETBIOSALIAS",
 479
 480     # rpc proxy controller stuff
 481     "RPC_PROXY_SERVER",
 482     "RPC_PROXY_SERVER_IP",
 483     "RPC_PROXY_NETBIOSNAME",
 484     "RPC_PROXY_NETBIOSALIAS",
 485
 486     # domain controller stuff for Vampired DC
 487     "VAMPIRE_DC_SERVER",
 488     "VAMPIRE_DC_SERVER_IP",
 489     "VAMPIRE_DC_NETBIOSNAME",
 490     "VAMPIRE_DC_NETBIOSALIAS",
 491
 492     # server stuff
 493     "SERVER",
 494     "SERVER_IP",
 495     "NETBIOSNAME",
 496     "NETBIOSALIAS",
 497
 498     # user stuff
 499     "USERNAME",
 500     "USERID",
 501     "PASSWORD",
 502     "DC_USERNAME",
 503     "DC_PASSWORD",
 504
 505     # misc stuff
 506     "KRB5_CONFIG",
 507     "WINBINDD_SOCKET_DIR",
 508     "WINBINDD_PRIV_PIPE_DIR",
 509     "NMBD_SOCKET_DIR",
 510     "LOCAL_PATH"
 511 ]
 512
 513 def exported_envvars_str(testenv_vars):
 514     out = ""
 515
 516     for n in exported_envvars:
 517         if not n in testenv_vars:
 518             continue
 519         out += "%s=%s\n" % (n, testenv_vars[n])
 520
 521     return out
 522
 523
 524 def switch_env(name, prefix):
 525     if ":" in name:
 526         (envname, option) = name.split(":", 1)
 527     else:
 528         envname = name
 529         option = "client"
 530
 531     env = env_manager.setup_env(envname, prefix)
 532
 533     testenv_vars = env.get_vars()
 534
 535     if option == "local":
 536         socket_wrapper.set_default_iface(testenv_vars["SOCKET_WRAPPER_DEFAULT_IFACE"])
 537         os.environ["SMB_CONF_PATH"] = testenv_vars["SERVERCONFFILE"]
 538     elif option == "client":
 539         socket_wrapper.set_default_iface(11)
 540         write_clientconf(conffile, clientdir, testenv_vars)
 541         os.environ["SMB_CONF_PATH"] = conffile
 542     else:
 543         raise Exception("Unknown option[%s] for envname[%s]" % (option,
 544             envname))
 545
 546     for name in exported_envvars:
 547         if name in testenv_vars:
 548             os.environ[name] = testenv_vars[name]
 549         elif name in os.environ:
 550             del os.environ[name]
 551
 552     return testenv_vars
 553
 554 # This 'global' file needs to be empty when we start
 555 dns_host_file_path = os.path.join(prefix_abs, "dns_host_file")
 556 if os.path.exists(dns_host_file_path):
 557     os.unlink(dns_host_file_path)
 558
 559 if opts.testenv:
 560     testenv_name = os.environ.get("SELFTEST_TESTENV", testenv_default)
 561
 562     testenv_vars = switch_env(testenv_name, prefix)
 563
 564     os.environ["PIDDIR"] = testenv_vars["PIDDIR"]
 565     os.environ["ENVNAME"] = testenv_name
 566
 567     envvarstr = exported_envvars_str(testenv_vars)
 568
 569     term = os.environ.get("TERMINAL", "xterm -e")
 570     cmd = """'echo -e "
 571 Welcome to the Samba4 Test environment '%(testenv_name)'
 572
 573 This matches the client environment used in make test
 574 server is pid `cat \$PIDDIR/samba.pid`
 575
 576 Some useful environment variables:
 577 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 578 SMB_CONF_PATH=\$SMB_CONF_PATH
 579
 580 $envvarstr
 581 \" && LD_LIBRARY_PATH=%(LD_LIBRARY_PATH)s $(SHELL)'""" % {
 582         "testenv_name": testenv_name,
 583         "LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"]}
 584     subprocess.call(term + ' ' + cmd, shell=True)
 585     env_manager.teardown_env(testenv_name)
 586 elif opts.list:
 587     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 588         if not "$LISTOPT" in cmd:
 589             warnings.warn("Unable to list tests in %s" % name)
 590             continue
 591
 592         cmd = cmd.replace("$LISTOPT", "--list")
 593
 594         exitcode = subprocess.call(cmd, shell=True)
 595
 596         if exitcode != 0:
 597             sys.stderr.write("%s exited with exit code %s\n" % (cmd, exitcode))
 598             sys.exit(1)
 599 else:
 600     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 601         try:
 602             envvars = switch_env(envname, prefix)
 603         except UnsupportedEnvironment:
 604             subunit_ops.start_testsuite(name)
 605             subunit_ops.end_testsuite(name, "skip",
 606                 "environment %s is unknown in this test backend - skipping" % envname)
 607             continue
 608         except Exception, e:
 609             subunit_ops.start_testsuite(name)
 610             traceback.print_exc()
 611             subunit_ops.end_testsuite(name, "error",
 612                 "unable to set up environment %s: %s" % (envname, e))
 613             continue
 614
 615         # Generate a file with the individual tests to run, if the
 616         # test runner for this test suite supports it.
 617         if subtests is not None:
 618             if supports_loadfile:
 619                 (fd, listid_file) = tempfile.mkstemp()
 620                 # FIXME: Remove tempfile afterwards
 621                 f = os.fdopen(fd)
 622                 try:
 623                     for test in subtests:
 624                         f.write(test+"\n")
 625                 finally:
 626                     f.close()
 627                 cmd = cmd.replace("$LOADLIST", "--load-list=%s" % listid_file)
 628             elif supports_idlist:
 629                 cmd += " ".join(subtests)
 630
 631         run_testsuite(envname, name, cmd)
 632
 633         if opts.resetup_environment:
 634             env_manager.teardown_env(envname)
 635
 636 sys.stdout.write("\n")
 637
 638 if not opts.list:
 639     env_manager.teardown_all()
 640
 641 # if there were any valgrind failures, show them
 642 for fn in os.listdir(prefix):
 643     if fn.startswith("valgrind.log"):
 644         sys.stdout.write("VALGRIND FAILURE\n")
 645         f = open(os.path.join(prefix, fn), 'r')
 646         try:
 647             sys.stdout.write(f.read())
 648         finally:
 649             f.close()
 650
 651 sys.exit(0)