selftest/selftest.py

   1 #!/usr/bin/python -u
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2012 Jelmer Vernooij <jelmer@samba.org>
   4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import atexit
  20 from cStringIO import StringIO
  21 import os
  22 import sys
  23 import signal
  24 import subprocess
  25 import subunit
  26 import traceback
  27 import warnings
  28
  29 import optparse
  30
  31 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
  32
  33 from selftest import (
  34     socket_wrapper,
  35     subunithelper,
  36     testlist,
  37     )
  38 from selftest.client import write_clientconf
  39 from selftest.run import (
  40     expand_command_list,
  41     expand_command_run,
  42     exported_envvars_str,
  43     now,
  44     run_testsuite_command,
  45     )
  46 from selftest.target import (
  47     EnvironmentManager,
  48     NoneTarget,
  49     UnsupportedEnvironment,
  50     )
  51
  52 includes = ()
  53 excludes = ()
  54
  55 def read_excludes(fn):
  56     excludes.extend(testlist.read_test_regexes(fn))
  57
  58 def read_includes(fn):
  59     includes.extend(testlist.read_test_regexes(fn))
  60
  61 parser = optparse.OptionParser("TEST-REGEXES")
  62 parser.add_option("--target", type="choice", choices=["samba", "samba3", "none"], default="samba", help="Samba version to target")
  63 parser.add_option("--quick", help="run quick overall test", action="store_true", default=False)
  64 parser.add_option("--list", help="list available tests", action="store_true", default=False)
  65 parser.add_option("--socket-wrapper", help="enable socket wrapper", action="store_true", default=False)
  66 parser.add_option("--socket-wrapper-pcap", help="save traffic to pcap directories", type="str")
  67 parser.add_option("--socket-wrapper-keep-pcap", help="keep all pcap files, not just those for tests that failed", action="store_true", default=False)
  68 parser.add_option("--one", help="abort when the first test fails", action="store_true", default=False)
  69 parser.add_option("--exclude", action="callback", help="Add file to exclude files", callback=read_excludes)
  70 parser.add_option("--include", action="callback", help="Add file to include files", callback=read_includes)
  71 parser.add_option("--testenv", help="run a shell in the requested test environment", action="store_true", default=False)
  72 parser.add_option("--resetup-environment", help="Re-setup environment", action="store_true", default=False)
  73 parser.add_option("--binary-mapping", help="Map binaries to use", type=str)
  74 parser.add_option("--load-list", help="Load list of tests to load from a file", type=str)
  75 parser.add_option("--prefix", help="prefix to run tests in", type=str, default="./st")
  76 parser.add_option("--srcdir", type=str, default=".", help="source directory")
  77 parser.add_option("--bindir", type=str, default="./bin", help="binaries directory")
  78 parser.add_option("--testlist", type=str, action="append", help="file to read available tests from")
  79 parser.add_option("--ldap", help="back samba onto specified ldap server", choices=["openldap", "fedora-ds"], type="choice")
  80
  81 opts, args = parser.parse_args()
  82
  83 subunit_ops = subunithelper.SubunitOps(sys.stdout)
  84
  85 def handle_signal(sig, frame):
  86     sys.stderr.write("Exiting early because of signal %s.\n" % sig)
  87     sys.exit(1)
  88
  89 for sig in (signal.SIGINT, signal.SIGQUIT, signal.SIGTERm, signal.SIGPIPE):
  90     signal.signal(sig, handle_signal)
  91
  92 def skip(name):
  93     return testlist.find_in_list(excludes, name)
  94
  95 def setup_pcap(name):
  96     if (not opts.socket_wrapper_pcap or
  97         not os.environ.get("SOCKET_WRAPPER_PCAP_DIR")):
  98         return
  99
 100     fname = "".join([x for x in name if x.isalnum() or x == '-'])
 101
 102     pcap_file = os.path.join(
 103         os.environ["SOCKET_WRAPPER_PCAP_DIR"], "%s.pcap" % fname)
 104
 105     socket_wrapper.setup_pcap(pcap_file)
 106     return pcap_file
 107
 108
 109 def cleanup_pcap(pcap_file, exit_code):
 110     if not opts.socket_wrapper_pcap:
 111         return
 112     if opts.socket_wrapper_keep_pcap:
 113         return
 114     if exitcode == 0:
 115         return
 116     if pcap_file is None:
 117         return
 118
 119     os.unlink(pcap_file)
 120
 121
 122 def run_testsuite(name, cmd, subunit_ops, env=None):
 123     """Run a single testsuite.
 124
 125     :param env: Environment to run in
 126     :param name: Name of the testsuite
 127     :param cmd: Name of the (fully expanded) command to run
 128     :return: exitcode of the command
 129     """
 130     pcap_file = setup_pcap(name)
 131
 132     exitcode = run_testsuite_command(name, cmd, subunit_ops, env)
 133     if exitcode is None:
 134         sys.exit(1)
 135
 136     cleanup_pcap(pcap_file, exitcode)
 137
 138     if not opts.socket_wrapper_keep_pcap and pcap_file is not None:
 139         sys.stdout.write("PCAP FILE: %s\n" % pcap_file)
 140
 141     if exitcode != 0 and opts.one:
 142         sys.exit(1)
 143
 144     return exitcode
 145
 146
 147 if opts.list and opts.testenv:
 148     sys.stderr.write("--list and --testenv are mutually exclusive\n")
 149     sys.exit(1)
 150
 151 tests = args
 152
 153 # quick hack to disable rpc validation when using valgrind - it is way too slow
 154 if not os.environ.get("VALGRIND"):
 155     os.environ["VALIDATE"] = "validate"
 156     os.environ["MALLOC_CHECK_"] = "3"
 157
 158 # make all our python scripts unbuffered
 159 os.environ["PYTHONUNBUFFERED"] = "1"
 160
 161 bindir_abs = os.path.abspath(opts.bindir)
 162
 163 # Backwards compatibility:
 164 if os.environ.get("TEST_LDAP") == "yes":
 165     if os.environ.get("FEDORA_DS_ROOT"):
 166         ldap = "fedora-ds"
 167     else:
 168         ldap = "openldap"
 169
 170 torture_maxtime = int(os.getenv("TORTURE_MAXTIME", "1200"))
 171 if opts.ldap:
 172     # LDAP is slow
 173     torture_maxtime *= 2
 174
 175 prefix = os.path.normpath(opts.prefix)
 176
 177 # Ensure we have the test prefix around.
 178 #
 179 # We need restrictive permissions on this as some subdirectories in this tree
 180 # will have wider permissions (ie 0777) and this would allow other users on the
 181 # host to subvert the test process.
 182 if not os.path.isdir(prefix):
 183     os.mkdir(prefix, 0700)
 184 else:
 185     os.chmod(prefix, 0700)
 186
 187 prefix_abs = os.path.abspath(prefix)
 188 tmpdir_abs = os.path.abspath(os.path.join(prefix_abs, "tmp"))
 189 if not os.path.isdir(tmpdir_abs):
 190     os.mkdir(tmpdir_abs, 0777)
 191
 192 srcdir_abs = os.path.abspath(opts.srcdir)
 193
 194 if prefix_abs == "/":
 195     raise Exception("using '/' as absolute prefix is a bad idea")
 196
 197 os.environ["PREFIX"] = prefix
 198 os.environ["KRB5CCNAME"] = os.path.join(prefix, "krb5ticket")
 199 os.environ["PREFIX_ABS"] = prefix_abs
 200 os.environ["SRCDIR"] = opts.srcdir
 201 os.environ["SRCDIR_ABS"] = srcdir_abs
 202 os.environ["BINDIR"] = bindir_abs
 203
 204 tls_enabled = not opts.quick
 205 if tls_enabled:
 206     os.environ["TLS_ENABLED"] = "yes"
 207 else:
 208     os.environ["TLS_ENABLED"] = "no"
 209
 210 def prefix_pathvar(name, newpath):
 211     if name in os.environ:
 212         os.environ[name] = "%s:%s" % (newpath, os.environ[name])
 213     else:
 214         os.environ[name] = newpath
 215 prefix_pathvar("PKG_CONFIG_PATH", os.path.join(bindir_abs, "pkgconfig"))
 216 prefix_pathvar("PYTHONPATH", os.path.join(bindir_abs, "python"))
 217
 218 if opts.socket_wrapper_keep_pcap:
 219     # Socket wrapper keep pcap implies socket wrapper pcap
 220     opts.socket_wrapper_pcap = True
 221
 222 if opts.socket_wrapper_pcap:
 223     # Socket wrapper pcap implies socket wrapper
 224     opts.socket_wrapper = True
 225
 226 if opts.socket_wrapper:
 227     socket_wrapper_dir = socket_wrapper.setup_dir(os.path.join(prefix_abs, "w"), opts.socket_wrapper_pcap)
 228     sys.stdout.write("SOCKET_WRAPPER_DIR=%s\n" % socket_wrapper_dir)
 229 elif not opts.list:
 230     if os.getuid() != 0:
 231         warnings.warn("not using socket wrapper, but also not running as root. Will not be able to listen on proper ports")
 232
 233 testenv_default = "none"
 234
 235 if opts.binary_mapping:
 236     binary_mapping = dict([l.split(":") for l in opts.binary_mapping.split(",")])
 237     os.environ["BINARY_MAPPING"] = opts.binary_mapping
 238 else:
 239     binary_mapping = {}
 240     os.environ["BINARY_MAPPING"] = ""
 241
 242 # After this many seconds, the server will self-terminate.  All tests
 243 # must terminate in this time, and testenv will only stay alive this
 244 # long
 245
 246 if os.environ.get("SMBD_MAXTIME", ""):
 247     server_maxtime = int(os.environ["SMBD_MAXTIME"])
 248 else:
 249     server_maxtime = 7500
 250
 251
 252 def has_socket_wrapper(bindir):
 253     """Check if Samba has been built with socket wrapper support.
 254     """
 255     f = StringIO()
 256     subprocess.check_call([os.path.join(bindir, "smbd"), "-b"], stdout=f)
 257     for l in f.readlines():
 258         if "SOCKET_WRAPPER" in l:
 259             return True
 260     return False
 261
 262
 263 if not opts.list:
 264     if opts.target == "samba":
 265         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 266             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 267             sys.exit(1)
 268         testenv_default = "dc"
 269         from selftest.target.samba import Samba
 270         target = Samba(opts.bindir, binary_mapping, ldap, opts.srcdir, server_maxtime)
 271     elif opts.target == "samba3":
 272         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 273             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 274             sys.exit(1)
 275         testenv_default = "member"
 276         from selftest.target.samba3 import Samba3
 277         target = Samba3(opts.bindir, binary_mapping, srcdir_abs, server_maxtime)
 278     elif opts.target == "none":
 279         testenv_default = "none"
 280         target = NoneTarget()
 281
 282     env_manager = EnvironmentManager(target)
 283     atexit.register(env_manager.teardown_all)
 284
 285 interfaces = ",".join([
 286     "127.0.0.11/8",
 287     "127.0.0.12/8",
 288     "127.0.0.13/8",
 289     "127.0.0.14/8",
 290     "127.0.0.15/8",
 291     "127.0.0.16/8"])
 292
 293 clientdir = os.path.join(prefix_abs, "client")
 294
 295 conffile = os.path.join(clientdir, "client.conf")
 296 os.environ["SMB_CONF_PATH"] = conffile
 297
 298 todo = []
 299
 300 if not opts.testlist:
 301     sys.stderr.write("No testlists specified\n")
 302     sys.exit(1)
 303
 304 os.environ["SELFTEST_PREFIX"] = prefix_abs
 305 os.environ["SELFTEST_TMPDIR"] = tmpdir_abs
 306 os.environ["TEST_DATA_PREFIX"] = tmpdir_abs
 307 if opts.socket_wrapper:
 308     os.environ["SELFTEST_INTERFACES"] = interfaces
 309 else:
 310     os.environ["SELFTEST_INTERFACES"] = ""
 311 if opts.quick:
 312     os.environ["SELFTEST_QUICK"] = "1"
 313 else:
 314     os.environ["SELFTEST_QUICK"] = ""
 315 os.environ["SELFTEST_MAXTIME"] = str(torture_maxtime)
 316
 317
 318 available = []
 319 for fn in opts.testlist:
 320     for testsuite in testlist.read_testlist_file(fn):
 321         if not testlist.should_run_test(tests, testsuite):
 322             continue
 323         name = testsuite[0]
 324         if (includes is not None and
 325             testlist.find_in_list(includes, name) is not None):
 326             continue
 327         available.append(testsuite)
 328
 329 if opts.load_list:
 330     restricted_mgr = testlist.RestrictedTestManager.from_path(opts.load_list)
 331 else:
 332     restricted_mgr = None
 333
 334 for testsuite in available:
 335     name = testsuite[0]
 336     skipreason = skip(name)
 337     if restricted_mgr is not None:
 338         match = restricted_mgr.should_run_testsuite(name)
 339         if match == []:
 340             continue
 341     else:
 342         match = None
 343     if skipreason is not None:
 344         if not opts.list:
 345             subunit_ops.skip_testsuite(name, skipreason)
 346     else:
 347         todo.append(testsuite + (match,))
 348
 349 if restricted_mgr is not None:
 350     for name in restricted_mgr.iter_unused():
 351         sys.stdout.write("No test or testsuite found matching %s\n" % name)
 352 if todo == []:
 353     sys.stderr.write("No tests to run\n")
 354     sys.exit(1)
 355
 356 suitestotal = len(todo)
 357
 358 if not opts.list:
 359     subunit_ops.progress(suitestotal, subunit.PROGRESS_SET)
 360     subunit_ops.time(now())
 361
 362 exported_envvars = [
 363     # domain stuff
 364     "DOMAIN",
 365     "REALM",
 366
 367     # domain controller stuff
 368     "DC_SERVER",
 369     "DC_SERVER_IP",
 370     "DC_NETBIOSNAME",
 371     "DC_NETBIOSALIAS",
 372
 373     # domain member
 374     "MEMBER_SERVER",
 375     "MEMBER_SERVER_IP",
 376     "MEMBER_NETBIOSNAME",
 377     "MEMBER_NETBIOSALIAS",
 378
 379     # rpc proxy controller stuff
 380     "RPC_PROXY_SERVER",
 381     "RPC_PROXY_SERVER_IP",
 382     "RPC_PROXY_NETBIOSNAME",
 383     "RPC_PROXY_NETBIOSALIAS",
 384
 385     # domain controller stuff for Vampired DC
 386     "VAMPIRE_DC_SERVER",
 387     "VAMPIRE_DC_SERVER_IP",
 388     "VAMPIRE_DC_NETBIOSNAME",
 389     "VAMPIRE_DC_NETBIOSALIAS",
 390
 391     # domain controller stuff for Vampired DC
 392     "PROMOTED_DC_SERVER",
 393     "PROMOTED_DC_SERVER_IP",
 394     "PROMOTED_DC_NETBIOSNAME",
 395     "PROMOTED_DC_NETBIOSALIAS",
 396
 397     # server stuff
 398     "SERVER",
 399     "SERVER_IP",
 400     "NETBIOSNAME",
 401     "NETBIOSALIAS",
 402
 403     # user stuff
 404     "USERNAME",
 405     "USERID",
 406     "PASSWORD",
 407     "DC_USERNAME",
 408     "DC_PASSWORD",
 409
 410     # misc stuff
 411     "KRB5_CONFIG",
 412     "WINBINDD_SOCKET_DIR",
 413     "WINBINDD_PRIV_PIPE_DIR",
 414     "NMBD_SOCKET_DIR",
 415     "LOCAL_PATH"
 416 ]
 417
 418 def switch_env(name, prefix):
 419     if ":" in name:
 420         (envname, option) = name.split(":", 1)
 421     else:
 422         envname = name
 423         option = "client"
 424
 425     env = env_manager.setup_env(envname, prefix)
 426
 427     testenv_vars = env.get_vars()
 428
 429     if option == "local":
 430         socket_wrapper.set_default_iface(testenv_vars["SOCKET_WRAPPER_DEFAULT_IFACE"])
 431         os.environ["SMB_CONF_PATH"] = testenv_vars["SERVERCONFFILE"]
 432     elif option == "client":
 433         socket_wrapper.set_default_iface(11)
 434         write_clientconf(conffile, clientdir, testenv_vars)
 435         os.environ["SMB_CONF_PATH"] = conffile
 436     else:
 437         raise Exception("Unknown option[%s] for envname[%s]" % (option,
 438             envname))
 439
 440     for name in exported_envvars:
 441         if name in testenv_vars:
 442             os.environ[name] = testenv_vars[name]
 443         elif name in os.environ:
 444             del os.environ[name]
 445
 446     return env
 447
 448 # This 'global' file needs to be empty when we start
 449 dns_host_file_path = os.path.join(prefix_abs, "dns_host_file")
 450 if os.path.exists(dns_host_file_path):
 451     os.unlink(dns_host_file_path)
 452
 453 if opts.testenv:
 454     testenv_name = os.environ.get("SELFTEST_TESTENV", testenv_default)
 455
 456     env = switch_env(testenv_name, prefix)
 457     testenv_vars = env.get_vars()
 458
 459     os.environ["PIDDIR"] = testenv_vars["PIDDIR"]
 460     os.environ["ENVNAME"] = testenv_name
 461
 462     envvarstr = exported_envvars_str(testenv_vars, exported_envvars)
 463
 464     term = os.environ.get("TERMINAL", "xterm -e")
 465     cmd = """'echo -e "
 466 Welcome to the Samba4 Test environment '%(testenv_name)'
 467
 468 This matches the client environment used in make test
 469 server is pid `cat \$PIDDIR/samba.pid`
 470
 471 Some useful environment variables:
 472 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 473 SMB_CONF_PATH=\$SMB_CONF_PATH
 474
 475 $envvarstr
 476 \" && LD_LIBRARY_PATH=%(LD_LIBRARY_PATH)s $(SHELL)'""" % {
 477         "testenv_name": testenv_name,
 478         "LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"]}
 479     subprocess.call(term + ' ' + cmd, shell=True)
 480     env_manager.teardown_env(testenv_name)
 481 elif opts.list:
 482     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 483         cmd = expand_command_list(cmd)
 484         if cmd is None:
 485             warnings.warn("Unable to list tests in %s" % name)
 486             continue
 487
 488         exitcode = subprocess.call(cmd, shell=True)
 489
 490         if exitcode != 0:
 491             sys.stderr.write("%s exited with exit code %s\n" % (cmd, exitcode))
 492             sys.exit(1)
 493 else:
 494     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 495         try:
 496             env = switch_env(envname, prefix)
 497         except UnsupportedEnvironment:
 498             subunit_ops.start_testsuite(name)
 499             subunit_ops.end_testsuite(name, "skip",
 500                 "environment %s is unknown in this test backend - skipping" % envname)
 501             continue
 502         except Exception, e:
 503             subunit_ops.start_testsuite(name)
 504             traceback.print_exc()
 505             subunit_ops.end_testsuite(name, "error",
 506                 "unable to set up environment %s: %s" % (envname, e))
 507             continue
 508
 509         cmd, tmpf = expand_command_run(cmd, supports_loadfile, supports_idlist,
 510             subtests)
 511
 512         run_testsuite(name, cmd, subunit_ops, env=env)
 513
 514         if tmpf is not None:
 515             os.remove(tmpf)
 516
 517         if opts.resetup_environment:
 518             env_manager.teardown_env(envname)
 519     env_manager.teardown_all()
 520
 521 sys.stdout.write("\n")
 522
 523 # if there were any valgrind failures, show them
 524 for fn in os.listdir(prefix):
 525     if fn.startswith("valgrind.log"):
 526         sys.stdout.write("VALGRIND FAILURE\n")
 527         f = open(os.path.join(prefix, fn), 'r')
 528         try:
 529             sys.stdout.write(f.read())
 530         finally:
 531             f.close()
 532
 533 sys.exit(0)