selftest/selftest.py

   1 #!/usr/bin/python -u
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2012 Jelmer Vernooij <jelmer@samba.org>
   4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 import atexit
  20 from cStringIO import StringIO
  21 import datetime
  22 import iso8601
  23 import os
  24 import sys
  25 import signal
  26 import subprocess
  27 import subunit
  28 import traceback
  29 import warnings
  30
  31 import optparse
  32
  33 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
  34
  35 from selftest import (
  36     socket_wrapper,
  37     subunithelper,
  38     testlist,
  39     )
  40 from selftest.client import write_clientconf
  41 from selftest.run import (
  42     expand_environment_strings,
  43     expand_command_list,
  44     expand_command_run,
  45     exported_envvars_str,
  46     )
  47 from selftest.target import (
  48     EnvironmentManager,
  49     NoneTarget,
  50     UnsupportedEnvironment,
  51     )
  52
  53 includes = ()
  54 excludes = ()
  55
  56 def now():
  57     return datetime.datetime.utcnow().replace(tzinfo=iso8601.iso8601.Utc())
  58
  59 def read_excludes(fn):
  60     excludes.extend(testlist.read_test_regexes(fn))
  61
  62 def read_includes(fn):
  63     includes.extend(testlist.read_test_regexes(fn))
  64
  65 parser = optparse.OptionParser("TEST-REGEXES")
  66 parser.add_option("--target", type="choice", choices=["samba", "samba3", "none"], default="samba", help="Samba version to target")
  67 parser.add_option("--quick", help="run quick overall test", action="store_true", default=False)
  68 parser.add_option("--list", help="list available tests", action="store_true", default=False)
  69 parser.add_option("--socket-wrapper", help="enable socket wrapper", action="store_true", default=False)
  70 parser.add_option("--socket-wrapper-pcap", help="save traffic to pcap directories", type="str")
  71 parser.add_option("--socket-wrapper-keep-pcap", help="keep all pcap files, not just those for tests that failed", action="store_true", default=False)
  72 parser.add_option("--one", help="abort when the first test fails", action="store_true", default=False)
  73 parser.add_option("--exclude", action="callback", help="Add file to exclude files", callback=read_excludes)
  74 parser.add_option("--include", action="callback", help="Add file to include files", callback=read_includes)
  75 parser.add_option("--testenv", help="run a shell in the requested test environment", action="store_true", default=False)
  76 parser.add_option("--resetup-environment", help="Re-setup environment", action="store_true", default=False)
  77 parser.add_option("--binary-mapping", help="Map binaries to use", type=str)
  78 parser.add_option("--load-list", help="Load list of tests to load from a file", type=str)
  79 parser.add_option("--prefix", help="prefix to run tests in", type=str, default="./st")
  80 parser.add_option("--srcdir", type=str, default=".", help="source directory")
  81 parser.add_option("--bindir", type=str, default="./bin", help="binaries directory")
  82 parser.add_option("--testlist", type=str, action="append", help="file to read available tests from")
  83 parser.add_option("--ldap", help="back samba onto specified ldap server", choices=["openldap", "fedora-ds"], type="choice")
  84
  85 opts, args = parser.parse_args()
  86
  87 subunit_ops = subunithelper.SubunitOps(sys.stdout)
  88
  89 def handle_signal(sig, frame):
  90     sys.stderr.write("Exiting early because of signal %s.\n" % sig)
  91     sys.exit(1)
  92
  93 for sig in (signal.SIGINT, signal.SIGQUIT, signal.SIGTERm, signal.SIGPIPE):
  94     signal.signal(sig, handle_signal)
  95
  96 def skip(name):
  97     return testlist.find_in_list(excludes, name)
  98
  99 def setup_pcap(name):
 100     if (not opts.socket_wrapper_pcap or
 101         not os.environ.get("SOCKET_WRAPPER_PCAP_DIR")):
 102         return
 103
 104     fname = "".join([x for x in name if x.isalnum() or x == '-'])
 105
 106     pcap_file = os.path.join(
 107         os.environ["SOCKET_WRAPPER_PCAP_DIR"], "%s.pcap" % fname)
 108
 109     socket_wrapper.setup_pcap(pcap_file)
 110     return pcap_file
 111
 112
 113 def cleanup_pcap(pcap_file, exit_code):
 114     if not opts.socket_wrapper_pcap:
 115         return
 116     if opts.socket_wrapper_keep_pcap:
 117         return
 118     if exitcode == 0:
 119         return
 120     if pcap_file is None:
 121         return
 122
 123     os.unlink(pcap_file)
 124
 125
 126 def run_testsuite(envname, name, cmd):
 127     """Run a single testsuite.
 128
 129     :param envname: Name of the environment to ru nin
 130     :param name: Name of the testsuite
 131     :param cmd: Name of the (fully expanded) command to run
 132     :return: exitcode of the command
 133     """
 134     pcap_file = setup_pcap(name)
 135
 136     subunit_ops.start_testsuite(name)
 137     subunit_ops.progress(None, subunit.PROGRESS_PUSH)
 138     subunit_ops.time(now())
 139     try:
 140         exitcode = subprocess.call(cmd, shell=True)
 141     except Exception, e:
 142         subunit_ops.time(now())
 143         subunit_ops.progress(None, subunit.PROGRESS_POP)
 144         subunit_ops.end_testsuite(name, "error", "Unable to run %r: %s" % (cmd, e))
 145         sys.exit(1)
 146
 147     subunit_ops.time(now())
 148     subunit_ops.progress(None, subunit.PROGRESS_POP)
 149
 150     envlog = env_manager.getlog_env(envname)
 151     if envlog != "":
 152         sys.stdout.write("envlog: %s\n" % envlog)
 153
 154     sys.stdout.write("command: %s\n" % cmd)
 155     sys.stdout.write("expanded command: %s\n" % expand_environment_strings(cmd, os.environ))
 156
 157     if exitcode == 0:
 158         subunit_ops.end_testsuite(name, "success")
 159     else:
 160         subunit_ops.end_testsuite(name, "failure", "Exit code was %d" % exitcode)
 161
 162     cleanup_pcap(pcap_file, exitcode)
 163
 164     if not opts.socket_wrapper_keep_pcap and pcap_file is not None:
 165         sys.stdout.write("PCAP FILE: %s\n" % pcap_file)
 166
 167     if exitcode != 0 and opts.one:
 168         sys.exit(1)
 169
 170     return exitcode
 171
 172 if opts.list and opts.testenv:
 173     sys.stderr.write("--list and --testenv are mutually exclusive\n")
 174     sys.exit(1)
 175
 176 tests = args
 177
 178 # quick hack to disable rpc validation when using valgrind - it is way too slow
 179 if not os.environ.get("VALGRIND"):
 180     os.environ["VALIDATE"] = "validate"
 181     os.environ["MALLOC_CHECK_"] = "2"
 182
 183 # make all our python scripts unbuffered
 184 os.environ["PYTHONUNBUFFERED"] = "1"
 185
 186 bindir_abs = os.path.abspath(opts.bindir)
 187
 188 # Backwards compatibility:
 189 if os.environ.get("TEST_LDAP") == "yes":
 190     if os.environ.get("FEDORA_DS_ROOT"):
 191         ldap = "fedora-ds"
 192     else:
 193         ldap = "openldap"
 194
 195 torture_maxtime = int(os.getenv("TORTURE_MAXTIME", "1200"))
 196 if opts.ldap:
 197     # LDAP is slow
 198     torture_maxtime *= 2
 199
 200 prefix = os.path.normpath(opts.prefix)
 201
 202 if prefix == "":
 203     raise Exception("using an empty prefix isn't allowed")
 204
 205 # Ensure we have the test prefix around.
 206 #
 207 # We need restrictive permissions on this as some subdirectories in this tree
 208 # will have wider permissions (ie 0777) and this would allow other users on the
 209 # host to subvert the test process.
 210 if not os.path.isdir(prefix):
 211     os.mkdir(prefix, 0700)
 212 else:
 213     os.chmod(prefix, 0700)
 214
 215 prefix_abs = os.path.abspath(prefix)
 216 tmpdir_abs = os.path.abspath(os.path.join(prefix_abs, "tmp"))
 217 if not os.path.isdir(tmpdir_abs):
 218     os.mkdir(tmpdir_abs, 0777)
 219
 220 srcdir_abs = os.path.abspath(opts.srcdir)
 221
 222 if prefix_abs == "":
 223     raise Exception("using an empty absolute prefix isn't allowed")
 224 if prefix_abs == "/":
 225     raise Exception("using '/' as absolute prefix isn't allowed")
 226
 227 os.environ["PREFIX"] = prefix
 228 os.environ["KRB5CCNAME"] = os.path.join(prefix, "krb5ticket")
 229 os.environ["PREFIX_ABS"] = prefix_abs
 230 os.environ["SRCDIR"] = opts.srcdir
 231 os.environ["SRCDIR_ABS"] = srcdir_abs
 232 os.environ["BINDIR"] = bindir_abs
 233
 234 tls_enabled = not opts.quick
 235 if tls_enabled:
 236     os.environ["TLS_ENABLED"] = "yes"
 237 else:
 238     os.environ["TLS_ENABLED"] = "no"
 239
 240 def prefix_pathvar(name, newpath):
 241     if name in os.environ:
 242         os.environ[name] = "%s:%s" % (newpath, os.environ[name])
 243     else:
 244         os.environ[name] = newpath
 245 prefix_pathvar("PKG_CONFIG_PATH", os.path.join(bindir_abs, "pkgconfig"))
 246 prefix_pathvar("PYTHONPATH", os.path.join(bindir_abs, "python"))
 247
 248 if opts.socket_wrapper_keep_pcap:
 249     # Socket wrapper keep pcap implies socket wrapper pcap
 250     opts.socket_wrapper_pcap = True
 251
 252 if opts.socket_wrapper_pcap:
 253     # Socket wrapper pcap implies socket wrapper
 254     opts.socket_wrapper = True
 255
 256 if opts.socket_wrapper:
 257     socket_wrapper_dir = socket_wrapper.setup_dir(os.path.join(prefix_abs, "w"), opts.socket_wrapper_pcap)
 258     sys.stdout.write("SOCKET_WRAPPER_DIR=%s\n" % socket_wrapper_dir)
 259 elif not opts.list:
 260     if os.getuid() != 0:
 261         warnings.warn("not using socket wrapper, but also not running as root. Will not be able to listen on proper ports")
 262
 263 testenv_default = "none"
 264
 265 if opts.binary_mapping:
 266     binary_mapping = dict([l.split(":") for l in opts.binary_mapping.split(",")])
 267     os.environ["BINARY_MAPPING"] = opts.binary_mapping
 268 else:
 269     binary_mapping = {}
 270     os.environ["BINARY_MAPPING"] = ""
 271
 272 # After this many seconds, the server will self-terminate.  All tests
 273 # must terminate in this time, and testenv will only stay alive this
 274 # long
 275
 276 server_maxtime = 7500
 277 if os.environ.get("SMBD_MAXTIME", ""):
 278     server_maxtime = int(os.environ["SMBD_MAXTIME"])
 279
 280
 281 def has_socket_wrapper(bindir):
 282     """Check if Samba has been built with socket wrapper support.
 283     """
 284     f = StringIO()
 285     subprocess.check_call([os.path.join(bindir, "smbd"), "-b"], stdout=f)
 286     for l in f.readlines():
 287         if "SOCKET_WRAPPER" in l:
 288             return True
 289     return False
 290
 291
 292 if not opts.list:
 293     if opts.target == "samba":
 294         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 295             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 296             sys.exit(1)
 297         testenv_default = "dc"
 298         from selftest.target.samba import Samba
 299         target = Samba(opts.bindir, binary_mapping, ldap, opts.srcdir, server_maxtime)
 300     elif opts.target == "samba3":
 301         if opts.socket_wrapper and not has_socket_wrapper(opts.bindir):
 302             sys.stderr.write("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....\n")
 303             sys.exit(1)
 304         testenv_default = "member"
 305         from selftest.target.samba3 import Samba3
 306         target = Samba3(opts.bindir, binary_mapping, srcdir_abs, server_maxtime)
 307     elif opts.target == "none":
 308         testenv_default = "none"
 309         target = NoneTarget()
 310
 311     env_manager = EnvironmentManager(target)
 312     atexit.register(env_manager.teardown_all)
 313
 314 interfaces = ",".join([
 315     "127.0.0.11/8",
 316     "127.0.0.12/8",
 317     "127.0.0.13/8",
 318     "127.0.0.14/8",
 319     "127.0.0.15/8",
 320     "127.0.0.16/8"])
 321
 322 clientdir = os.path.join(prefix_abs, "client")
 323
 324 conffile = os.path.join(clientdir, "client.conf")
 325 os.environ["SMB_CONF_PATH"] = conffile
 326
 327 todo = []
 328
 329 if not opts.testlist:
 330     sys.stderr.write("No testlists specified\n")
 331     sys.exit(1)
 332
 333 os.environ["SELFTEST_PREFIX"] = prefix_abs
 334 os.environ["SELFTEST_TMPDIR"] = tmpdir_abs
 335 os.environ["TEST_DATA_PREFIX"] = tmpdir_abs
 336 if opts.socket_wrapper:
 337     os.environ["SELFTEST_INTERFACES"] = interfaces
 338 else:
 339     os.environ["SELFTEST_INTERFACES"] = ""
 340 if opts.quick:
 341     os.environ["SELFTEST_QUICK"] = "1"
 342 else:
 343     os.environ["SELFTEST_QUICK"] = ""
 344 os.environ["SELFTEST_MAXTIME"] = str(torture_maxtime)
 345
 346
 347 available = []
 348 for fn in opts.testlist:
 349     for testsuite in testlist.read_testlist_file(fn):
 350         if not testlist.should_run_test(tests, testsuite):
 351             continue
 352         name = testsuite[0]
 353         if (includes is not None and
 354             testlist.find_in_list(includes, name) is not None):
 355             continue
 356         available.append(testsuite)
 357
 358 if opts.load_list:
 359     restricted_mgr = testlist.RestrictedTestManager.from_path(opts.load_list)
 360 else:
 361     restricted_mgr = None
 362
 363 for testsuite in available:
 364     name = testsuite[0]
 365     skipreason = skip(name)
 366     if restricted_mgr is not None:
 367         match = restricted_mgr.should_run_testsuite(name)
 368         if match == []:
 369             continue
 370     else:
 371         match = None
 372     if skipreason is not None:
 373         if not opts.list:
 374             subunit_ops.skip_testsuite(name, skipreason)
 375     else:
 376         todo.append(testsuite + (match,))
 377
 378 if restricted_mgr is not None:
 379     for name in restricted_mgr.iter_unused():
 380         sys.stdout.write("No test or testsuite found matching %s\n" % name)
 381 if todo == []:
 382     sys.stderr.write("No tests to run\n")
 383     sys.exit(1)
 384
 385 suitestotal = len(todo)
 386
 387 if not opts.list:
 388     subunit_ops.progress(suitestotal, subunit.PROGRESS_SET)
 389     subunit_ops.time(now())
 390
 391 exported_envvars = [
 392     # domain stuff
 393     "DOMAIN",
 394     "REALM",
 395
 396     # domain controller stuff
 397     "DC_SERVER",
 398     "DC_SERVER_IP",
 399     "DC_NETBIOSNAME",
 400     "DC_NETBIOSALIAS",
 401
 402     # domain member
 403     "MEMBER_SERVER",
 404     "MEMBER_SERVER_IP",
 405     "MEMBER_NETBIOSNAME",
 406     "MEMBER_NETBIOSALIAS",
 407
 408     # rpc proxy controller stuff
 409     "RPC_PROXY_SERVER",
 410     "RPC_PROXY_SERVER_IP",
 411     "RPC_PROXY_NETBIOSNAME",
 412     "RPC_PROXY_NETBIOSALIAS",
 413
 414     # domain controller stuff for Vampired DC
 415     "VAMPIRE_DC_SERVER",
 416     "VAMPIRE_DC_SERVER_IP",
 417     "VAMPIRE_DC_NETBIOSNAME",
 418     "VAMPIRE_DC_NETBIOSALIAS",
 419
 420     # server stuff
 421     "SERVER",
 422     "SERVER_IP",
 423     "NETBIOSNAME",
 424     "NETBIOSALIAS",
 425
 426     # user stuff
 427     "USERNAME",
 428     "USERID",
 429     "PASSWORD",
 430     "DC_USERNAME",
 431     "DC_PASSWORD",
 432
 433     # misc stuff
 434     "KRB5_CONFIG",
 435     "WINBINDD_SOCKET_DIR",
 436     "WINBINDD_PRIV_PIPE_DIR",
 437     "NMBD_SOCKET_DIR",
 438     "LOCAL_PATH"
 439 ]
 440
 441 def switch_env(name, prefix):
 442     if ":" in name:
 443         (envname, option) = name.split(":", 1)
 444     else:
 445         envname = name
 446         option = "client"
 447
 448     env = env_manager.setup_env(envname, prefix)
 449
 450     testenv_vars = env.get_vars()
 451
 452     if option == "local":
 453         socket_wrapper.set_default_iface(testenv_vars["SOCKET_WRAPPER_DEFAULT_IFACE"])
 454         os.environ["SMB_CONF_PATH"] = testenv_vars["SERVERCONFFILE"]
 455     elif option == "client":
 456         socket_wrapper.set_default_iface(11)
 457         write_clientconf(conffile, clientdir, testenv_vars)
 458         os.environ["SMB_CONF_PATH"] = conffile
 459     else:
 460         raise Exception("Unknown option[%s] for envname[%s]" % (option,
 461             envname))
 462
 463     for name in exported_envvars:
 464         if name in testenv_vars:
 465             os.environ[name] = testenv_vars[name]
 466         elif name in os.environ:
 467             del os.environ[name]
 468
 469     return testenv_vars
 470
 471 # This 'global' file needs to be empty when we start
 472 dns_host_file_path = os.path.join(prefix_abs, "dns_host_file")
 473 if os.path.exists(dns_host_file_path):
 474     os.unlink(dns_host_file_path)
 475
 476 if opts.testenv:
 477     testenv_name = os.environ.get("SELFTEST_TESTENV", testenv_default)
 478
 479     testenv_vars = switch_env(testenv_name, prefix)
 480
 481     os.environ["PIDDIR"] = testenv_vars["PIDDIR"]
 482     os.environ["ENVNAME"] = testenv_name
 483
 484     envvarstr = exported_envvars_str(testenv_vars, exported_envvars)
 485
 486     term = os.environ.get("TERMINAL", "xterm -e")
 487     cmd = """'echo -e "
 488 Welcome to the Samba4 Test environment '%(testenv_name)'
 489
 490 This matches the client environment used in make test
 491 server is pid `cat \$PIDDIR/samba.pid`
 492
 493 Some useful environment variables:
 494 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 495 SMB_CONF_PATH=\$SMB_CONF_PATH
 496
 497 $envvarstr
 498 \" && LD_LIBRARY_PATH=%(LD_LIBRARY_PATH)s $(SHELL)'""" % {
 499         "testenv_name": testenv_name,
 500         "LD_LIBRARY_PATH": os.environ["LD_LIBRARY_PATH"]}
 501     subprocess.call(term + ' ' + cmd, shell=True)
 502     env_manager.teardown_env(testenv_name)
 503 elif opts.list:
 504     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 505         cmd = expand_command_list(cmd)
 506         if cmd is None:
 507             warnings.warn("Unable to list tests in %s" % name)
 508             continue
 509
 510         exitcode = subprocess.call(cmd, shell=True)
 511
 512         if exitcode != 0:
 513             sys.stderr.write("%s exited with exit code %s\n" % (cmd, exitcode))
 514             sys.exit(1)
 515 else:
 516     for (name, envname, cmd, supports_loadfile, supports_idlist, subtests) in todo:
 517         try:
 518             envvars = switch_env(envname, prefix)
 519         except UnsupportedEnvironment:
 520             subunit_ops.start_testsuite(name)
 521             subunit_ops.end_testsuite(name, "skip",
 522                 "environment %s is unknown in this test backend - skipping" % envname)
 523             continue
 524         except Exception, e:
 525             subunit_ops.start_testsuite(name)
 526             traceback.print_exc()
 527             subunit_ops.end_testsuite(name, "error",
 528                 "unable to set up environment %s: %s" % (envname, e))
 529             continue
 530
 531         cmd, tmpf = expand_command_run(cmd, supports_loadfile, supports_idlist,
 532             subtests)
 533
 534         run_testsuite(envname, name, cmd)
 535
 536         if tmpf is not None:
 537             os.remove(tmpf)
 538
 539         if opts.resetup_environment:
 540             env_manager.teardown_env(envname)
 541     env_manager.teardown_all()
 542
 543 sys.stdout.write("\n")
 544
 545 # if there were any valgrind failures, show them
 546 for fn in os.listdir(prefix):
 547     if fn.startswith("valgrind.log"):
 548         sys.stdout.write("VALGRIND FAILURE\n")
 549         f = open(os.path.join(prefix, fn), 'r')
 550         try:
 551             sys.stdout.write(f.read())
 552         finally:
 553             f.close()
 554
 555 sys.exit(0)