selftest/tests.py

   1 #!/usr/bin/python
   2 # This script generates a list of testsuites that should be run as part of
   3 # the Samba test suite.
   4
   5 # The output of this script is parsed by selftest.pl, which then decides
   6 # which of the tests to actually run. It will, for example, skip all tests
   7 # listed in selftest/skip or only run a subset during "make quicktest".
   8
   9 # The idea is that this script outputs all of the tests of Samba, not
  10 # just those that are known to pass, and list those that should be skipped
  11 # or are known to fail in selftest/skip or selftest/knownfail. This makes it
  12 # very easy to see what functionality is still missing in Samba and makes
  13 # it possible to run the testsuite against other servers, such as
  14 # Windows that have a different set of features.
  15
  16 # The syntax for a testsuite is "-- TEST --" on a single line, followed
  17 # by the name of the test, the environment it needs and the command to run, all
  18 # three separated by newlines. All other lines in the output are considered
  19 # comments.
  20
  21 import os
  22 from selftesthelpers import bindir, srcdir, python
  23 from selftesthelpers import planpythontestsuite, samba4srcdir
  24 from selftesthelpers import plantestsuite, bbdir
  25 from selftesthelpers import configuration, valgrindify
  26 from selftesthelpers import skiptestsuite
  27
  28 try:
  29     config_h = os.environ["CONFIG_H"]
  30 except KeyError:
  31     samba4bindir = bindir()
  32     config_h = os.path.join(samba4bindir, "default/include/config.h")
  33
  34 # check available features
  35 config_hash = dict()
  36 f = open(config_h, 'r')
  37 try:
  38     lines = f.readlines()
  39     config_hash = dict((x[0], ' '.join(x[1:]))
  40                        for x in map(lambda line: line.strip().split(' ')[1:],
  41                                     list(filter(lambda line: (line[0:7] == '#define') and (len(line.split(' ')) > 2), lines))))
  42 finally:
  43     f.close()
  44
  45 have_man_pages_support = ("XSLTPROC_MANPAGES" in config_hash)
  46 with_pam = ("WITH_PAM" in config_hash)
  47 pam_wrapper_so_path = config_hash["LIBPAM_WRAPPER_SO_PATH"]
  48 pam_set_items_so_path = config_hash["PAM_SET_ITEMS_SO_PATH"]
  49
  50 planpythontestsuite("none", "samba.tests.source")
  51 if have_man_pages_support:
  52     planpythontestsuite("none", "samba.tests.docs")
  53
  54 try:
  55     import testscenarios
  56 except ImportError:
  57     skiptestsuite("subunit", "testscenarios not available")
  58 else:
  59     planpythontestsuite("none", "subunit.tests.test_suite")
  60 planpythontestsuite("none", "samba.tests.blackbox.ndrdump")
  61 planpythontestsuite("none", "samba.tests.blackbox.check_output")
  62 planpythontestsuite("none", "api", name="ldb.python", extra_path=['lib/ldb/tests/python'])
  63 planpythontestsuite("none", "samba.tests.credentials")
  64 planpythontestsuite("none", "samba.tests.registry")
  65 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.auth")
  66 planpythontestsuite("none", "samba.tests.get_opt")
  67 planpythontestsuite("none", "samba.tests.security")
  68 planpythontestsuite("none", "samba.tests.dcerpc.misc")
  69 planpythontestsuite("none", "samba.tests.dcerpc.integer")
  70 planpythontestsuite("none", "samba.tests.param")
  71 planpythontestsuite("none", "samba.tests.upgrade")
  72 planpythontestsuite("none", "samba.tests.core")
  73 planpythontestsuite("none", "samba.tests.common")
  74 planpythontestsuite("none", "samba.tests.provision")
  75 planpythontestsuite("none", "samba.tests.password_quality")
  76 planpythontestsuite("none", "samba.tests.strings")
  77 planpythontestsuite("none", "samba.tests.netcmd")
  78 planpythontestsuite("none", "samba.tests.dcerpc.rpc_talloc")
  79 planpythontestsuite("none", "samba.tests.dcerpc.array")
  80 planpythontestsuite("none", "samba.tests.dcerpc.string_tests")
  81 planpythontestsuite("none", "samba.tests.hostconfig")
  82 planpythontestsuite("ad_dc_ntvfs:local", "samba.tests.messaging")
  83 planpythontestsuite("none", "samba.tests.s3param")
  84 planpythontestsuite("none", "samba.tests.s3passdb")
  85 planpythontestsuite("none", "samba.tests.s3registry")
  86 planpythontestsuite("none", "samba.tests.s3windb")
  87 planpythontestsuite("none", "samba.tests.s3idmapdb")
  88 planpythontestsuite("none", "samba.tests.samba3sam")
  89 planpythontestsuite(
  90     "none", "wafsamba.tests.test_suite",
  91     extra_path=[os.path.join(samba4srcdir, "..", "buildtools"),
  92                 os.path.join(samba4srcdir, "..", "third_party", "waf")])
  93 planpythontestsuite("fileserver", "samba.tests.smbd_fuzztest")
  94
  95
  96 def cmdline(script, *args):
  97     """
  98     Prefix PYTHON env var and append --configurefile option to abs script path.
  99
 100     script.sh arg1 arg2
 101     -->
 102     PYTHON=python /path/to/bbdir/script.sh arg1 arg2 \
 103     --configurefile $SMB_CONF_FILE
 104     """
 105     return [
 106         "PYTHON=%s" % python,
 107         os.path.join(bbdir, script),
 108     ] + list(args) + [configuration]
 109
 110
 111 plantestsuite(
 112     "samba4.blackbox.demote-saveddb", "none",
 113     cmdline('demote-saveddb.sh', '$PREFIX_ABS/demote'))
 114
 115 plantestsuite(
 116     "samba4.blackbox.dbcheck.alpha13", "none",
 117     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 118             'alpha13'))
 119
 120 # same test as above but skip member link checks
 121 plantestsuite(
 122     "samba4.blackbox.dbcheck.alpha13.quick", "none",
 123     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 124             'alpha13', '--quick-membership-checks'))
 125
 126 plantestsuite(
 127     "samba4.blackbox.dbcheck.release-4-0-0", "none",
 128     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 129             'release-4-0-0'))
 130
 131 # same test as above but skip member link checks
 132 plantestsuite(
 133     "samba4.blackbox.dbcheck.release-4-0-0.quick", "none",
 134     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 135             'release-4-0-0', '--quick-membership-checks'))
 136
 137 plantestsuite(
 138     "samba4.blackbox.dbcheck.release-4-1-0rc3", "none",
 139     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 140             'release-4-1-0rc3'))
 141
 142 # same test as above but skip member link checks
 143 plantestsuite(
 144     "samba4.blackbox.dbcheck.release-4-1-0rc3.quick", "none",
 145     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 146             'release-4-1-0rc3', '--quick-membership-checks'))
 147
 148 plantestsuite(
 149     "samba4.blackbox.dbcheck.release-4-1-6-partial-object", "none",
 150     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 151             'release-4-1-6-partial-object'))
 152
 153 # same test as above but skip member link checks
 154 plantestsuite(
 155     "samba4.blackbox.dbcheck.release-4-1-6-partial-object.quick", "none",
 156     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 157             'release-4-1-6-partial-object', '--quick-membership-checks'))
 158
 159 plantestsuite(
 160     "samba4.blackbox.dbcheck.release-4-5-0-pre1", "none",
 161     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 162             'release-4-5-0-pre1'))
 163
 164 # same test as above but skip member link checks
 165 plantestsuite(
 166     "samba4.blackbox.dbcheck.release-4-5-0-pre1.quick", "none",
 167     cmdline('dbcheck-oldrelease.sh', '$PREFIX_ABS/provision',
 168             'release-4-5-0-pre1', '--quick-membership-checks'))
 169
 170 plantestsuite(
 171     "samba4.blackbox.upgradeprovision.alpha13", "none",
 172     cmdline('upgradeprovision-oldrelease.sh', '$PREFIX_ABS/provision',
 173             'alpha13'))
 174
 175 plantestsuite(
 176     "samba4.blackbox.upgradeprovision.release-4-0-0", "none",
 177     cmdline('upgradeprovision-oldrelease.sh', '$PREFIX_ABS/provision',
 178             'release-4-0-0'))
 179
 180 plantestsuite(
 181     "samba4.blackbox.tombstones-expunge.release-4-5-0-pre1", "none",
 182     cmdline('tombstones-expunge.sh', '$PREFIX_ABS/provision',
 183             'release-4-5-0-pre1'))
 184
 185 plantestsuite(
 186     "samba4.blackbox.dbcheck-links.release-4-5-0-pre1", "none",
 187     cmdline('dbcheck-links.sh', '$PREFIX_ABS/provision',
 188             'release-4-5-0-pre1'))
 189
 190 plantestsuite(
 191     "samba4.blackbox.runtime-links.release-4-5-0-pre1", "none",
 192     cmdline('runtime-links.sh', '$PREFIX_ABS/provision',
 193             'release-4-5-0-pre1'))
 194
 195 plantestsuite(
 196     "samba4.blackbox.schemaupgrade", "none",
 197     cmdline('schemaupgrade.sh', '$PREFIX_ABS/provision'))
 198
 199 plantestsuite(
 200     "samba4.blackbox.functionalprep", "none",
 201     cmdline('functionalprep.sh', '$PREFIX_ABS/provision'))
 202
 203 planpythontestsuite("none", "samba.tests.upgradeprovision")
 204 planpythontestsuite("none", "samba.tests.xattr")
 205 planpythontestsuite("none", "samba.tests.ntacls")
 206 planpythontestsuite("none", "samba.tests.policy")
 207 planpythontestsuite("none", "samba.tests.kcc.graph")
 208 planpythontestsuite("none", "samba.tests.kcc.graph_utils")
 209 planpythontestsuite("none", "samba.tests.kcc.ldif_import_export")
 210 planpythontestsuite("none", "samba.tests.graph")
 211 plantestsuite("wafsamba.duplicate_symbols", "none", [os.path.join(srcdir(), "buildtools/wafsamba/test_duplicate_symbol.sh")])
 212 planpythontestsuite("none", "samba.tests.glue")
 213 planpythontestsuite("none", "samba.tests.tdb_util")
 214 planpythontestsuite("none", "samba.tests.samdb_api")
 215
 216 if with_pam:
 217     env = "ad_member"
 218     options = [
 219         {
 220             "description": "krb5",
 221             "pam_options": "krb5_auth krb5_ccache_type=FILE",
 222         },
 223         {
 224             "description": "default",
 225             "pam_options": "",
 226         },
 227     ]
 228     for o in options:
 229         description = o["description"]
 230         pam_options = "'%s'" % o["pam_options"]
 231
 232         plantestsuite("samba.tests.pam_winbind(local+%s)" % description, env,
 233                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 234                        valgrindify(python), pam_wrapper_so_path,
 235                        "$SERVER", "$USERNAME", "$PASSWORD",
 236                        pam_options])
 237         plantestsuite("samba.tests.pam_winbind(domain1+%s)" % description, env,
 238                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 239                        valgrindify(python), pam_wrapper_so_path,
 240                        "$DOMAIN", "$DC_USERNAME", "$DC_PASSWORD",
 241                        pam_options])
 242         plantestsuite("samba.tests.pam_winbind(domain2+%s)" % description, env,
 243                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 244                        valgrindify(python), pam_wrapper_so_path,
 245                        "$REALM", "$DC_USERNAME", "$DC_PASSWORD",
 246                        pam_options])
 247         plantestsuite("samba.tests.pam_winbind(domain3+%s)" % description, env,
 248                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 249                        valgrindify(python), pam_wrapper_so_path,
 250                        "''", "${DC_USERNAME}@${DOMAIN}", "$DC_PASSWORD",
 251                        pam_options])
 252         plantestsuite("samba.tests.pam_winbind(domain4+%s)" % description, env,
 253                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 254                        valgrindify(python), pam_wrapper_so_path,
 255                        "''", "${DC_USERNAME}@${REALM}", "$DC_PASSWORD",
 256                        pam_options])
 257         plantestsuite("samba.tests.pam_winbind(domain5+%s)" % description, env,
 258                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 259                        valgrindify(python), pam_wrapper_so_path,
 260                        "$REALM", "${DC_USERNAME}@${DOMAIN}", "$DC_PASSWORD",
 261                        pam_options])
 262         plantestsuite("samba.tests.pam_winbind(domain6+%s)" % description, env,
 263                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 264                        valgrindify(python), pam_wrapper_so_path,
 265                        "$DOMAIN", "${DC_USERNAME}@${REALM}", "$DC_PASSWORD",
 266                        pam_options])
 267         plantestsuite("samba.tests.pam_winbind(trust_f_both1+%s)" % description, env,
 268                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 269                        valgrindify(python), pam_wrapper_so_path,
 270                        "$TRUST_F_BOTH_DOMAIN",
 271                        "$TRUST_F_BOTH_USERNAME",
 272                        "$TRUST_F_BOTH_PASSWORD",
 273                        pam_options])
 274         plantestsuite("samba.tests.pam_winbind(trust_f_both2+%s)" % description, env,
 275                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 276                        valgrindify(python), pam_wrapper_so_path,
 277                        "$TRUST_F_BOTH_REALM",
 278                        "$TRUST_F_BOTH_USERNAME",
 279                        "$TRUST_F_BOTH_PASSWORD",
 280                        pam_options])
 281         plantestsuite("samba.tests.pam_winbind(trust_f_both3+%s)" % description, env,
 282                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 283                        valgrindify(python), pam_wrapper_so_path,
 284                        "''",
 285                        "${TRUST_F_BOTH_USERNAME}@${TRUST_F_BOTH_DOMAIN}",
 286                        "$TRUST_F_BOTH_PASSWORD",
 287                        pam_options])
 288         plantestsuite("samba.tests.pam_winbind(trust_f_both4+%s)" % description, env,
 289                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 290                        valgrindify(python), pam_wrapper_so_path,
 291                        "''",
 292                        "${TRUST_F_BOTH_USERNAME}@${TRUST_F_BOTH_REALM}",
 293                        "$TRUST_F_BOTH_PASSWORD",
 294                        pam_options])
 295         plantestsuite("samba.tests.pam_winbind(trust_f_both5+%s)" % description, env,
 296                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 297                        valgrindify(python), pam_wrapper_so_path,
 298                        "${TRUST_F_BOTH_REALM}",
 299                        "${TRUST_F_BOTH_USERNAME}@${TRUST_F_BOTH_DOMAIN}",
 300                        "$TRUST_F_BOTH_PASSWORD",
 301                        pam_options])
 302         plantestsuite("samba.tests.pam_winbind(trust_f_both6+%s)" % description, env,
 303                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 304                        valgrindify(python), pam_wrapper_so_path,
 305                        "${TRUST_F_BOTH_DOMAIN}",
 306                        "${TRUST_F_BOTH_USERNAME}@${TRUST_F_BOTH_REALM}",
 307                        "$TRUST_F_BOTH_PASSWORD",
 308                        pam_options])
 309         plantestsuite("samba.tests.pam_winbind(trust_e_both1+%s)" % description, env,
 310                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 311                        valgrindify(python), pam_wrapper_so_path,
 312                        "$TRUST_E_BOTH_DOMAIN",
 313                        "$TRUST_E_BOTH_USERNAME",
 314                        "$TRUST_E_BOTH_PASSWORD",
 315                        pam_options])
 316         plantestsuite("samba.tests.pam_winbind(trust_e_both2+%s)" % description, env,
 317                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 318                        valgrindify(python), pam_wrapper_so_path,
 319                        "$TRUST_E_BOTH_REALM",
 320                        "$TRUST_E_BOTH_USERNAME",
 321                        "$TRUST_E_BOTH_PASSWORD",
 322                        pam_options])
 323         plantestsuite("samba.tests.pam_winbind(trust_e_both3+%s)" % description, env,
 324                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 325                        valgrindify(python), pam_wrapper_so_path,
 326                        "''",
 327                        "${TRUST_E_BOTH_USERNAME}@${TRUST_E_BOTH_DOMAIN}",
 328                        "$TRUST_E_BOTH_PASSWORD",
 329                        pam_options])
 330         plantestsuite("samba.tests.pam_winbind(trust_e_both4+%s)" % description, env,
 331                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 332                        valgrindify(python), pam_wrapper_so_path,
 333                        "''",
 334                        "${TRUST_E_BOTH_USERNAME}@${TRUST_E_BOTH_REALM}",
 335                        "$TRUST_E_BOTH_PASSWORD",
 336                        pam_options])
 337         plantestsuite("samba.tests.pam_winbind(trust_e_both5+%s)" % description, env,
 338                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 339                        valgrindify(python), pam_wrapper_so_path,
 340                        "${TRUST_E_BOTH_REALM}",
 341                        "${TRUST_E_BOTH_USERNAME}@${TRUST_E_BOTH_DOMAIN}",
 342                        "$TRUST_E_BOTH_PASSWORD",
 343                        pam_options])
 344         plantestsuite("samba.tests.pam_winbind(trust_e_both6+%s)" % description, env,
 345                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind.sh"),
 346                        valgrindify(python), pam_wrapper_so_path,
 347                        "${TRUST_E_BOTH_DOMAIN}",
 348                        "${TRUST_E_BOTH_USERNAME}@${TRUST_E_BOTH_REALM}",
 349                        "$TRUST_E_BOTH_PASSWORD",
 350                        pam_options])
 351
 352         for authtok_options in ["", "use_authtok", "try_authtok"]:
 353             _pam_options = "'%s %s'" % (o["pam_options"], authtok_options)
 354             _description = "%s %s" % (description, authtok_options)
 355             plantestsuite("samba.tests.pam_winbind_chauthtok(domain+%s)" % _description, env,
 356                           [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind_chauthtok.sh"),
 357                            valgrindify(python), pam_wrapper_so_path, pam_set_items_so_path,
 358                            "$DOMAIN", "TestPamOptionsUser", "oldp@ssword0", "newp@ssword0",
 359                            _pam_options, 'yes',
 360                            "$DC_SERVER", "$DC_USERNAME", "$DC_PASSWORD"])
 361
 362         plantestsuite("samba.tests.pam_winbind_warn_pwd_expire(domain+%s)" % description, env,
 363                       [os.path.join(srcdir(), "python/samba/tests/test_pam_winbind_warn_pwd_expire.sh"),
 364                        valgrindify(python), pam_wrapper_so_path,
 365                        "$DOMAIN", "alice", "Secret007",
 366                        pam_options])
 367
 368
 369 plantestsuite("samba.unittests.krb5samba", "none",
 370               [os.path.join(bindir(), "default/testsuite/unittests/test_krb5samba")])
 371 plantestsuite("samba.unittests.sambafs_srv_pipe", "none",
 372               [os.path.join(bindir(), "default/testsuite/unittests/test_sambafs_srv_pipe")])
 373 plantestsuite("samba.unittests.lib_util_modules", "none",
 374               [os.path.join(bindir(), "default/testsuite/unittests/test_lib_util_modules")])
 375
 376 plantestsuite("samba.unittests.smb1cli_session", "none",
 377               [os.path.join(bindir(), "default/libcli/smb/test_smb1cli_session")])
 378
 379 plantestsuite("samba.unittests.talloc_keep_secret", "none",
 380               [os.path.join(bindir(), "default/lib/util/test_talloc_keep_secret")])
 381
 382 plantestsuite("samba.unittests.tldap", "none",
 383               [os.path.join(bindir(), "default/source3/test_tldap")])
 384 plantestsuite("samba.unittests.rfc1738", "none",
 385               [os.path.join(bindir(), "default/lib/util/test_rfc1738")])
 386 plantestsuite("samba.unittests.kerberos", "none",
 387               [os.path.join(bindir(), "test_kerberos")])
 388 plantestsuite("samba.unittests.ms_fnmatch", "none",
 389               [os.path.join(bindir(), "default/lib/util/test_ms_fnmatch")])
 390 plantestsuite("samba.unittests.byteorder", "none",
 391               [os.path.join(bindir(), "default/lib/util/test_byteorder")])
 392 plantestsuite("samba.unittests.bytearray", "none",
 393               [os.path.join(bindir(), "default/lib/util/test_bytearray")])
 394 plantestsuite("samba.unittests.byteorder_verify", "none",
 395               [os.path.join(bindir(), "default/lib/util/test_byteorder_verify")])
 396 plantestsuite("samba.unittests.ntlm_check", "none",
 397               [os.path.join(bindir(), "default/libcli/auth/test_ntlm_check")])
 398 plantestsuite("samba.unittests.gnutls", "none",
 399               [os.path.join(bindir(), "default/libcli/auth/test_gnutls")])
 400 plantestsuite("samba.unittests.rc4_passwd_buffer", "none",
 401               [os.path.join(bindir(), "default/libcli/auth/test_rc4_passwd_buffer")])
 402 plantestsuite("samba.unittests.schannel", "none",
 403               [os.path.join(bindir(), "default/libcli/auth/test_schannel")])
 404 plantestsuite("samba.unittests.test_registry_regfio", "none",
 405               [os.path.join(bindir(), "default/source3/test_registry_regfio")])
 406 plantestsuite("samba.unittests.test_oLschema2ldif", "none",
 407               [os.path.join(bindir(), "default/source4/utils/oLschema2ldif/test_oLschema2ldif")])
 408 plantestsuite("samba.unittests.mdsparser_es", "none",
 409               [os.path.join(bindir(), "default/source3/test_mdsparser_es")] + [configuration])