script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 if args:
 142     # If we are only running specific test,
 143     # do not sleep randomly to wait for it to start
 144     def random_sleep(low, high):
 145         return 'sleep 1'
 146 else:
 147     def random_sleep(low, high):
 148         return 'sleep {}'.format(random.randint(low, high))
 149
 150 cleanup_list = []
 151
 152 builddirs = {
 153     "ctdb": "ctdb",
 154     "tdb": "lib/tdb",
 155     "talloc": "lib/talloc",
 156     "replace": "lib/replace",
 157     "tevent": "lib/tevent",
 158     "pidl": "pidl",
 159     "docs-xml": "docs-xml"
 160 }
 161
 162 ctdb_configure_params = " --enable-developer ${PREFIX}"
 163 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 164
 165 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 166 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 167 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 168 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 169 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 170 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!tevent,!pytevent,!popt"
 171 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 172
 173
 174 def format_option(name, value=None):
 175     """Format option as str list."""
 176     if value is None:  # boolean option
 177         return [name]
 178     if not isinstance(value, list):  # single value option
 179         value = [value]
 180     # repeatable option
 181     return ['{}={}'.format(name, item) for item in value]
 182
 183
 184 def make_test(
 185         cmd='make testonly',
 186         INJECT_SELFTEST_PREFIX=1,
 187         TESTS='',
 188         include_envs=None,
 189         exclude_envs=None):
 190
 191     test_options = []
 192     if include_envs:
 193         test_options = format_option('--include-env', include_envs)
 194     if exclude_envs:
 195         test_options = format_option('--exclude-env', exclude_envs)
 196     if test_options:
 197         # join envs options to original test options
 198         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 199
 200     _options = []
 201
 202     # Allow getting a full CI with
 203     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 204
 205     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 206
 207     if int(FAIL_IMMEDIATELY):
 208         _options.append('FAIL_IMMEDIATELY=1')
 209     if TESTS:
 210         _options.append("TESTS='{}'".format(TESTS))
 211
 212     if INJECT_SELFTEST_PREFIX:
 213         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 214         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 215
 216     return ' '.join([cmd] + _options)
 217
 218
 219 # When updating this list, also update .gitlab-ci.yml to add the job
 220 # and to make it a dependency of 'page' for the coverage report.
 221
 222 tasks = {
 223     "ctdb": {
 224         "sequence": [
 225             ("random-sleep", random_sleep(300, 900)),
 226             ("configure", "./configure " + ctdb_configure_params),
 227             ("make", "make all"),
 228             ("install", "make install"),
 229             ("test", "make autotest"),
 230             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 231             ("clean", "make clean"),
 232         ],
 233     },
 234     "docs-xml": {
 235         "sequence": [
 236             ("random-sleep", random_sleep(300, 900)),
 237             ("autoconf", "autoconf"),
 238             ("configure", "./configure"),
 239             ("make", "make html htmlman"),
 240             ("publish-docs", PUBLISH_DOCS),
 241             ("clean", "make clean"),
 242         ],
 243     },
 244
 245     "samba-def-build": {
 246         "git-clone-required": True,
 247         "sequence": [
 248             ("configure", "./configure.developer" + samba_configure_params),
 249             ("make", "make -j"),
 250             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 251             ("chmod-R-a-w", "chmod -R a-w ."),
 252         ],
 253     },
 254
 255     "samba-mit-build": {
 256         "git-clone-required": True,
 257         "sequence": [
 258             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 259             ("make", "make -j"),
 260             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 261             ("chmod-R-a-w", "chmod -R a-w ."),
 262         ],
 263     },
 264
 265     "samba-nt4-build": {
 266         "git-clone-required": True,
 267         "sequence": [
 268             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 269             ("make", "make -j"),
 270             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 271             ("chmod-R-a-w", "chmod -R a-w ."),
 272         ],
 273     },
 274
 275     "samba-h5l-build": {
 276         "git-clone-required": True,
 277         "sequence": [
 278             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 279             ("make", "make -j"),
 280             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 281             ("chmod-R-a-w", "chmod -R a-w ."),
 282         ],
 283     },
 284
 285     "samba-without-smb1-build": {
 286         "git-clone-required": True,
 287         "sequence": [
 288             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 289             ("make", "make -j"),
 290             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 291             ("chmod-R-a-w", "chmod -R a-w ."),
 292         ],
 293     },
 294
 295     "samba-no-opath-build": {
 296         "git-clone-required": True,
 297         "sequence": [
 298             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 299             ("make", "make -j"),
 300             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 301             ("chmod-R-a-w", "chmod -R a-w ."),
 302         ],
 303     },
 304
 305     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 306     "samba": {
 307         "sequence": [
 308             ("random-sleep", random_sleep(300, 900)),
 309             ("configure", "./configure.developer" + samba_configure_params),
 310             ("make", "make -j"),
 311             ("test", make_test(exclude_envs=[
 312             "none",
 313             "nt4_dc",
 314             "nt4_dc_smb1",
 315             "nt4_dc_smb1_done",
 316             "nt4_dc_schannel",
 317             "nt4_member",
 318             "ad_dc",
 319             "ad_dc_smb1",
 320             "ad_dc_smb1_done",
 321             "ad_dc_backup",
 322             "ad_dc_ntvfs",
 323             "ad_dc_default",
 324             "ad_dc_default_smb1",
 325             "ad_dc_slowtests",
 326             "ad_dc_no_nss",
 327             "ad_dc_no_ntlm",
 328             "fl2003dc",
 329             "fl2008dc",
 330             "fl2008r2dc",
 331             "ad_member",
 332             "ad_member_idmap_rid",
 333             "admem_idmap_autorid",
 334             "ad_member_idmap_ad",
 335             "ad_member_rfc2307",
 336             "ad_member_idmap_nss",
 337             "ad_member_oneway",
 338             "chgdcpass",
 339             "vampire_2000_dc",
 340             "fl2000dc",
 341             "fileserver",
 342             "fileserver_smb1",
 343             "fileserver_smb1_done",
 344             "maptoguest",
 345             "simpleserver",
 346             "backupfromdc",
 347             "restoredc",
 348             "renamedc",
 349             "offlinebackupdc",
 350             "labdc",
 351             "preforkrestartdc",
 352             "proclimitdc",
 353             "promoted_dc",
 354             "vampire_dc",
 355             "rodc",
 356             "ad_dc_default",
 357             "ad_dc_default_smb1",
 358             "ad_dc_default_smb1_done",
 359             "ad_dc_slowtests",
 360             "schema_pair_dc",
 361             "schema_dc",
 362             "clusteredmember",
 363             "ad_dc_fips",
 364             "ad_member_fips",
 365             ])),
 366             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 367             ("lcov", LCOV_CMD),
 368             ("install", "make install"),
 369             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 370             ("clean", "make clean"),
 371         ],
 372     },
 373
 374     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 375     "samba-mitkrb5": {
 376         "sequence": [
 377             ("random-sleep", random_sleep(300, 900)),
 378             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 379             ("make", "make -j"),
 380             ("test", make_test(exclude_envs=[
 381             "none",
 382             "nt4_dc",
 383             "nt4_dc_smb1",
 384             "nt4_dc_smb1_done",
 385             "nt4_dc_schannel",
 386             "nt4_member",
 387             "ad_dc",
 388             "ad_dc_smb1",
 389             "ad_dc_smb1_done",
 390             "ad_dc_backup",
 391             "ad_dc_ntvfs",
 392             "ad_dc_default",
 393             "ad_dc_default_smb1",
 394             "ad_dc_default_smb1_done",
 395             "ad_dc_slowtests",
 396             "ad_dc_no_nss",
 397             "ad_dc_no_ntlm",
 398             "fl2003dc",
 399             "fl2008dc",
 400             "fl2008r2dc",
 401             "ad_member",
 402             "ad_member_idmap_rid",
 403             "admem_idmap_autorid",
 404             "ad_member_idmap_ad",
 405             "ad_member_rfc2307",
 406             "ad_member_idmap_nss",
 407             "ad_member_oneway",
 408             "chgdcpass",
 409             "vampire_2000_dc",
 410             "fl2000dc",
 411             "fileserver",
 412             "fileserver_smb1",
 413             "fileserver_smb1_done",
 414             "maptoguest",
 415             "simpleserver",
 416             "backupfromdc",
 417             "restoredc",
 418             "renamedc",
 419             "offlinebackupdc",
 420             "labdc",
 421             "preforkrestartdc",
 422             "proclimitdc",
 423             "promoted_dc",
 424             "vampire_dc",
 425             "rodc",
 426             "ad_dc_default",
 427             "ad_dc_default_smb1",
 428             "ad_dc_default_smb1_done",
 429             "ad_dc_slowtests",
 430             "schema_pair_dc",
 431             "schema_dc",
 432             "clusteredmember",
 433             "ad_dc_fips",
 434             "ad_member_fips",
 435             ])),
 436             ("lcov", LCOV_CMD),
 437             ("install", "make install"),
 438             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 439             ("clean", "make clean"),
 440         ],
 441     },
 442
 443     "samba-nt4": {
 444         "dependency": "samba-nt4-build",
 445         "sequence": [
 446             ("random-sleep", random_sleep(300, 900)),
 447             ("test", make_test(include_envs=[
 448             "nt4_dc",
 449             "nt4_dc_smb1",
 450             "nt4_dc_smb1_done",
 451             "nt4_dc_schannel",
 452             "nt4_member",
 453             "simpleserver",
 454             ])),
 455             ("lcov", LCOV_CMD),
 456             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 457         ],
 458     },
 459
 460     "samba-fileserver": {
 461         "dependency": "samba-h5l-build",
 462         "sequence": [
 463             ("random-sleep", random_sleep(300, 900)),
 464             ("test", make_test(include_envs=[
 465             "fileserver",
 466             "fileserver_smb1",
 467             "fileserver_smb1_done",
 468             "maptoguest",
 469             "ktest", # ktest is also tested in samba-ktest-mit samba
 470                      # and samba-mitkrb5 but is tested here against
 471                      # a system Heimdal
 472             ])),
 473             ("lcov", LCOV_CMD),
 474             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 475         ],
 476     },
 477
 478     "samba-fileserver-without-smb1": {
 479         "dependency": "samba-without-smb1-build",
 480         "sequence": [
 481             ("random-sleep", random_sleep(300, 900)),
 482             ("test", make_test(include_envs=["fileserver"])),
 483             ("lcov", LCOV_CMD),
 484             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 485         ],
 486     },
 487
 488     # This is a full build without the AD DC so we test the build with
 489     # MIT Kerberos from the current system.  Runtime behaviour is
 490     # confirmed via the ktest (static ccache and keytab) environment
 491
 492     # This environment also used to confirm we can still build with --with-libunwind
 493     "samba-ktest-mit": {
 494         "sequence": [
 495             ("random-sleep", random_sleep(300, 900)),
 496             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 497             ("make", "make -j"),
 498             ("test", make_test(include_envs=[
 499             "ktest", # ktest is also tested in fileserver, samba and
 500                      # samba-mitkrb5 but is tested here against a
 501                      # system MIT krb5
 502             ])),
 503             ("lcov", LCOV_CMD),
 504             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 505         ],
 506     },
 507
 508     "samba-admem": {
 509         "dependency": "samba-def-build",
 510         "sequence": [
 511             ("random-sleep", random_sleep(300, 900)),
 512             ("test", make_test(include_envs=[
 513             "ad_member",
 514             "ad_member_idmap_rid",
 515             "admem_idmap_autorid",
 516             "ad_member_idmap_ad",
 517             "ad_member_rfc2307",
 518             "ad_member_idmap_nss",
 519             "ad_member_offlogon",
 520             ])),
 521             ("lcov", LCOV_CMD),
 522             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 523         ],
 524     },
 525
 526     "samba-no-opath1": {
 527         "dependency": "samba-no-opath-build",
 528         "sequence": [
 529             ("random-sleep", random_sleep(300, 900)),
 530             ("test", make_test(
 531                 cmd="make testonly DISABLE_OPATH=1",
 532                 include_envs=[
 533                 "nt4_dc",
 534                 "nt4_dc_smb1",
 535                 "nt4_dc_smb1_done",
 536                 "nt4_dc_schannel",
 537                 "nt4_member",
 538                 "simpleserver",
 539                 ])),
 540             ("lcov", LCOV_CMD),
 541             ("check-clean-tree", "script/clean-source-tree.sh"),
 542         ],
 543     },
 544
 545     "samba-no-opath2": {
 546         "dependency": "samba-no-opath-build",
 547         "sequence": [
 548             ("random-sleep", random_sleep(300, 900)),
 549             ("test", make_test(
 550                 cmd="make testonly DISABLE_OPATH=1",
 551                 include_envs=[
 552                 "fileserver",
 553                 "fileserver_smb1",
 554                 "fileserver_smb1_done",
 555                 ])),
 556             ("lcov", LCOV_CMD),
 557             ("check-clean-tree", "script/clean-source-tree.sh"),
 558         ],
 559     },
 560
 561     "samba-ad-dc-1": {
 562         "dependency": "samba-def-build",
 563         "sequence": [
 564             ("random-sleep", random_sleep(1, 1)),
 565             ("test", make_test(include_envs=[
 566             "ad_dc",
 567             "ad_dc_smb1",
 568             "ad_dc_smb1_done",
 569             "ad_dc_no_nss",
 570             "ad_dc_no_ntlm",
 571             ])),
 572             ("lcov", LCOV_CMD),
 573             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 574         ],
 575     },
 576
 577     "samba-ad-dc-2": {
 578         "dependency": "samba-def-build",
 579         "sequence": [
 580             ("random-sleep", random_sleep(1, 1)),
 581             ("test", make_test(include_envs=[
 582             "vampire_dc",
 583             "vampire_2000_dc",
 584             "rodc",
 585             ])),
 586             ("lcov", LCOV_CMD),
 587             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 588         ],
 589     },
 590
 591     "samba-ad-dc-3": {
 592         "dependency": "samba-def-build",
 593         "sequence": [
 594             ("random-sleep", random_sleep(1, 1)),
 595             ("test", make_test(include_envs=[
 596             "promoted_dc",
 597             "chgdcpass",
 598             "preforkrestartdc",
 599             "proclimitdc",
 600             ])),
 601             ("lcov", LCOV_CMD),
 602             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 603         ],
 604     },
 605
 606     "samba-ad-dc-4a": {
 607         "dependency": "samba-def-build",
 608         "sequence": [
 609             ("random-sleep", random_sleep(1, 1)),
 610             ("test", make_test(include_envs=[
 611             "fl2000dc",
 612             "ad_member_oneway",
 613             "fl2003dc",
 614             ])),
 615             ("lcov", LCOV_CMD),
 616             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 617         ],
 618     },
 619     "samba-ad-dc-4b": {
 620         "dependency": "samba-def-build",
 621         "sequence": [
 622             ("random-sleep", random_sleep(1, 1)),
 623             ("test", make_test(include_envs=[
 624             "fl2008dc",
 625             "fl2008r2dc",
 626             ])),
 627             ("lcov", LCOV_CMD),
 628             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 629         ],
 630     },
 631
 632     "samba-ad-dc-5": {
 633         "dependency": "samba-def-build",
 634         "sequence": [
 635             ("random-sleep", random_sleep(1, 1)),
 636             ("test", make_test(include_envs=[
 637             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-ad-dc-6": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     "samba-schemaupgrade": {
 654         "dependency": "samba-def-build",
 655         "sequence": [
 656             ("random-sleep", random_sleep(1, 1)),
 657             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 658             ("lcov", LCOV_CMD),
 659             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 660         ],
 661     },
 662
 663     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 664     # This is currently the longest task, so we don't randomly delay it.
 665     "samba-ad-dc-ntvfs": {
 666         "dependency": "samba-def-build",
 667         "sequence": [
 668             ("random-sleep", random_sleep(1, 1)),
 669             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 670             ("lcov", LCOV_CMD),
 671             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 672         ],
 673     },
 674
 675     # Test fips compliance
 676     "samba-fips": {
 677         "dependency": "samba-mit-build",
 678         "sequence": [
 679             ("random-sleep", random_sleep(1, 1)),
 680             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 681             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 682             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 683         ],
 684     },
 685
 686     # run the backup/restore testenvs separately as they're fairly standalone
 687     # (and CI seems to max out at ~3 different DCs running at once)
 688     "samba-ad-back1": {
 689         "dependency": "samba-def-build",
 690         "sequence": [
 691             ("random-sleep", random_sleep(300, 900)),
 692             ("test", make_test(include_envs=[
 693             "backupfromdc",
 694             "restoredc",
 695             "renamedc",
 696             ])),
 697             ("lcov", LCOV_CMD),
 698             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 699         ],
 700     },
 701     "samba-ad-back2": {
 702         "dependency": "samba-def-build",
 703         "sequence": [
 704             ("random-sleep", random_sleep(300, 900)),
 705             ("test", make_test(include_envs=[
 706             "backupfromdc",
 707             "offlinebackupdc",
 708             "labdc",
 709             ])),
 710             ("lcov", LCOV_CMD),
 711             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 712         ],
 713     },
 714
 715     "samba-admem-mit": {
 716         "dependency": "samba-mit-build",
 717         "sequence": [
 718             ("random-sleep", random_sleep(1, 1)),
 719             ("test", make_test(include_envs=[
 720             "ad_member",
 721             "ad_member_idmap_rid",
 722             "admem_idmap_autorid",
 723             "ad_member_idmap_ad",
 724             "ad_member_rfc2307",
 725             "ad_member_idmap_nss",
 726             "ad_member_offlogon",
 727             ])),
 728             ("lcov", LCOV_CMD),
 729             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 730         ],
 731     },
 732
 733     "samba-addc-mit-1": {
 734         "dependency": "samba-mit-build",
 735         "sequence": [
 736             ("random-sleep", random_sleep(1, 1)),
 737             ("test", make_test(include_envs=[
 738             "ad_dc",
 739             "ad_dc_smb1",
 740             "ad_dc_smb1_done",
 741             "ad_dc_no_nss",
 742             "ad_dc_no_ntlm",
 743             ])),
 744             ("lcov", LCOV_CMD),
 745             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 746         ],
 747     },
 748
 749     "samba-addc-mit-4a": {
 750         "dependency": "samba-mit-build",
 751         "sequence": [
 752             ("random-sleep", random_sleep(1, 1)),
 753             ("test", make_test(include_envs=[
 754             "fl2000dc",
 755             "ad_member_oneway",
 756             "fl2003dc",
 757             ])),
 758             ("lcov", LCOV_CMD),
 759             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 760         ],
 761     },
 762     "samba-addc-mit-4b": {
 763         "dependency": "samba-mit-build",
 764         "sequence": [
 765             ("random-sleep", random_sleep(1, 1)),
 766             ("test", make_test(include_envs=[
 767             "fl2008dc",
 768             "fl2008r2dc",
 769             ])),
 770             ("lcov", LCOV_CMD),
 771             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 772         ],
 773     },
 774
 775     "samba-test-only": {
 776         "sequence": [
 777             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 778             ("make", "make -j"),
 779             ("test", make_test(TESTS="${TESTS}")),
 780             ("lcov", LCOV_CMD),
 781         ],
 782     },
 783
 784     # Test cross-compile infrastructure
 785     "samba-xc": {
 786         "sequence": [
 787             ("random-sleep", random_sleep(900, 1500)),
 788             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 789             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 790             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 791             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 792             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 793             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 794             ("compare-results", "script/compare_cc_results.py "
 795             "./bin/c4che/default{} "
 796             "./bin-xe/c4che/default{} "
 797             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 798             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 799             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 800             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 801             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 802             " = \"'1234'\"".format(CACHE_SUFFIX)),
 803             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 804             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 805             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 806             " ; test $? -ne 0"),
 807         ],
 808     },
 809
 810     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 811     "samba-o3": {
 812         "sequence": [
 813             ("random-sleep", random_sleep(300, 900)),
 814             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 815             ("make", "make -j"),
 816             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 817             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 818             ("lcov", LCOV_CMD),
 819             ("install", "make install"),
 820             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 821             ("clean", "make clean"),
 822         ],
 823     },
 824
 825     "samba-32bit": {
 826         "sequence": [
 827             ("random-sleep", random_sleep(300, 900)),
 828             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 829             ("make", "make -j"),
 830             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 831             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 832             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 833             ("install", "make install"),
 834             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 835             ("clean", "make clean"),
 836         ],
 837     },
 838
 839     "samba-ctdb": {
 840         "sequence": [
 841             ("random-sleep", random_sleep(900, 1500)),
 842
 843         # make sure we have tdb around:
 844             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 845             ("tdb-make", "cd lib/tdb && make"),
 846             ("tdb-install", "cd lib/tdb && make install"),
 847
 848         # build samba with cluster support (also building ctdb):
 849             ("samba-configure",
 850          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 851          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 852          "./configure.developer ${PREFIX} "
 853          "--with-selftest-prefix=./bin/ab "
 854          "--with-cluster-support "
 855          "--without-ad-dc "
 856          "--bundled-libraries=!tdb"),
 857             ("samba-make", "make"),
 858             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 859             ("samba-install", "make install"),
 860             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 861
 862             ("test", make_test(
 863                 cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
 864                 INJECT_SELFTEST_PREFIX=0,
 865                 include_envs=["clusteredmember"])
 866             ),
 867
 868         # clean up:
 869             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 870             ("clean", "make clean"),
 871             ("ctdb-clean", "cd ./ctdb && make clean"),
 872         ],
 873     },
 874
 875     "samba-libs": {
 876         "sequence": [
 877             ("random-sleep", random_sleep(300, 900)),
 878             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 879             ("talloc-make", "cd lib/talloc && make"),
 880             ("talloc-install", "cd lib/talloc && make install"),
 881
 882             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 883             ("tdb-make", "cd lib/tdb && make"),
 884             ("tdb-install", "cd lib/tdb && make install"),
 885
 886             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 887             ("tevent-make", "cd lib/tevent && make"),
 888             ("tevent-install", "cd lib/tevent && make install"),
 889
 890             ("nondevel-configure", samba_libs_envvars + " ./configure --private-libraries='!ldb' --vendor-suffix=TEST-STRING~5.1.2 ${PREFIX}"),
 891             ("nondevel-make", "make -j"),
 892             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 893             ("nondevel-check", "./bin/smbd --version | grep -F 'TEST-STRING~5.1.2' && exit 0; exit 1"),
 894             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 895             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 896             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 897             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 898             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 899             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 900             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 901             ("nondevel-no-public-nss_winbind",
 902                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 903             ("nondevel-no-public-nss_wins",
 904                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 905             ("nondevel-no-public-libwbclient",
 906                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 907             ("nondevel-no-public-pam_winbind",
 908                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 909             ("nondevel-no-public-winbind_krb5_locator",
 910                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 911             ("nondevel-no-public-async_dns_krb5_locator",
 912                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 913             ("nondevel-install", "make -j install"),
 914             ("nondevel-dist", "make dist"),
 915
 916             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 917             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 918             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 919             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb.so' && exit 1; exit 0"),
 920             ("prefix-public-libldb", "find ${PREFIX_DIR} | grep 'lib/libldb.so' && exit 0; exit 1"),
 921             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 922             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 923             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 924             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 925             ("prefix-no-public-nss_winbind",
 926                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 927             ("prefix-no-public-nss_wins",
 928                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 929             ("prefix-no-public-libwbclient",
 930                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 931             ("prefix-no-public-pam_winbind",
 932                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 933             ("prefix-no-public-winbind_krb5_locator",
 934                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 935                               "service_locator")),
 936             ("prefix-no-public-async_dns_krb5_locator",
 937                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 938                               "service_locator")),
 939
 940             # retry with all modules shared
 941             ("allshared-distclean", "make distclean"),
 942             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 943             ("allshared-make", "make -j"),
 944             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 945             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 946             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 947             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 948             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 949             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 950             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 951             ("allshared-no-public-nss_winbind",
 952                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 953             ("allshared-no-public-nss_wins",
 954                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 955             ("allshared-no-public-libwbclient",
 956                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 957             ("allshared-no-public-pam_winbind",
 958                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 959             ("allshared-no-public-winbind_krb5_locator",
 960                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 961             ("allshared-no-public-async_dns_krb5_locator",
 962                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 963         ],
 964     },
 965
 966     "samba-fuzz": {
 967         "sequence": [
 968         # build the fuzzers (static) via the oss-fuzz script
 969             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 970             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 971         ],
 972     },
 973
 974     # * Test smbd and smbtorture can build semi-static
 975     #
 976     # * Test Samba without python still builds.
 977     #
 978     # When this test fails due to more use of Python, the expectations
 979     # is that the newly failing part of the code should be disabled
 980     # when --disable-python is set (rather than major work being done
 981     # to support this environment).
 982     #
 983     # The target here is for vendors shipping a minimal smbd.
 984     "samba-minimal-smbd": {
 985         "sequence": [
 986             ("random-sleep", random_sleep(300, 900)),
 987
 988         # build with all modules static
 989             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 990             ("allstatic-make", "make -j"),
 991             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 992             ("allstatic-lcov", LCOV_CMD),
 993             ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 994             ("allstatic-def-clean", "make clean"),
 995
 996         # force all libraries as private
 997             ("allprivate-def-distclean", "make distclean"),
 998             ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
 999             ("allprivate-def-make", "make -j"),
1000             # note wrapper libraries need to be public
1001             ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1002             ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
1003             ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
1004             ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1005             ("allprivate-def-lcov", LCOV_CMD),
1006             ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1007             ("allprivate-def-clean", "make clean"),
1008
1009         # force all libraries as private with a non default
1010         # extension and 2 exceptions
1011             ("allprivate-ext-distclean", "make distclean"),
1012             ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
1013             ("allprivate-ext-make", "make -j"),
1014             # note wrapper libraries need to be public
1015             ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1016             ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
1017             ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
1018             ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
1019             ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
1020             ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1021             ("allprivate-ext-lcov", LCOV_CMD),
1022             ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1023             ("allprivate-ext-clean", "make clean"),
1024
1025         # retry with nonshared smbd and smbtorture
1026             ("nonshared-distclean", "make distclean"),
1027             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
1028             ("nonshared-make", "make -j"),
1029             ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1030             ("nonshared-lcov", LCOV_CMD),
1031             ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1032             ("nonshared-clean", "make clean"),
1033         ],
1034     },
1035
1036     "samba-nopython": {
1037         "sequence": [
1038             ("random-sleep", random_sleep(300, 900)),
1039
1040             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1041             ("make", "make -j"),
1042             ("find-python", "script/find_python.sh ${PREFIX}"),
1043             ("test", "make test-nopython"),
1044             ("lcov", LCOV_CMD),
1045             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1046             ("clean", "make clean"),
1047
1048             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1049             ("talloc-make", "cd lib/talloc && make"),
1050             ("talloc-install", "cd lib/talloc && make install"),
1051
1052             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1053             ("tdb-make", "cd lib/tdb && make"),
1054             ("tdb-install", "cd lib/tdb && make install"),
1055
1056             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1057             ("tevent-make", "cd lib/tevent && make"),
1058             ("tevent-install", "cd lib/tevent && make install"),
1059
1060         # retry against installed library packages, but no required modules
1061             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1062             ("libs-make", "make -j"),
1063             ("libs-install", "make install"),
1064             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1065             ("libs-clean", "make clean"),
1066
1067         ],
1068     },
1069
1070     "samba-codecheck": {
1071         "sequence": [
1072             ("run", "script/check-shell-scripts.sh ."),
1073             ("run", "script/codespell.sh ."),
1074         ],
1075     },
1076
1077     "tdb": {
1078         "sequence": [
1079             ("random-sleep", random_sleep(60, 600)),
1080             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1081             ("make", "make"),
1082             ("install", "make install"),
1083             ("test", "make test"),
1084             ("lcov", LCOV_CMD),
1085             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1086             ("distcheck", "make distcheck"),
1087             ("clean", "make clean"),
1088         ],
1089     },
1090
1091     "talloc": {
1092         "sequence": [
1093             ("random-sleep", random_sleep(60, 600)),
1094             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1095             ("make", "make"),
1096             ("install", "make install"),
1097             ("test", "make test"),
1098             ("lcov", LCOV_CMD),
1099             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1100             ("distcheck", "make distcheck"),
1101             ("clean", "make clean"),
1102         ],
1103     },
1104
1105     "replace": {
1106         "sequence": [
1107             ("random-sleep", random_sleep(60, 600)),
1108             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1109             ("make", "make"),
1110             ("install", "make install"),
1111             ("test", "make test"),
1112             ("lcov", LCOV_CMD),
1113             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1114             ("distcheck", "make distcheck"),
1115             ("clean", "make clean"),
1116         ],
1117     },
1118
1119     "tevent": {
1120         "sequence": [
1121             ("random-sleep", random_sleep(60, 600)),
1122             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1123             ("make", "make"),
1124             ("install", "make install"),
1125             ("test", "make test"),
1126             ("lcov", LCOV_CMD),
1127             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1128             ("distcheck", "make distcheck"),
1129             ("clean", "make clean"),
1130         ],
1131     },
1132
1133     "pidl": {
1134         "git-clone-required": True,
1135         "sequence": [
1136             ("random-sleep", random_sleep(60, 600)),
1137             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1138             ("touch", "touch *.yp"),
1139             ("make", "make"),
1140             ("test", "make test"),
1141             ("install", "make install"),
1142             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1143             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1144             ("clean", "make clean"),
1145         ],
1146     },
1147
1148     # these are useful for debugging autobuild
1149     "pass": {
1150         "sequence": [
1151             ("pass", 'echo passing && /bin/true'),
1152         ],
1153     },
1154     "fail": {
1155         "sequence": [
1156             ("fail", 'echo failing && /bin/false'),
1157         ],
1158     },
1159 }
1160
1161 defaulttasks = list(tasks.keys())
1162
1163 defaulttasks.remove("pass")
1164 defaulttasks.remove("fail")
1165
1166 # The build tasks will be brought in by the test tasks as needed
1167 defaulttasks.remove("samba-def-build")
1168 defaulttasks.remove("samba-nt4-build")
1169 defaulttasks.remove("samba-mit-build")
1170 defaulttasks.remove("samba-h5l-build")
1171 defaulttasks.remove("samba-no-opath-build")
1172
1173 # This is not a normal test, but a task to support manually running
1174 # one test under autobuild
1175 defaulttasks.remove("samba-test-only")
1176
1177 # Only built on GitLab CI and not in the default autobuild because it
1178 # uses too much space (4GB of semi-static binaries)
1179 defaulttasks.remove("samba-fuzz")
1180
1181 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1182 # container where a simulated FIPS mode is possible.
1183 defaulttasks.remove("samba-fips")
1184
1185 # The MIT build runs on a current Fedora where an up to date MIT KDC
1186 # is already packaged.  This avoids needing to backport a current MIT
1187 # to the default Ubuntu 18.04, particularly during development, and
1188 # the need to install on the shared sn-devel-184.
1189
1190 defaulttasks.remove("samba-mitkrb5")
1191 defaulttasks.remove("samba-admem-mit")
1192 defaulttasks.remove("samba-addc-mit-1")
1193 defaulttasks.remove("samba-addc-mit-4a")
1194 defaulttasks.remove("samba-addc-mit-4b")
1195
1196 defaulttasks.remove("samba-32bit")
1197
1198 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1199     defaulttasks.remove("samba-o3")
1200
1201
1202 def do_print(msg):
1203     logger.info(msg)
1204     sys.stdout.flush()
1205     sys.stderr.flush()
1206
1207 def do_debug(msg):
1208     logger.debug(msg)
1209     sys.stdout.flush()
1210     sys.stderr.flush()
1211
1212
1213 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1214     if show is None:
1215         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1216     elif show:
1217         do_print("Running: '%s' in '%s'" % (cmd, dir))
1218
1219     if output:
1220         out = check_output([cmd], shell=True, cwd=dir)
1221         return out.decode(encoding='utf-8', errors='backslashreplace')
1222     elif checkfail:
1223         return check_call(cmd, shell=True, cwd=dir)
1224     else:
1225         return call(cmd, shell=True, cwd=dir)
1226
1227 def rmdir_force(dirname, re_raise=True):
1228     try:
1229         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1230                 dirname, dirname, dirname), output=True, show=True)
1231     except CalledProcessError as e:
1232         do_print("Failed: '%s'" % (str(e)))
1233         run_cmd("tree %s" % dirname, output=True, show=True)
1234         if re_raise:
1235             raise
1236         return False
1237     return True
1238
1239 class builder(object):
1240     '''handle build of one directory'''
1241
1242     def __init__(self, name, definition):
1243         self.name = name
1244         self.dir = builddirs.get(name, '.')
1245         self.tag = self.name.replace('/', '_')
1246         self.definition = definition
1247         self.sequence = definition["sequence"]
1248         self.git_clone_required = False
1249         if "git-clone-required" in definition:
1250             self.git_clone_required = bool(definition["git-clone-required"])
1251         self.proc = None
1252         self.done = False
1253         self.next = 0
1254         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1255         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1256         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1257         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1258         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1259         self.stdout = open(self.stdout_path, 'w')
1260         self.stderr = open(self.stderr_path, 'w')
1261         self.stdin  = open("/dev/null", 'r')
1262         self.builder_dir = "%s/%s" % (testbase, self.tag)
1263         self.test_source_dir = self.builder_dir
1264         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1265         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1266         self.prefix = "%s/%s" % (test_prefix, self.tag)
1267         self.consumers = []
1268         self.producer = None
1269
1270         if self.git_clone_required:
1271             assert "dependency" not in definition
1272
1273     def mark_existing(self):
1274         do_debug('%s: Mark as existing dependency' % self.name)
1275         self.next = len(self.sequence)
1276         self.done = True
1277
1278     def add_consumer(self, consumer):
1279         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1280         consumer.producer = self
1281         consumer.test_source_dir = self.test_source_dir
1282         self.consumers.append(consumer)
1283
1284     def start_next(self):
1285         if self.producer is not None:
1286             if not self.producer.done:
1287                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1288                 return
1289
1290         if self.next == 0:
1291             rmdir_force(self.builder_dir)
1292             rmdir_force(self.prefix)
1293             if self.producer is not None:
1294                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1295             elif not self.git_clone_required:
1296                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1297             else:
1298                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1299
1300         if self.next == len(self.sequence):
1301             if not self.done:
1302                 do_print('%s: Completed OK' % self.name)
1303                 self.done = True
1304             if not options.nocleanup and len(self.consumers) == 0:
1305                 do_print('%s: Cleaning up' % self.name)
1306                 rmdir_force(self.builder_dir)
1307                 rmdir_force(self.prefix)
1308             for consumer in self.consumers:
1309                 if consumer.next != 0:
1310                     continue
1311                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1312                 consumer.start_next()
1313             if self.producer is not None:
1314                 self.producer.consumers.remove(self)
1315                 assert self.producer.done
1316                 self.producer.start_next()
1317             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1318             return
1319         (self.stage, self.cmd) = self.sequence[self.next]
1320         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1321                                     get_path(name='platlib',
1322                                              scheme="posix_prefix",
1323                                              vars={"base": self.prefix,
1324                                                    "platbase": self.prefix}))
1325         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1326         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1327         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1328         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1329         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1330         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1331         self.cmd = self.cmd.replace("${NAME}", self.name)
1332         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1333         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1334         self.proc = Popen(self.cmd, shell=True,
1335                           close_fds=True, cwd=self.cwd,
1336                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1337         self.next += 1
1338
1339 def expand_dependencies(n):
1340     deps = list()
1341     if "dependency" in tasks[n]:
1342         depname = tasks[n]["dependency"]
1343         assert depname in tasks
1344         sdeps = expand_dependencies(depname)
1345         assert n not in sdeps
1346         for sdep in sdeps:
1347             deps.append(sdep)
1348         deps.append(depname)
1349     return deps
1350
1351
1352 class buildlist(object):
1353     '''handle build of multiple directories'''
1354
1355     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1356         self.tail_proc = None
1357         self.retry = None
1358         if not tasknames:
1359             if options.restrict_tests:
1360                 tasknames = ["samba-test-only"]
1361             else:
1362                 tasknames = defaulttasks
1363
1364         given_tasknames = tasknames.copy()
1365         implicit_tasknames = []
1366         for n in given_tasknames:
1367             deps = expand_dependencies(n)
1368             for dep in deps:
1369                 if dep in given_tasknames:
1370                     continue
1371                 if dep in implicit_tasknames:
1372                     continue
1373                 implicit_tasknames.append(dep)
1374
1375         tasknames = implicit_tasknames.copy()
1376         tasknames.extend(given_tasknames)
1377         do_debug("given_tasknames: %s" % given_tasknames)
1378         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1379         do_debug("tasknames: %s" % tasknames)
1380         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1381
1382         if options.retry:
1383             rebase_remote = "rebaseon"
1384             retry_task = {
1385                     "git-clone-required": True,
1386                     "sequence": [
1387                             ("retry",
1388                             '''set -e
1389                             git remote add -t %s %s %s
1390                             git fetch %s
1391                             while :; do
1392                               sleep 60
1393                               git describe %s/%s > old_remote_branch.desc
1394                               git fetch %s
1395                               git describe %s/%s > remote_branch.desc
1396                               diff old_remote_branch.desc remote_branch.desc
1397                             done
1398                            ''' % (
1399                                rebase_branch, rebase_remote, rebase_url,
1400                                rebase_remote,
1401                                rebase_remote, rebase_branch,
1402                                rebase_remote,
1403                                rebase_remote, rebase_branch
1404                             ))]}
1405
1406             self.retry = builder('retry', retry_task)
1407             self.need_retry = False
1408
1409         if options.skip_dependencies:
1410             for b in self.tlist:
1411                 if b.name in implicit_tasknames:
1412                     b.mark_existing()
1413
1414         for b in self.tlist:
1415             do_debug("b.name=%s" % b.name)
1416             if "dependency" not in b.definition:
1417                 continue
1418             depname = b.definition["dependency"]
1419             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1420             for p in self.tlist:
1421                 if p.name == depname:
1422                     p.add_consumer(b)
1423
1424     def kill_kids(self):
1425         if self.tail_proc is not None:
1426             self.tail_proc.terminate()
1427             self.tail_proc.wait()
1428             self.tail_proc = None
1429         if self.retry is not None:
1430             self.retry.proc.terminate()
1431             self.retry.proc.wait()
1432             self.retry = None
1433         for b in self.tlist:
1434             if b.proc is not None:
1435                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1436                 b.proc.terminate()
1437                 b.proc.wait()
1438                 b.proc = None
1439
1440     def wait_one(self):
1441         while True:
1442             none_running = True
1443             for b in self.tlist:
1444                 if b.proc is None:
1445                     continue
1446                 none_running = False
1447                 b.status = b.proc.poll()
1448                 if b.status is None:
1449                     continue
1450                 b.proc = None
1451                 return b
1452             if options.retry:
1453                 ret = self.retry.proc.poll()
1454                 if ret is not None:
1455                     self.need_retry = True
1456                     self.retry = None
1457                     return None
1458             if none_running:
1459                 return None
1460             time.sleep(0.1)
1461
1462     def run(self):
1463         for b in self.tlist:
1464             b.start_next()
1465         if options.retry:
1466             self.retry.start_next()
1467         while True:
1468             b = self.wait_one()
1469             if options.retry and self.need_retry:
1470                 self.kill_kids()
1471                 do_print("retry needed")
1472                 return (0, None, None, None, "retry")
1473             if b is None:
1474                 break
1475             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1476                 self.kill_kids()
1477                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1478             b.start_next()
1479         self.kill_kids()
1480         return (0, None, None, None, "All OK")
1481
1482     def write_system_info(self, filename):
1483         with open(filename, 'w') as f:
1484             for cmd in ['uname -a',
1485                         'lsb_release -a',
1486                         'free',
1487                         'mount',
1488                         'cat /proc/cpuinfo',
1489                         'cc --version',
1490                         'df -m .',
1491                         'df -m %s' % testbase]:
1492                 try:
1493                     out = run_cmd(cmd, output=True, checkfail=False)
1494                 except CalledProcessError as e:
1495                     out = "<failed: %s>" % str(e)
1496                 print('### %s' % cmd, file=f)
1497                 print(out, file=f)
1498                 print(file=f)
1499
1500     def tarlogs(self, fname):
1501         with tarfile.open(fname, "w:gz") as tar:
1502             for b in self.tlist:
1503                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1504                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1505             if os.path.exists("autobuild.log"):
1506                 tar.add("autobuild.log")
1507             filename = 'system-info.txt'
1508             self.write_system_info(filename)
1509             tar.add(filename)
1510
1511     def remove_logs(self):
1512         for b in self.tlist:
1513             os.unlink(b.stdout_path)
1514             os.unlink(b.stderr_path)
1515
1516     def start_tail(self):
1517         cmd = ["tail", "-f"]
1518         for b in self.tlist:
1519             cmd.append(b.stdout_path)
1520             cmd.append(b.stderr_path)
1521         self.tail_proc = Popen(cmd, close_fds=True)
1522
1523
1524 def cleanup(do_raise=False):
1525     if options.nocleanup:
1526         return
1527     run_cmd("stat %s || true" % test_tmpdir, show=True)
1528     run_cmd("stat %s" % testbase, show=True)
1529     do_print("Cleaning up %r" % cleanup_list)
1530     for d in cleanup_list:
1531         ok = rmdir_force(d, re_raise=False)
1532         if ok:
1533             continue
1534         if os.path.isdir(d):
1535             do_print("Killing, waiting and retry")
1536             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1537         else:
1538             do_print("Waiting and retry")
1539         time.sleep(1)
1540         rmdir_force(d, re_raise=do_raise)
1541
1542
1543 def daemonize(logfile):
1544     pid = os.fork()
1545     if pid == 0:  # Parent
1546         os.setsid()
1547         pid = os.fork()
1548         if pid != 0:  # Actual daemon
1549             os._exit(0)
1550     else:  # Grandparent
1551         os._exit(0)
1552
1553     import resource      # Resource usage information.
1554     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1555     if maxfd == resource.RLIM_INFINITY:
1556         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1557     for fd in range(0, maxfd):
1558         try:
1559             os.close(fd)
1560         except OSError:
1561             pass
1562     os.open(logfile, os.O_RDWR | os.O_CREAT)
1563     os.dup2(0, 1)
1564     os.dup2(0, 2)
1565
1566
1567 def write_pidfile(fname):
1568     '''write a pid file, cleanup on exit'''
1569     with open(fname, mode='w') as f:
1570         f.write("%u\n" % os.getpid())
1571
1572
1573 def rebase_tree(rebase_url, rebase_branch="master"):
1574     rebase_remote = "rebaseon"
1575     do_print("Rebasing on %s" % rebase_url)
1576     run_cmd("git describe HEAD", show=True, dir=test_master)
1577     run_cmd("git remote add -t %s %s %s" %
1578             (rebase_branch, rebase_remote, rebase_url),
1579             show=True, dir=test_master)
1580     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1581     if options.fix_whitespace:
1582         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1583                 (rebase_remote, rebase_branch),
1584                 show=True, dir=test_master)
1585     else:
1586         run_cmd("git rebase --force-rebase %s/%s" %
1587                 (rebase_remote, rebase_branch),
1588                 show=True, dir=test_master)
1589     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1590                    (rebase_remote, rebase_branch),
1591                    dir=test_master, output=True)
1592     if diff == '':
1593         do_print("No differences between HEAD and %s/%s - exiting" %
1594                  (rebase_remote, rebase_branch))
1595         sys.exit(0)
1596     run_cmd("git describe %s/%s" %
1597             (rebase_remote, rebase_branch),
1598             show=True, dir=test_master)
1599     run_cmd("git describe HEAD", show=True, dir=test_master)
1600     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1601             (rebase_remote, rebase_branch),
1602             show=True, dir=test_master)
1603
1604
1605 def push_to(push_url, push_branch="master"):
1606     push_remote = "pushto"
1607     do_print("Pushing to %s" % push_url)
1608     if options.mark:
1609         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1610         run_cmd("git commit --amend -c HEAD", dir=test_master)
1611         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1612         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1613     run_cmd("git remote add -t %s %s %s" %
1614             (push_branch, push_remote, push_url),
1615             show=True, dir=test_master)
1616     run_cmd("git push %s +HEAD:%s" %
1617             (push_remote, push_branch),
1618             show=True, dir=test_master)
1619
1620
1621 def send_email(subject, text, log_tar):
1622     if options.email is None:
1623         do_print("not sending email because the recipient is not set")
1624         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1625                  (subject, text))
1626         return
1627     outer = MIMEMultipart()
1628     outer['Subject'] = subject
1629     outer['To'] = options.email
1630     outer['From'] = options.email_from
1631     outer['Date'] = email.utils.formatdate(localtime=True)
1632     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1633     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1634     if options.attach_logs:
1635         with open(log_tar, 'rb') as fp:
1636             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1637         # Set the filename parameter
1638         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1639         outer.attach(msg)
1640     content = outer.as_string()
1641     s = smtplib.SMTP(options.email_server)
1642     email_user = os.getenv('SMTP_USERNAME')
1643     email_password = os.getenv('SMTP_PASSWORD')
1644     if email_user is not None:
1645         s.starttls()
1646         s.login(email_user, email_password)
1647
1648     s.sendmail(options.email_from, [options.email], content)
1649     s.set_debuglevel(1)
1650     s.quit()
1651
1652
1653 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1654                   elapsed_time, log_base=None, add_log_tail=True):
1655     '''send an email to options.email about the failure'''
1656     elapsed_minutes = elapsed_time / 60.0
1657     if log_base is None:
1658         log_base = gitroot
1659     text = '''
1660 Dear Developer,
1661
1662 Your autobuild on %s failed after %.1f minutes
1663 when trying to test %s with the following error:
1664
1665    %s
1666
1667 the autobuild has been abandoned. Please fix the error and resubmit.
1668
1669 A summary of the autobuild process is here:
1670
1671   %s/autobuild.log
1672 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1673
1674     if options.restrict_tests:
1675         text += """
1676 The build was restricted to tests matching %s\n""" % options.restrict_tests
1677
1678     if failed_task != 'rebase':
1679         text += '''
1680 You can see logs of the failed task here:
1681
1682   %s/%s.stdout
1683   %s/%s.stderr
1684
1685 or you can get full logs of all tasks in this job here:
1686
1687   %s/logs.tar.gz
1688
1689 The top commit for the tree that was built was:
1690
1691 %s
1692
1693 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1694
1695     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1696     if add_log_tail and os.access(log_stdout, os.R_OK):
1697         f = open(log_stdout, 'r')
1698         lines = f.readlines()
1699         log_tail = "".join(lines[-50:])
1700         num_lines = len(lines)
1701         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1702         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1703             # Also include stderr (compile failures) if < 50 lines of stdout
1704             f = open(log_stderr, 'r')
1705             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1706
1707         text += '''
1708 The last 50 lines of log messages:
1709
1710 %s
1711     ''' % log_tail
1712         f.close()
1713
1714     logs = os.path.join(gitroot, 'logs.tar.gz')
1715     send_email('autobuild[%s] failure on %s for task %s during %s'
1716                % (options.branch, platform.node(), failed_task, failed_stage),
1717                text, logs)
1718
1719
1720 def email_success(elapsed_time, log_base=None):
1721     '''send an email to options.email about a successful build'''
1722     if log_base is None:
1723         log_base = gitroot
1724     text = '''
1725 Dear Developer,
1726
1727 Your autobuild on %s has succeeded after %.1f minutes.
1728
1729 ''' % (platform.node(), elapsed_time / 60.)
1730
1731     if options.restrict_tests:
1732         text += """
1733 The build was restricted to tests matching %s\n""" % options.restrict_tests
1734
1735     if options.keeplogs:
1736         text += '''
1737
1738 you can get full logs of all tasks in this job here:
1739
1740   %s/logs.tar.gz
1741
1742 ''' % log_base
1743
1744     text += '''
1745 The top commit for the tree that was built was:
1746
1747 %s
1748 ''' % top_commit_msg
1749
1750     logs = os.path.join(gitroot, 'logs.tar.gz')
1751     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1752                text, logs)
1753
1754
1755 # get the top commit message, for emails
1756 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1757
1758 try:
1759     if options.skip_dependencies:
1760         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1761     else:
1762         os.makedirs(testbase)
1763 except Exception as reason:
1764     raise Exception("Unable to create %s : %s" % (testbase, reason))
1765 cleanup_list.append(testbase)
1766
1767 if options.daemon:
1768     logfile = os.path.join(testbase, "log")
1769     do_print("Forking into the background, writing progress to %s" % logfile)
1770     daemonize(logfile)
1771
1772 write_pidfile(gitroot + "/autobuild.pid")
1773
1774 start_time = time.time()
1775
1776 while True:
1777     try:
1778         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1779         os.makedirs(test_tmpdir)
1780         # The waf uninstall code removes empty directories all the way
1781         # up the tree.  Creating a file in test_tmpdir stops it from
1782         # being removed.
1783         run_cmd("touch %s" % os.path.join(test_tmpdir,
1784                                           ".directory-is-not-empty"), show=True)
1785         run_cmd("stat %s" % test_tmpdir, show=True)
1786         run_cmd("stat %s" % testbase, show=True)
1787         if options.skip_dependencies:
1788             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1789         else:
1790             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1791     except Exception:
1792         cleanup()
1793         raise
1794
1795     try:
1796         if options.rebase is not None:
1797             rebase_tree(options.rebase, rebase_branch=options.branch)
1798     except Exception:
1799         cleanup_list.append(gitroot + "/autobuild.pid")
1800         cleanup()
1801         elapsed_time = time.time() - start_time
1802         email_failure(-1, 'rebase', 'rebase', 'rebase',
1803                       'rebase on %s failed' % options.branch,
1804                       elapsed_time, log_base=options.log_base)
1805         sys.exit(1)
1806
1807     try:
1808         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1809         if options.tail:
1810             blist.start_tail()
1811         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1812         if status != 0 or errstr != "retry":
1813             break
1814         cleanup(do_raise=True)
1815     except Exception:
1816         cleanup()
1817         raise
1818
1819 cleanup_list.append(gitroot + "/autobuild.pid")
1820
1821 do_print(errstr)
1822
1823 blist.kill_kids()
1824 if options.tail:
1825     do_print("waiting for tail to flush")
1826     time.sleep(1)
1827
1828 elapsed_time = time.time() - start_time
1829 if status == 0:
1830     if options.passcmd is not None:
1831         do_print("Running passcmd: %s" % options.passcmd)
1832         run_cmd(options.passcmd, dir=test_master)
1833     if options.pushto is not None:
1834         push_to(options.pushto, push_branch=options.branch)
1835     if options.keeplogs or options.attach_logs:
1836         blist.tarlogs("logs.tar.gz")
1837         do_print("Logs in logs.tar.gz")
1838     if options.always_email:
1839         email_success(elapsed_time, log_base=options.log_base)
1840     blist.remove_logs()
1841     cleanup()
1842     do_print(errstr)
1843     sys.exit(0)
1844
1845 # something failed, gather a tar of the logs
1846 blist.tarlogs("logs.tar.gz")
1847
1848 if options.email is not None:
1849     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1850                   elapsed_time, log_base=options.log_base)
1851 else:
1852     elapsed_minutes = elapsed_time / 60.0
1853     print('''
1854
1855 ####################################################################
1856
1857 AUTOBUILD FAILURE
1858
1859 Your autobuild[%s] on %s failed after %.1f minutes
1860 when trying to test %s with the following error:
1861
1862    %s
1863
1864 the autobuild has been abandoned. Please fix the error and resubmit.
1865
1866 ####################################################################
1867
1868 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1869
1870 cleanup()
1871 do_print(errstr)
1872 do_print("Logs in logs.tar.gz")
1873 sys.exit(status)