script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 if args:
 142     # If we are only running specific test,
 143     # do not sleep randomly to wait for it to start
 144     def random_sleep(low, high):
 145         return 'sleep 1'
 146 else:
 147     def random_sleep(low, high):
 148         return 'sleep {}'.format(random.randint(low, high))
 149
 150 cleanup_list = []
 151
 152 builddirs = {
 153     "ctdb": "ctdb",
 154     "ldb": "lib/ldb",
 155     "tdb": "lib/tdb",
 156     "talloc": "lib/talloc",
 157     "replace": "lib/replace",
 158     "tevent": "lib/tevent",
 159     "pidl": "pidl",
 160     "docs-xml": "docs-xml"
 161 }
 162
 163 ctdb_configure_params = " --enable-developer ${PREFIX}"
 164 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 165
 166 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 167 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 168 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 169 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 170 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 171 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 172 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 173
 174
 175 def format_option(name, value=None):
 176     """Format option as str list."""
 177     if value is None:  # boolean option
 178         return [name]
 179     if not isinstance(value, list):  # single value option
 180         value = [value]
 181     # repeatable option
 182     return ['{}={}'.format(name, item) for item in value]
 183
 184
 185 def make_test(
 186         cmd='make testonly',
 187         INJECT_SELFTEST_PREFIX=1,
 188         TESTS='',
 189         include_envs=None,
 190         exclude_envs=None):
 191
 192     test_options = []
 193     if include_envs:
 194         test_options = format_option('--include-env', include_envs)
 195     if exclude_envs:
 196         test_options = format_option('--exclude-env', exclude_envs)
 197     if test_options:
 198         # join envs options to original test options
 199         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 200
 201     _options = []
 202
 203     # Allow getting a full CI with
 204     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 205
 206     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 207
 208     if int(FAIL_IMMEDIATELY):
 209         _options.append('FAIL_IMMEDIATELY=1')
 210     if TESTS:
 211         _options.append("TESTS='{}'".format(TESTS))
 212
 213     if INJECT_SELFTEST_PREFIX:
 214         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 215         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 216
 217     return ' '.join([cmd] + _options)
 218
 219
 220 # When updating this list, also update .gitlab-ci.yml to add the job
 221 # and to make it a dependency of 'page' for the coverage report.
 222
 223 tasks = {
 224     "ctdb": {
 225         "sequence": [
 226             ("random-sleep", random_sleep(300, 900)),
 227             ("configure", "./configure " + ctdb_configure_params),
 228             ("make", "make all"),
 229             ("install", "make install"),
 230             ("test", "make autotest"),
 231             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 232             ("clean", "make clean"),
 233         ],
 234     },
 235     "docs-xml": {
 236         "sequence": [
 237             ("random-sleep", random_sleep(300, 900)),
 238             ("autoconf", "autoconf"),
 239             ("configure", "./configure"),
 240             ("make", "make html htmlman"),
 241             ("publish-docs", PUBLISH_DOCS),
 242             ("clean", "make clean"),
 243         ],
 244     },
 245
 246     "samba-def-build": {
 247         "git-clone-required": True,
 248         "sequence": [
 249             ("configure", "./configure.developer" + samba_configure_params),
 250             ("make", "make -j"),
 251             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 252             ("chmod-R-a-w", "chmod -R a-w ."),
 253         ],
 254     },
 255
 256     "samba-mit-build": {
 257         "git-clone-required": True,
 258         "sequence": [
 259             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 260             ("make", "make -j"),
 261             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 262             ("chmod-R-a-w", "chmod -R a-w ."),
 263         ],
 264     },
 265
 266     "samba-nt4-build": {
 267         "git-clone-required": True,
 268         "sequence": [
 269             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 270             ("make", "make -j"),
 271             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 272             ("chmod-R-a-w", "chmod -R a-w ."),
 273         ],
 274     },
 275
 276     "samba-h5l-build": {
 277         "git-clone-required": True,
 278         "sequence": [
 279             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 280             ("make", "make -j"),
 281             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 282             ("chmod-R-a-w", "chmod -R a-w ."),
 283         ],
 284     },
 285
 286     "samba-without-smb1-build": {
 287         "git-clone-required": True,
 288         "sequence": [
 289             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 290             ("make", "make -j"),
 291             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 292             ("chmod-R-a-w", "chmod -R a-w ."),
 293         ],
 294     },
 295
 296     "samba-no-opath-build": {
 297         "git-clone-required": True,
 298         "sequence": [
 299             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 300             ("make", "make -j"),
 301             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 302             ("chmod-R-a-w", "chmod -R a-w ."),
 303         ],
 304     },
 305
 306     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 307     "samba": {
 308         "sequence": [
 309             ("random-sleep", random_sleep(300, 900)),
 310             ("configure", "./configure.developer" + samba_configure_params),
 311             ("make", "make -j"),
 312             ("test", make_test(exclude_envs=[
 313             "none",
 314             "nt4_dc",
 315             "nt4_dc_smb1",
 316             "nt4_dc_smb1_done",
 317             "nt4_dc_schannel",
 318             "nt4_member",
 319             "ad_dc",
 320             "ad_dc_smb1",
 321             "ad_dc_smb1_done",
 322             "ad_dc_backup",
 323             "ad_dc_ntvfs",
 324             "ad_dc_default",
 325             "ad_dc_default_smb1",
 326             "ad_dc_slowtests",
 327             "ad_dc_no_nss",
 328             "ad_dc_no_ntlm",
 329             "fl2003dc",
 330             "fl2008dc",
 331             "fl2008r2dc",
 332             "ad_member",
 333             "ad_member_idmap_rid",
 334             "admem_idmap_autorid",
 335             "ad_member_idmap_ad",
 336             "ad_member_rfc2307",
 337             "ad_member_oneway",
 338             "chgdcpass",
 339             "vampire_2000_dc",
 340             "fl2000dc",
 341             "fileserver",
 342             "fileserver_smb1",
 343             "fileserver_smb1_done",
 344             "maptoguest",
 345             "simpleserver",
 346             "backupfromdc",
 347             "restoredc",
 348             "renamedc",
 349             "offlinebackupdc",
 350             "labdc",
 351             "preforkrestartdc",
 352             "proclimitdc",
 353             "promoted_dc",
 354             "vampire_dc",
 355             "rodc",
 356             "ad_dc_default",
 357             "ad_dc_default_smb1",
 358             "ad_dc_default_smb1_done",
 359             "ad_dc_slowtests",
 360             "schema_pair_dc",
 361             "schema_dc",
 362             "clusteredmember",
 363             "ad_dc_fips",
 364             "ad_member_fips",
 365             ])),
 366             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 367             ("lcov", LCOV_CMD),
 368             ("install", "make install"),
 369             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 370             ("clean", "make clean"),
 371         ],
 372     },
 373
 374     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 375     "samba-mitkrb5": {
 376         "sequence": [
 377             ("random-sleep", random_sleep(300, 900)),
 378             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 379             ("make", "make -j"),
 380             ("test", make_test(exclude_envs=[
 381             "none",
 382             "nt4_dc",
 383             "nt4_dc_smb1",
 384             "nt4_dc_smb1_done",
 385             "nt4_dc_schannel",
 386             "nt4_member",
 387             "ad_dc",
 388             "ad_dc_smb1",
 389             "ad_dc_smb1_done",
 390             "ad_dc_backup",
 391             "ad_dc_ntvfs",
 392             "ad_dc_default",
 393             "ad_dc_default_smb1",
 394             "ad_dc_default_smb1_done",
 395             "ad_dc_slowtests",
 396             "ad_dc_no_nss",
 397             "ad_dc_no_ntlm",
 398             "fl2003dc",
 399             "fl2008dc",
 400             "fl2008r2dc",
 401             "ad_member",
 402             "ad_member_idmap_rid",
 403             "admem_idmap_autorid",
 404             "ad_member_idmap_ad",
 405             "ad_member_rfc2307",
 406             "ad_member_oneway",
 407             "chgdcpass",
 408             "vampire_2000_dc",
 409             "fl2000dc",
 410             "fileserver",
 411             "fileserver_smb1",
 412             "fileserver_smb1_done",
 413             "maptoguest",
 414             "simpleserver",
 415             "backupfromdc",
 416             "restoredc",
 417             "renamedc",
 418             "offlinebackupdc",
 419             "labdc",
 420             "preforkrestartdc",
 421             "proclimitdc",
 422             "promoted_dc",
 423             "vampire_dc",
 424             "rodc",
 425             "ad_dc_default",
 426             "ad_dc_default_smb1",
 427             "ad_dc_default_smb1_done",
 428             "ad_dc_slowtests",
 429             "schema_pair_dc",
 430             "schema_dc",
 431             "clusteredmember",
 432             "ad_dc_fips",
 433             "ad_member_fips",
 434             ])),
 435             ("lcov", LCOV_CMD),
 436             ("install", "make install"),
 437             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 438             ("clean", "make clean"),
 439         ],
 440     },
 441
 442     "samba-nt4": {
 443         "dependency": "samba-nt4-build",
 444         "sequence": [
 445             ("random-sleep", random_sleep(300, 900)),
 446             ("test", make_test(include_envs=[
 447             "nt4_dc",
 448             "nt4_dc_smb1",
 449             "nt4_dc_smb1_done",
 450             "nt4_dc_schannel",
 451             "nt4_member",
 452             "simpleserver",
 453             ])),
 454             ("lcov", LCOV_CMD),
 455             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 456         ],
 457     },
 458
 459     "samba-fileserver": {
 460         "dependency": "samba-h5l-build",
 461         "sequence": [
 462             ("random-sleep", random_sleep(300, 900)),
 463             ("test", make_test(include_envs=[
 464             "fileserver",
 465             "fileserver_smb1",
 466             "fileserver_smb1_done",
 467             "maptoguest",
 468             "ktest", # ktest is also tested in samba-ktest-mit samba
 469                      # and samba-mitkrb5 but is tested here against
 470                      # a system Heimdal
 471             ])),
 472             ("lcov", LCOV_CMD),
 473             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 474         ],
 475     },
 476
 477     "samba-fileserver-without-smb1": {
 478         "dependency": "samba-without-smb1-build",
 479         "sequence": [
 480             ("random-sleep", random_sleep(300, 900)),
 481             ("test", make_test(include_envs=["fileserver"])),
 482             ("lcov", LCOV_CMD),
 483             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 484         ],
 485     },
 486
 487     # This is a full build without the AD DC so we test the build with
 488     # MIT Kerberos from the current system.  Runtime behaviour is
 489     # confirmed via the ktest (static ccache and keytab) environment
 490
 491     # This environment also used to confirm we can still build with --with-libunwind
 492     "samba-ktest-mit": {
 493         "sequence": [
 494             ("random-sleep", random_sleep(300, 900)),
 495             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 496             ("make", "make -j"),
 497             ("test", make_test(include_envs=[
 498             "ktest", # ktest is also tested in fileserver, samba and
 499                      # samba-mitkrb5 but is tested here against a
 500                      # system MIT krb5
 501             ])),
 502             ("lcov", LCOV_CMD),
 503             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 504         ],
 505     },
 506
 507     "samba-admem": {
 508         "dependency": "samba-def-build",
 509         "sequence": [
 510             ("random-sleep", random_sleep(300, 900)),
 511             ("test", make_test(include_envs=[
 512             "ad_member",
 513             "ad_member_idmap_rid",
 514             "admem_idmap_autorid",
 515             "ad_member_idmap_ad",
 516             "ad_member_rfc2307",
 517             "ad_member_offlogon",
 518             ])),
 519             ("lcov", LCOV_CMD),
 520             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 521         ],
 522     },
 523
 524     "samba-no-opath1": {
 525         "dependency": "samba-no-opath-build",
 526         "sequence": [
 527             ("random-sleep", random_sleep(300, 900)),
 528             ("test", make_test(
 529                 cmd="make testonly DISABLE_OPATH=1",
 530                 include_envs=[
 531                 "nt4_dc",
 532                 "nt4_dc_smb1",
 533                 "nt4_dc_smb1_done",
 534                 "nt4_dc_schannel",
 535                 "nt4_member",
 536                 "simpleserver",
 537                 ])),
 538             ("lcov", LCOV_CMD),
 539             ("check-clean-tree", "script/clean-source-tree.sh"),
 540         ],
 541     },
 542
 543     "samba-no-opath2": {
 544         "dependency": "samba-no-opath-build",
 545         "sequence": [
 546             ("random-sleep", random_sleep(300, 900)),
 547             ("test", make_test(
 548                 cmd="make testonly DISABLE_OPATH=1",
 549                 include_envs=[
 550                 "fileserver",
 551                 "fileserver_smb1",
 552                 "fileserver_smb1_done",
 553                 ])),
 554             ("lcov", LCOV_CMD),
 555             ("check-clean-tree", "script/clean-source-tree.sh"),
 556         ],
 557     },
 558
 559     "samba-ad-dc-1": {
 560         "dependency": "samba-def-build",
 561         "sequence": [
 562             ("random-sleep", random_sleep(1, 1)),
 563             ("test", make_test(include_envs=[
 564             "ad_dc",
 565             "ad_dc_smb1",
 566             "ad_dc_smb1_done",
 567             "ad_dc_no_nss",
 568             "ad_dc_no_ntlm",
 569             ])),
 570             ("lcov", LCOV_CMD),
 571             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 572         ],
 573     },
 574
 575     "samba-ad-dc-2": {
 576         "dependency": "samba-def-build",
 577         "sequence": [
 578             ("random-sleep", random_sleep(1, 1)),
 579             ("test", make_test(include_envs=[
 580             "vampire_dc",
 581             "vampire_2000_dc",
 582             "rodc",
 583             ])),
 584             ("lcov", LCOV_CMD),
 585             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 586         ],
 587     },
 588
 589     "samba-ad-dc-3": {
 590         "dependency": "samba-def-build",
 591         "sequence": [
 592             ("random-sleep", random_sleep(1, 1)),
 593             ("test", make_test(include_envs=[
 594             "promoted_dc",
 595             "chgdcpass",
 596             "preforkrestartdc",
 597             "proclimitdc",
 598             ])),
 599             ("lcov", LCOV_CMD),
 600             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 601         ],
 602     },
 603
 604     "samba-ad-dc-4a": {
 605         "dependency": "samba-def-build",
 606         "sequence": [
 607             ("random-sleep", random_sleep(1, 1)),
 608             ("test", make_test(include_envs=[
 609             "fl2000dc",
 610             "ad_member_oneway",
 611             "fl2003dc",
 612             ])),
 613             ("lcov", LCOV_CMD),
 614             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 615         ],
 616     },
 617     "samba-ad-dc-4b": {
 618         "dependency": "samba-def-build",
 619         "sequence": [
 620             ("random-sleep", random_sleep(1, 1)),
 621             ("test", make_test(include_envs=[
 622             "fl2008dc",
 623             "fl2008r2dc",
 624             ])),
 625             ("lcov", LCOV_CMD),
 626             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 627         ],
 628     },
 629
 630     "samba-ad-dc-5": {
 631         "dependency": "samba-def-build",
 632         "sequence": [
 633             ("random-sleep", random_sleep(1, 1)),
 634             ("test", make_test(include_envs=[
 635             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 636             ("lcov", LCOV_CMD),
 637             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 638         ],
 639     },
 640
 641     "samba-ad-dc-6": {
 642         "dependency": "samba-def-build",
 643         "sequence": [
 644             ("random-sleep", random_sleep(1, 1)),
 645             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 646             ("lcov", LCOV_CMD),
 647             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 648         ],
 649     },
 650
 651     "samba-schemaupgrade": {
 652         "dependency": "samba-def-build",
 653         "sequence": [
 654             ("random-sleep", random_sleep(1, 1)),
 655             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 656             ("lcov", LCOV_CMD),
 657             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 658         ],
 659     },
 660
 661     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 662     # This is currently the longest task, so we don't randomly delay it.
 663     "samba-ad-dc-ntvfs": {
 664         "dependency": "samba-def-build",
 665         "sequence": [
 666             ("random-sleep", random_sleep(1, 1)),
 667             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 668             ("lcov", LCOV_CMD),
 669             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 670         ],
 671     },
 672
 673     # Test fips compliance
 674     "samba-fips": {
 675         "dependency": "samba-mit-build",
 676         "sequence": [
 677             ("random-sleep", random_sleep(1, 1)),
 678             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 679             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 680             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 681         ],
 682     },
 683
 684     # run the backup/restore testenvs separately as they're fairly standalone
 685     # (and CI seems to max out at ~3 different DCs running at once)
 686     "samba-ad-back1": {
 687         "dependency": "samba-def-build",
 688         "sequence": [
 689             ("random-sleep", random_sleep(300, 900)),
 690             ("test", make_test(include_envs=[
 691             "backupfromdc",
 692             "restoredc",
 693             "renamedc",
 694             ])),
 695             ("lcov", LCOV_CMD),
 696             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 697         ],
 698     },
 699     "samba-ad-back2": {
 700         "dependency": "samba-def-build",
 701         "sequence": [
 702             ("random-sleep", random_sleep(300, 900)),
 703             ("test", make_test(include_envs=[
 704             "backupfromdc",
 705             "offlinebackupdc",
 706             "labdc",
 707             ])),
 708             ("lcov", LCOV_CMD),
 709             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 710         ],
 711     },
 712
 713     "samba-admem-mit": {
 714         "dependency": "samba-mit-build",
 715         "sequence": [
 716             ("random-sleep", random_sleep(1, 1)),
 717             ("test", make_test(include_envs=[
 718             "ad_member",
 719             "ad_member_idmap_rid",
 720             "admem_idmap_autorid",
 721             "ad_member_idmap_ad",
 722             "ad_member_rfc2307",
 723             "ad_member_offlogon",
 724             ])),
 725             ("lcov", LCOV_CMD),
 726             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 727         ],
 728     },
 729
 730     "samba-addc-mit-1": {
 731         "dependency": "samba-mit-build",
 732         "sequence": [
 733             ("random-sleep", random_sleep(1, 1)),
 734             ("test", make_test(include_envs=[
 735             "ad_dc",
 736             "ad_dc_smb1",
 737             "ad_dc_smb1_done",
 738             "ad_dc_no_nss",
 739             "ad_dc_no_ntlm",
 740             ])),
 741             ("lcov", LCOV_CMD),
 742             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 743         ],
 744     },
 745
 746     "samba-addc-mit-4a": {
 747         "dependency": "samba-mit-build",
 748         "sequence": [
 749             ("random-sleep", random_sleep(1, 1)),
 750             ("test", make_test(include_envs=[
 751             "fl2000dc",
 752             "ad_member_oneway",
 753             "fl2003dc",
 754             ])),
 755             ("lcov", LCOV_CMD),
 756             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 757         ],
 758     },
 759     "samba-addc-mit-4b": {
 760         "dependency": "samba-mit-build",
 761         "sequence": [
 762             ("random-sleep", random_sleep(1, 1)),
 763             ("test", make_test(include_envs=[
 764             "fl2008dc",
 765             "fl2008r2dc",
 766             ])),
 767             ("lcov", LCOV_CMD),
 768             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 769         ],
 770     },
 771
 772     "samba-test-only": {
 773         "sequence": [
 774             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 775             ("make", "make -j"),
 776             ("test", make_test(TESTS="${TESTS}")),
 777             ("lcov", LCOV_CMD),
 778         ],
 779     },
 780
 781     # Test cross-compile infrastructure
 782     "samba-xc": {
 783         "sequence": [
 784             ("random-sleep", random_sleep(900, 1500)),
 785             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 786             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 787             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 788             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 789             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 790             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 791             ("compare-results", "script/compare_cc_results.py "
 792             "./bin/c4che/default{} "
 793             "./bin-xe/c4che/default{} "
 794             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 795             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 796             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 797             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 798             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 799             " = \"'1234'\"".format(CACHE_SUFFIX)),
 800             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 801             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 802             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 803             " ; test $? -ne 0"),
 804         ],
 805     },
 806
 807     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 808     "samba-o3": {
 809         "sequence": [
 810             ("random-sleep", random_sleep(300, 900)),
 811             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 812             ("make", "make -j"),
 813             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 814             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 815             ("lcov", LCOV_CMD),
 816             ("install", "make install"),
 817             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 818             ("clean", "make clean"),
 819         ],
 820     },
 821
 822     "samba-32bit": {
 823         "sequence": [
 824             ("random-sleep", random_sleep(300, 900)),
 825             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 826             ("make", "make -j"),
 827             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 828             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 829             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 830             ("install", "make install"),
 831             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 832             ("clean", "make clean"),
 833         ],
 834     },
 835
 836     "samba-ctdb": {
 837         "sequence": [
 838             ("random-sleep", random_sleep(900, 1500)),
 839
 840         # make sure we have tdb around:
 841             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 842             ("tdb-make", "cd lib/tdb && make"),
 843             ("tdb-install", "cd lib/tdb && make install"),
 844
 845         # build samba with cluster support (also building ctdb):
 846             ("samba-configure",
 847          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 848          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 849          "./configure.developer ${PREFIX} "
 850          "--with-selftest-prefix=./bin/ab "
 851          "--with-cluster-support "
 852          "--without-ad-dc "
 853          "--bundled-libraries=!tdb"),
 854             ("samba-make", "make"),
 855             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 856             ("samba-install", "make install"),
 857             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 858
 859             ("test", make_test(
 860                 cmd='make test',
 861                 INJECT_SELFTEST_PREFIX=0,
 862                 include_envs=["clusteredmember"])
 863             ),
 864
 865         # clean up:
 866             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 867             ("clean", "make clean"),
 868             ("ctdb-clean", "cd ./ctdb && make clean"),
 869         ],
 870     },
 871
 872     "samba-libs": {
 873         "sequence": [
 874             ("random-sleep", random_sleep(300, 900)),
 875             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 876             ("talloc-make", "cd lib/talloc && make"),
 877             ("talloc-install", "cd lib/talloc && make install"),
 878
 879             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 880             ("tdb-make", "cd lib/tdb && make"),
 881             ("tdb-install", "cd lib/tdb && make install"),
 882
 883             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 884             ("tevent-make", "cd lib/tevent && make"),
 885             ("tevent-install", "cd lib/tevent && make install"),
 886
 887             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 888             ("ldb-make", "cd lib/ldb && make"),
 889             ("ldb-install", "cd lib/ldb && make install"),
 890
 891             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 892             ("nondevel-make", "make -j"),
 893             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 894             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 895             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 896             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 897             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 898             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 899             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 900             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 901             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 902             ("nondevel-no-public-nss_winbind",
 903                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 904             ("nondevel-no-public-nss_wins",
 905                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 906             ("nondevel-no-public-libwbclient",
 907                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 908             ("nondevel-no-public-pam_winbind",
 909                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 910             ("nondevel-no-public-winbind_krb5_locator",
 911                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 912             ("nondevel-no-public-async_dns_krb5_locator",
 913                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 914             ("nondevel-install", "make -j install"),
 915             ("nondevel-dist", "make dist"),
 916
 917             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 918             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 919             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 920             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 921             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 922             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 923             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 924             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 925             ("prefix-no-public-nss_winbind",
 926                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 927             ("prefix-no-public-nss_wins",
 928                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 929             ("prefix-no-public-libwbclient",
 930                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 931             ("prefix-no-public-pam_winbind",
 932                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 933             ("prefix-no-public-winbind_krb5_locator",
 934                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 935                               "service_locator")),
 936             ("prefix-no-public-async_dns_krb5_locator",
 937                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 938                               "service_locator")),
 939
 940             # retry with all modules shared
 941             ("allshared-distclean", "make distclean"),
 942             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 943             ("allshared-make", "make -j"),
 944             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 945             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 946             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 947             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 948             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 949             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 950             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 951             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 952             ("allshared-no-public-nss_winbind",
 953                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 954             ("allshared-no-public-nss_wins",
 955                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 956             ("allshared-no-public-libwbclient",
 957                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 958             ("allshared-no-public-pam_winbind",
 959                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 960             ("allshared-no-public-winbind_krb5_locator",
 961                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 962             ("allshared-no-public-async_dns_krb5_locator",
 963                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 964         ],
 965     },
 966
 967     "samba-fuzz": {
 968         "sequence": [
 969         # build the fuzzers (static) via the oss-fuzz script
 970             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 971             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 972         ],
 973     },
 974
 975     # * Test smbd and smbtorture can build semi-static
 976     #
 977     # * Test Samba without python still builds.
 978     #
 979     # When this test fails due to more use of Python, the expectations
 980     # is that the newly failing part of the code should be disabled
 981     # when --disable-python is set (rather than major work being done
 982     # to support this environment).
 983     #
 984     # The target here is for vendors shipping a minimal smbd.
 985     "samba-minimal-smbd": {
 986         "sequence": [
 987             ("random-sleep", random_sleep(300, 900)),
 988
 989         # build with all modules static
 990             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 991             ("allstatic-make", "make -j"),
 992             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 993             ("allstatic-lcov", LCOV_CMD),
 994
 995         # retry with nonshared smbd and smbtorture
 996             ("nonshared-distclean", "make distclean"),
 997             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 998             ("nonshared-make", "make -j"),
 999             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1000             # TODO ("nonshared-lcov", LCOV_CMD),
1001
1002             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1003             ("clean", "make clean"),
1004         ],
1005     },
1006
1007     "samba-nopython": {
1008         "sequence": [
1009             ("random-sleep", random_sleep(300, 900)),
1010
1011             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1012             ("make", "make -j"),
1013             ("find-python", "script/find_python.sh ${PREFIX}"),
1014             ("test", "make test-nopython"),
1015             ("lcov", LCOV_CMD),
1016             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1017             ("clean", "make clean"),
1018
1019             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1020             ("talloc-make", "cd lib/talloc && make"),
1021             ("talloc-install", "cd lib/talloc && make install"),
1022
1023             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1024             ("tdb-make", "cd lib/tdb && make"),
1025             ("tdb-install", "cd lib/tdb && make install"),
1026
1027             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1028             ("tevent-make", "cd lib/tevent && make"),
1029             ("tevent-install", "cd lib/tevent && make install"),
1030
1031             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1032             ("ldb-make", "cd lib/ldb && make"),
1033             ("ldb-install", "cd lib/ldb && make install"),
1034
1035         # retry against installed library packages, but no required modules
1036             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1037             ("libs-make", "make -j"),
1038             ("libs-install", "make install"),
1039             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1040             ("libs-clean", "make clean"),
1041
1042         ],
1043     },
1044
1045     "samba-shellcheck": {
1046         "sequence": [
1047             ("run", "script/check-shell-scripts.sh ."),
1048         ],
1049     },
1050
1051     "ldb": {
1052         "sequence": [
1053             ("random-sleep", random_sleep(60, 600)),
1054             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1055             ("make", "make"),
1056             ("install", "make install"),
1057             ("test", "make test"),
1058             ("lcov", LCOV_CMD),
1059             ("clean", "make clean"),
1060             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1061             ("make-no-lmdb", "make"),
1062             ("test-no-lmdb", "make test"),
1063             ("lcov-no-lmdb", LCOV_CMD),
1064             ("install-no-lmdb", "make install"),
1065             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1066             ("distcheck", "make distcheck"),
1067             ("clean", "make clean"),
1068         ],
1069     },
1070
1071     "tdb": {
1072         "sequence": [
1073             ("random-sleep", random_sleep(60, 600)),
1074             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1075             ("make", "make"),
1076             ("install", "make install"),
1077             ("test", "make test"),
1078             ("lcov", LCOV_CMD),
1079             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1080             ("distcheck", "make distcheck"),
1081             ("clean", "make clean"),
1082         ],
1083     },
1084
1085     "talloc": {
1086         "sequence": [
1087             ("random-sleep", random_sleep(60, 600)),
1088             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1089             ("make", "make"),
1090             ("install", "make install"),
1091             ("test", "make test"),
1092             ("lcov", LCOV_CMD),
1093             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1094             ("distcheck", "make distcheck"),
1095             ("clean", "make clean"),
1096         ],
1097     },
1098
1099     "replace": {
1100         "sequence": [
1101             ("random-sleep", random_sleep(60, 600)),
1102             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1103             ("make", "make"),
1104             ("install", "make install"),
1105             ("test", "make test"),
1106             ("lcov", LCOV_CMD),
1107             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1108             ("distcheck", "make distcheck"),
1109             ("clean", "make clean"),
1110         ],
1111     },
1112
1113     "tevent": {
1114         "sequence": [
1115             ("random-sleep", random_sleep(60, 600)),
1116             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1117             ("make", "make"),
1118             ("install", "make install"),
1119             ("test", "make test"),
1120             ("lcov", LCOV_CMD),
1121             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1122             ("distcheck", "make distcheck"),
1123             ("clean", "make clean"),
1124         ],
1125     },
1126
1127     "pidl": {
1128         "git-clone-required": True,
1129         "sequence": [
1130             ("random-sleep", random_sleep(60, 600)),
1131             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1132             ("touch", "touch *.yp"),
1133             ("make", "make"),
1134             ("test", "make test"),
1135             ("install", "make install"),
1136             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1137             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1138             ("clean", "make clean"),
1139         ],
1140     },
1141
1142     # these are useful for debugging autobuild
1143     "pass": {
1144         "sequence": [
1145             ("pass", 'echo passing && /bin/true'),
1146         ],
1147     },
1148     "fail": {
1149         "sequence": [
1150             ("fail", 'echo failing && /bin/false'),
1151         ],
1152     },
1153 }
1154
1155 defaulttasks = list(tasks.keys())
1156
1157 defaulttasks.remove("pass")
1158 defaulttasks.remove("fail")
1159
1160 # The build tasks will be brought in by the test tasks as needed
1161 defaulttasks.remove("samba-def-build")
1162 defaulttasks.remove("samba-nt4-build")
1163 defaulttasks.remove("samba-mit-build")
1164 defaulttasks.remove("samba-h5l-build")
1165 defaulttasks.remove("samba-no-opath-build")
1166
1167 # This is not a normal test, but a task to support manually running
1168 # one test under autobuild
1169 defaulttasks.remove("samba-test-only")
1170
1171 # Only built on GitLab CI and not in the default autobuild because it
1172 # uses too much space (4GB of semi-static binaries)
1173 defaulttasks.remove("samba-fuzz")
1174
1175 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1176 # container where a simulated FIPS mode is possible.
1177 defaulttasks.remove("samba-fips")
1178
1179 # The MIT build runs on a current Fedora where an up to date MIT KDC
1180 # is already packaged.  This avoids needing to backport a current MIT
1181 # to the default Ubuntu 18.04, particularly during development, and
1182 # the need to install on the shared sn-devel-184.
1183
1184 defaulttasks.remove("samba-mitkrb5")
1185 defaulttasks.remove("samba-admem-mit")
1186 defaulttasks.remove("samba-addc-mit-1")
1187 defaulttasks.remove("samba-addc-mit-4a")
1188 defaulttasks.remove("samba-addc-mit-4b")
1189
1190 defaulttasks.remove("samba-32bit")
1191
1192 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1193     defaulttasks.remove("samba-o3")
1194
1195
1196 def do_print(msg):
1197     logger.info(msg)
1198     sys.stdout.flush()
1199     sys.stderr.flush()
1200
1201 def do_debug(msg):
1202     logger.debug(msg)
1203     sys.stdout.flush()
1204     sys.stderr.flush()
1205
1206
1207 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1208     if show is None:
1209         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1210     elif show:
1211         do_print("Running: '%s' in '%s'" % (cmd, dir))
1212
1213     if output:
1214         out = check_output([cmd], shell=True, cwd=dir)
1215         return out.decode(encoding='utf-8', errors='backslashreplace')
1216     elif checkfail:
1217         return check_call(cmd, shell=True, cwd=dir)
1218     else:
1219         return call(cmd, shell=True, cwd=dir)
1220
1221 def rmdir_force(dirname, re_raise=True):
1222     try:
1223         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1224                 dirname, dirname, dirname), output=True, show=True)
1225     except CalledProcessError as e:
1226         do_print("Failed: '%s'" % (str(e)))
1227         run_cmd("tree %s" % dirname, output=True, show=True)
1228         if re_raise:
1229             raise
1230         return False
1231     return True
1232
1233 class builder(object):
1234     '''handle build of one directory'''
1235
1236     def __init__(self, name, definition):
1237         self.name = name
1238         self.dir = builddirs.get(name, '.')
1239         self.tag = self.name.replace('/', '_')
1240         self.definition = definition
1241         self.sequence = definition["sequence"]
1242         self.git_clone_required = False
1243         if "git-clone-required" in definition:
1244             self.git_clone_required = bool(definition["git-clone-required"])
1245         self.proc = None
1246         self.done = False
1247         self.next = 0
1248         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1249         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1250         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1251         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1252         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1253         self.stdout = open(self.stdout_path, 'w')
1254         self.stderr = open(self.stderr_path, 'w')
1255         self.stdin  = open("/dev/null", 'r')
1256         self.builder_dir = "%s/%s" % (testbase, self.tag)
1257         self.test_source_dir = self.builder_dir
1258         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1259         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1260         self.prefix = "%s/%s" % (test_prefix, self.tag)
1261         self.consumers = []
1262         self.producer = None
1263
1264         if self.git_clone_required:
1265             assert "dependency" not in definition
1266
1267     def mark_existing(self):
1268         do_debug('%s: Mark as existing dependency' % self.name)
1269         self.next = len(self.sequence)
1270         self.done = True
1271
1272     def add_consumer(self, consumer):
1273         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1274         consumer.producer = self
1275         consumer.test_source_dir = self.test_source_dir
1276         self.consumers.append(consumer)
1277
1278     def start_next(self):
1279         if self.producer is not None:
1280             if not self.producer.done:
1281                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1282                 return
1283
1284         if self.next == 0:
1285             rmdir_force(self.builder_dir)
1286             rmdir_force(self.prefix)
1287             if self.producer is not None:
1288                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1289             elif not self.git_clone_required:
1290                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1291             else:
1292                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1293
1294         if self.next == len(self.sequence):
1295             if not self.done:
1296                 do_print('%s: Completed OK' % self.name)
1297                 self.done = True
1298             if not options.nocleanup and len(self.consumers) == 0:
1299                 do_print('%s: Cleaning up' % self.name)
1300                 rmdir_force(self.builder_dir)
1301                 rmdir_force(self.prefix)
1302             for consumer in self.consumers:
1303                 if consumer.next != 0:
1304                     continue
1305                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1306                 consumer.start_next()
1307             if self.producer is not None:
1308                 self.producer.consumers.remove(self)
1309                 assert self.producer.done
1310                 self.producer.start_next()
1311             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1312             return
1313         (self.stage, self.cmd) = self.sequence[self.next]
1314         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1315                                     get_path(name='platlib',
1316                                              scheme="posix_prefix",
1317                                              vars={"base": self.prefix,
1318                                                    "platbase": self.prefix}))
1319         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1320         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1321         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1322         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1323         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1324         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1325         self.cmd = self.cmd.replace("${NAME}", self.name)
1326         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1327         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1328         self.proc = Popen(self.cmd, shell=True,
1329                           close_fds=True, cwd=self.cwd,
1330                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1331         self.next += 1
1332
1333 def expand_dependencies(n):
1334     deps = list()
1335     if "dependency" in tasks[n]:
1336         depname = tasks[n]["dependency"]
1337         assert depname in tasks
1338         sdeps = expand_dependencies(depname)
1339         assert n not in sdeps
1340         for sdep in sdeps:
1341             deps.append(sdep)
1342         deps.append(depname)
1343     return deps
1344
1345
1346 class buildlist(object):
1347     '''handle build of multiple directories'''
1348
1349     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1350         self.tail_proc = None
1351         self.retry = None
1352         if not tasknames:
1353             if options.restrict_tests:
1354                 tasknames = ["samba-test-only"]
1355             else:
1356                 tasknames = defaulttasks
1357
1358         given_tasknames = tasknames.copy()
1359         implicit_tasknames = []
1360         for n in given_tasknames:
1361             deps = expand_dependencies(n)
1362             for dep in deps:
1363                 if dep in given_tasknames:
1364                     continue
1365                 if dep in implicit_tasknames:
1366                     continue
1367                 implicit_tasknames.append(dep)
1368
1369         tasknames = implicit_tasknames.copy()
1370         tasknames.extend(given_tasknames)
1371         do_debug("given_tasknames: %s" % given_tasknames)
1372         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1373         do_debug("tasknames: %s" % tasknames)
1374         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1375
1376         if options.retry:
1377             rebase_remote = "rebaseon"
1378             retry_task = {
1379                     "git-clone-required": True,
1380                     "sequence": [
1381                             ("retry",
1382                             '''set -e
1383                             git remote add -t %s %s %s
1384                             git fetch %s
1385                             while :; do
1386                               sleep 60
1387                               git describe %s/%s > old_remote_branch.desc
1388                               git fetch %s
1389                               git describe %s/%s > remote_branch.desc
1390                               diff old_remote_branch.desc remote_branch.desc
1391                             done
1392                            ''' % (
1393                                rebase_branch, rebase_remote, rebase_url,
1394                                rebase_remote,
1395                                rebase_remote, rebase_branch,
1396                                rebase_remote,
1397                                rebase_remote, rebase_branch
1398                             ))]}
1399
1400             self.retry = builder('retry', retry_task)
1401             self.need_retry = False
1402
1403         if options.skip_dependencies:
1404             for b in self.tlist:
1405                 if b.name in implicit_tasknames:
1406                     b.mark_existing()
1407
1408         for b in self.tlist:
1409             do_debug("b.name=%s" % b.name)
1410             if "dependency" not in b.definition:
1411                 continue
1412             depname = b.definition["dependency"]
1413             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1414             for p in self.tlist:
1415                 if p.name == depname:
1416                     p.add_consumer(b)
1417
1418     def kill_kids(self):
1419         if self.tail_proc is not None:
1420             self.tail_proc.terminate()
1421             self.tail_proc.wait()
1422             self.tail_proc = None
1423         if self.retry is not None:
1424             self.retry.proc.terminate()
1425             self.retry.proc.wait()
1426             self.retry = None
1427         for b in self.tlist:
1428             if b.proc is not None:
1429                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1430                 b.proc.terminate()
1431                 b.proc.wait()
1432                 b.proc = None
1433
1434     def wait_one(self):
1435         while True:
1436             none_running = True
1437             for b in self.tlist:
1438                 if b.proc is None:
1439                     continue
1440                 none_running = False
1441                 b.status = b.proc.poll()
1442                 if b.status is None:
1443                     continue
1444                 b.proc = None
1445                 return b
1446             if options.retry:
1447                 ret = self.retry.proc.poll()
1448                 if ret is not None:
1449                     self.need_retry = True
1450                     self.retry = None
1451                     return None
1452             if none_running:
1453                 return None
1454             time.sleep(0.1)
1455
1456     def run(self):
1457         for b in self.tlist:
1458             b.start_next()
1459         if options.retry:
1460             self.retry.start_next()
1461         while True:
1462             b = self.wait_one()
1463             if options.retry and self.need_retry:
1464                 self.kill_kids()
1465                 do_print("retry needed")
1466                 return (0, None, None, None, "retry")
1467             if b is None:
1468                 break
1469             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1470                 self.kill_kids()
1471                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1472             b.start_next()
1473         self.kill_kids()
1474         return (0, None, None, None, "All OK")
1475
1476     def write_system_info(self, filename):
1477         with open(filename, 'w') as f:
1478             for cmd in ['uname -a',
1479                         'lsb_release -a',
1480                         'free',
1481                         'mount',
1482                         'cat /proc/cpuinfo',
1483                         'cc --version',
1484                         'df -m .',
1485                         'df -m %s' % testbase]:
1486                 try:
1487                     out = run_cmd(cmd, output=True, checkfail=False)
1488                 except CalledProcessError as e:
1489                     out = "<failed: %s>" % str(e)
1490                 print('### %s' % cmd, file=f)
1491                 print(out, file=f)
1492                 print(file=f)
1493
1494     def tarlogs(self, fname):
1495         with tarfile.open(fname, "w:gz") as tar:
1496             for b in self.tlist:
1497                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1498                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1499             if os.path.exists("autobuild.log"):
1500                 tar.add("autobuild.log")
1501             filename = 'system-info.txt'
1502             self.write_system_info(filename)
1503             tar.add(filename)
1504
1505     def remove_logs(self):
1506         for b in self.tlist:
1507             os.unlink(b.stdout_path)
1508             os.unlink(b.stderr_path)
1509
1510     def start_tail(self):
1511         cmd = ["tail", "-f"]
1512         for b in self.tlist:
1513             cmd.append(b.stdout_path)
1514             cmd.append(b.stderr_path)
1515         self.tail_proc = Popen(cmd, close_fds=True)
1516
1517
1518 def cleanup(do_raise=False):
1519     if options.nocleanup:
1520         return
1521     run_cmd("stat %s || true" % test_tmpdir, show=True)
1522     run_cmd("stat %s" % testbase, show=True)
1523     do_print("Cleaning up %r" % cleanup_list)
1524     for d in cleanup_list:
1525         ok = rmdir_force(d, re_raise=False)
1526         if ok:
1527             continue
1528         if os.path.isdir(d):
1529             do_print("Killing, waiting and retry")
1530             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1531         else:
1532             do_print("Waiting and retry")
1533         time.sleep(1)
1534         rmdir_force(d, re_raise=do_raise)
1535
1536
1537 def daemonize(logfile):
1538     pid = os.fork()
1539     if pid == 0:  # Parent
1540         os.setsid()
1541         pid = os.fork()
1542         if pid != 0:  # Actual daemon
1543             os._exit(0)
1544     else:  # Grandparent
1545         os._exit(0)
1546
1547     import resource      # Resource usage information.
1548     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1549     if maxfd == resource.RLIM_INFINITY:
1550         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1551     for fd in range(0, maxfd):
1552         try:
1553             os.close(fd)
1554         except OSError:
1555             pass
1556     os.open(logfile, os.O_RDWR | os.O_CREAT)
1557     os.dup2(0, 1)
1558     os.dup2(0, 2)
1559
1560
1561 def write_pidfile(fname):
1562     '''write a pid file, cleanup on exit'''
1563     with open(fname, mode='w') as f:
1564         f.write("%u\n" % os.getpid())
1565
1566
1567 def rebase_tree(rebase_url, rebase_branch="master"):
1568     rebase_remote = "rebaseon"
1569     do_print("Rebasing on %s" % rebase_url)
1570     run_cmd("git describe HEAD", show=True, dir=test_master)
1571     run_cmd("git remote add -t %s %s %s" %
1572             (rebase_branch, rebase_remote, rebase_url),
1573             show=True, dir=test_master)
1574     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1575     if options.fix_whitespace:
1576         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1577                 (rebase_remote, rebase_branch),
1578                 show=True, dir=test_master)
1579     else:
1580         run_cmd("git rebase --force-rebase %s/%s" %
1581                 (rebase_remote, rebase_branch),
1582                 show=True, dir=test_master)
1583     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1584                    (rebase_remote, rebase_branch),
1585                    dir=test_master, output=True)
1586     if diff == '':
1587         do_print("No differences between HEAD and %s/%s - exiting" %
1588                  (rebase_remote, rebase_branch))
1589         sys.exit(0)
1590     run_cmd("git describe %s/%s" %
1591             (rebase_remote, rebase_branch),
1592             show=True, dir=test_master)
1593     run_cmd("git describe HEAD", show=True, dir=test_master)
1594     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1595             (rebase_remote, rebase_branch),
1596             show=True, dir=test_master)
1597
1598
1599 def push_to(push_url, push_branch="master"):
1600     push_remote = "pushto"
1601     do_print("Pushing to %s" % push_url)
1602     if options.mark:
1603         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1604         run_cmd("git commit --amend -c HEAD", dir=test_master)
1605         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1606         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1607     run_cmd("git remote add -t %s %s %s" %
1608             (push_branch, push_remote, push_url),
1609             show=True, dir=test_master)
1610     run_cmd("git push %s +HEAD:%s" %
1611             (push_remote, push_branch),
1612             show=True, dir=test_master)
1613
1614
1615 def send_email(subject, text, log_tar):
1616     if options.email is None:
1617         do_print("not sending email because the recipient is not set")
1618         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1619                  (subject, text))
1620         return
1621     outer = MIMEMultipart()
1622     outer['Subject'] = subject
1623     outer['To'] = options.email
1624     outer['From'] = options.email_from
1625     outer['Date'] = email.utils.formatdate(localtime=True)
1626     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1627     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1628     if options.attach_logs:
1629         with open(log_tar, 'rb') as fp:
1630             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1631         # Set the filename parameter
1632         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1633         outer.attach(msg)
1634     content = outer.as_string()
1635     s = smtplib.SMTP(options.email_server)
1636     email_user = os.getenv('SMTP_USERNAME')
1637     email_password = os.getenv('SMTP_PASSWORD')
1638     if email_user is not None:
1639         s.starttls()
1640         s.login(email_user, email_password)
1641
1642     s.sendmail(options.email_from, [options.email], content)
1643     s.set_debuglevel(1)
1644     s.quit()
1645
1646
1647 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1648                   elapsed_time, log_base=None, add_log_tail=True):
1649     '''send an email to options.email about the failure'''
1650     elapsed_minutes = elapsed_time / 60.0
1651     if log_base is None:
1652         log_base = gitroot
1653     text = '''
1654 Dear Developer,
1655
1656 Your autobuild on %s failed after %.1f minutes
1657 when trying to test %s with the following error:
1658
1659    %s
1660
1661 the autobuild has been abandoned. Please fix the error and resubmit.
1662
1663 A summary of the autobuild process is here:
1664
1665   %s/autobuild.log
1666 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1667
1668     if options.restrict_tests:
1669         text += """
1670 The build was restricted to tests matching %s\n""" % options.restrict_tests
1671
1672     if failed_task != 'rebase':
1673         text += '''
1674 You can see logs of the failed task here:
1675
1676   %s/%s.stdout
1677   %s/%s.stderr
1678
1679 or you can get full logs of all tasks in this job here:
1680
1681   %s/logs.tar.gz
1682
1683 The top commit for the tree that was built was:
1684
1685 %s
1686
1687 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1688
1689     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1690     if add_log_tail and os.access(log_stdout, os.R_OK):
1691         f = open(log_stdout, 'r')
1692         lines = f.readlines()
1693         log_tail = "".join(lines[-50:])
1694         num_lines = len(lines)
1695         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1696         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1697             # Also include stderr (compile failures) if < 50 lines of stdout
1698             f = open(log_stderr, 'r')
1699             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1700
1701         text += '''
1702 The last 50 lines of log messages:
1703
1704 %s
1705     ''' % log_tail
1706         f.close()
1707
1708     logs = os.path.join(gitroot, 'logs.tar.gz')
1709     send_email('autobuild[%s] failure on %s for task %s during %s'
1710                % (options.branch, platform.node(), failed_task, failed_stage),
1711                text, logs)
1712
1713
1714 def email_success(elapsed_time, log_base=None):
1715     '''send an email to options.email about a successful build'''
1716     if log_base is None:
1717         log_base = gitroot
1718     text = '''
1719 Dear Developer,
1720
1721 Your autobuild on %s has succeeded after %.1f minutes.
1722
1723 ''' % (platform.node(), elapsed_time / 60.)
1724
1725     if options.restrict_tests:
1726         text += """
1727 The build was restricted to tests matching %s\n""" % options.restrict_tests
1728
1729     if options.keeplogs:
1730         text += '''
1731
1732 you can get full logs of all tasks in this job here:
1733
1734   %s/logs.tar.gz
1735
1736 ''' % log_base
1737
1738     text += '''
1739 The top commit for the tree that was built was:
1740
1741 %s
1742 ''' % top_commit_msg
1743
1744     logs = os.path.join(gitroot, 'logs.tar.gz')
1745     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1746                text, logs)
1747
1748
1749 # get the top commit message, for emails
1750 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1751
1752 try:
1753     if options.skip_dependencies:
1754         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1755     else:
1756         os.makedirs(testbase)
1757 except Exception as reason:
1758     raise Exception("Unable to create %s : %s" % (testbase, reason))
1759 cleanup_list.append(testbase)
1760
1761 if options.daemon:
1762     logfile = os.path.join(testbase, "log")
1763     do_print("Forking into the background, writing progress to %s" % logfile)
1764     daemonize(logfile)
1765
1766 write_pidfile(gitroot + "/autobuild.pid")
1767
1768 start_time = time.time()
1769
1770 while True:
1771     try:
1772         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1773         os.makedirs(test_tmpdir)
1774         # The waf uninstall code removes empty directories all the way
1775         # up the tree.  Creating a file in test_tmpdir stops it from
1776         # being removed.
1777         run_cmd("touch %s" % os.path.join(test_tmpdir,
1778                                           ".directory-is-not-empty"), show=True)
1779         run_cmd("stat %s" % test_tmpdir, show=True)
1780         run_cmd("stat %s" % testbase, show=True)
1781         if options.skip_dependencies:
1782             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1783         else:
1784             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1785     except Exception:
1786         cleanup()
1787         raise
1788
1789     try:
1790         if options.rebase is not None:
1791             rebase_tree(options.rebase, rebase_branch=options.branch)
1792     except Exception:
1793         cleanup_list.append(gitroot + "/autobuild.pid")
1794         cleanup()
1795         elapsed_time = time.time() - start_time
1796         email_failure(-1, 'rebase', 'rebase', 'rebase',
1797                       'rebase on %s failed' % options.branch,
1798                       elapsed_time, log_base=options.log_base)
1799         sys.exit(1)
1800
1801     try:
1802         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1803         if options.tail:
1804             blist.start_tail()
1805         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1806         if status != 0 or errstr != "retry":
1807             break
1808         cleanup(do_raise=True)
1809     except Exception:
1810         cleanup()
1811         raise
1812
1813 cleanup_list.append(gitroot + "/autobuild.pid")
1814
1815 do_print(errstr)
1816
1817 blist.kill_kids()
1818 if options.tail:
1819     do_print("waiting for tail to flush")
1820     time.sleep(1)
1821
1822 elapsed_time = time.time() - start_time
1823 if status == 0:
1824     if options.passcmd is not None:
1825         do_print("Running passcmd: %s" % options.passcmd)
1826         run_cmd(options.passcmd, dir=test_master)
1827     if options.pushto is not None:
1828         push_to(options.pushto, push_branch=options.branch)
1829     if options.keeplogs or options.attach_logs:
1830         blist.tarlogs("logs.tar.gz")
1831         do_print("Logs in logs.tar.gz")
1832     if options.always_email:
1833         email_success(elapsed_time, log_base=options.log_base)
1834     blist.remove_logs()
1835     cleanup()
1836     do_print(errstr)
1837     sys.exit(0)
1838
1839 # something failed, gather a tar of the logs
1840 blist.tarlogs("logs.tar.gz")
1841
1842 if options.email is not None:
1843     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1844                   elapsed_time, log_base=options.log_base)
1845 else:
1846     elapsed_minutes = elapsed_time / 60.0
1847     print('''
1848
1849 ####################################################################
1850
1851 AUTOBUILD FAILURE
1852
1853 Your autobuild[%s] on %s failed after %.1f minutes
1854 when trying to test %s with the following error:
1855
1856    %s
1857
1858 the autobuild has been abandoned. Please fix the error and resubmit.
1859
1860 ####################################################################
1861
1862 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1863
1864 cleanup()
1865 do_print(errstr)
1866 do_print("Logs in logs.tar.gz")
1867 sys.exit(status)