script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 if args:
 142     # If we are only running specific test,
 143     # do not sleep randomly to wait for it to start
 144     def random_sleep(low, high):
 145         return 'sleep 1'
 146 else:
 147     def random_sleep(low, high):
 148         return 'sleep {}'.format(random.randint(low, high))
 149
 150 cleanup_list = []
 151
 152 builddirs = {
 153     "ctdb": "ctdb",
 154     "ldb": "lib/ldb",
 155     "tdb": "lib/tdb",
 156     "talloc": "lib/talloc",
 157     "replace": "lib/replace",
 158     "tevent": "lib/tevent",
 159     "pidl": "pidl",
 160     "docs-xml": "docs-xml"
 161 }
 162
 163 ctdb_configure_params = " --enable-developer ${PREFIX}"
 164 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 165
 166 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 167 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 168 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 169 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 170 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 171 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 172 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 173
 174
 175 def format_option(name, value=None):
 176     """Format option as str list."""
 177     if value is None:  # boolean option
 178         return [name]
 179     if not isinstance(value, list):  # single value option
 180         value = [value]
 181     # repeatable option
 182     return ['{}={}'.format(name, item) for item in value]
 183
 184
 185 def make_test(
 186         cmd='make testonly',
 187         INJECT_SELFTEST_PREFIX=1,
 188         TESTS='',
 189         include_envs=None,
 190         exclude_envs=None):
 191
 192     test_options = []
 193     if include_envs:
 194         test_options = format_option('--include-env', include_envs)
 195     if exclude_envs:
 196         test_options = format_option('--exclude-env', exclude_envs)
 197     if test_options:
 198         # join envs options to original test options
 199         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 200
 201     _options = []
 202
 203     # Allow getting a full CI with
 204     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 205
 206     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 207
 208     if int(FAIL_IMMEDIATELY):
 209         _options.append('FAIL_IMMEDIATELY=1')
 210     if TESTS:
 211         _options.append("TESTS='{}'".format(TESTS))
 212
 213     if INJECT_SELFTEST_PREFIX:
 214         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 215         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 216
 217     return ' '.join([cmd] + _options)
 218
 219
 220 # When updating this list, also update .gitlab-ci.yml to add the job
 221 # and to make it a dependency of 'page' for the coverage report.
 222
 223 tasks = {
 224     "ctdb": {
 225         "sequence": [
 226             ("random-sleep", random_sleep(300, 900)),
 227             ("configure", "./configure " + ctdb_configure_params),
 228             ("make", "make all"),
 229             ("install", "make install"),
 230             ("test", "make autotest"),
 231             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 232             ("clean", "make clean"),
 233         ],
 234     },
 235     "docs-xml": {
 236         "sequence": [
 237             ("random-sleep", random_sleep(300, 900)),
 238             ("autoconf", "autoconf"),
 239             ("configure", "./configure"),
 240             ("make", "make html htmlman"),
 241             ("publish-docs", PUBLISH_DOCS),
 242             ("clean", "make clean"),
 243         ],
 244     },
 245
 246     "samba-def-build": {
 247         "git-clone-required": True,
 248         "sequence": [
 249             ("configure", "./configure.developer" + samba_configure_params),
 250             ("make", "make -j"),
 251             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 252             ("chmod-R-a-w", "chmod -R a-w ."),
 253         ],
 254     },
 255
 256     "samba-mit-build": {
 257         "git-clone-required": True,
 258         "sequence": [
 259             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 260             ("make", "make -j"),
 261             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 262             ("chmod-R-a-w", "chmod -R a-w ."),
 263         ],
 264     },
 265
 266     "samba-nt4-build": {
 267         "git-clone-required": True,
 268         "sequence": [
 269             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 270             ("make", "make -j"),
 271             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 272             ("chmod-R-a-w", "chmod -R a-w ."),
 273         ],
 274     },
 275
 276     "samba-h5l-build": {
 277         "git-clone-required": True,
 278         "sequence": [
 279             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 280             ("make", "make -j"),
 281             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 282             ("chmod-R-a-w", "chmod -R a-w ."),
 283         ],
 284     },
 285
 286     "samba-without-smb1-build": {
 287         "git-clone-required": True,
 288         "sequence": [
 289             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 290             ("make", "make -j"),
 291             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 292             ("chmod-R-a-w", "chmod -R a-w ."),
 293         ],
 294     },
 295
 296     "samba-no-opath-build": {
 297         "git-clone-required": True,
 298         "sequence": [
 299             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 300             ("make", "make -j"),
 301             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 302             ("chmod-R-a-w", "chmod -R a-w ."),
 303         ],
 304     },
 305
 306     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 307     "samba": {
 308         "sequence": [
 309             ("random-sleep", random_sleep(300, 900)),
 310             ("configure", "./configure.developer" + samba_configure_params),
 311             ("make", "make -j"),
 312             ("test", make_test(exclude_envs=[
 313             "none",
 314             "nt4_dc",
 315             "nt4_dc_smb1",
 316             "nt4_dc_smb1_done",
 317             "nt4_dc_schannel",
 318             "nt4_member",
 319             "ad_dc",
 320             "ad_dc_smb1",
 321             "ad_dc_smb1_done",
 322             "ad_dc_backup",
 323             "ad_dc_ntvfs",
 324             "ad_dc_default",
 325             "ad_dc_default_smb1",
 326             "ad_dc_slowtests",
 327             "ad_dc_no_nss",
 328             "ad_dc_no_ntlm",
 329             "fl2003dc",
 330             "fl2008dc",
 331             "fl2008r2dc",
 332             "ad_member",
 333             "ad_member_idmap_rid",
 334             "admem_idmap_autorid",
 335             "ad_member_idmap_ad",
 336             "ad_member_rfc2307",
 337             "ad_member_oneway",
 338             "chgdcpass",
 339             "vampire_2000_dc",
 340             "fl2000dc",
 341             "fileserver",
 342             "fileserver_smb1",
 343             "fileserver_smb1_done",
 344             "maptoguest",
 345             "simpleserver",
 346             "backupfromdc",
 347             "restoredc",
 348             "renamedc",
 349             "offlinebackupdc",
 350             "labdc",
 351             "preforkrestartdc",
 352             "proclimitdc",
 353             "promoted_dc",
 354             "vampire_dc",
 355             "rodc",
 356             "ad_dc_default",
 357             "ad_dc_default_smb1",
 358             "ad_dc_default_smb1_done",
 359             "ad_dc_slowtests",
 360             "schema_pair_dc",
 361             "schema_dc",
 362             "clusteredmember",
 363             "ad_dc_fips",
 364             "ad_member_fips",
 365             ])),
 366             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 367             ("lcov", LCOV_CMD),
 368             ("install", "make install"),
 369             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 370             ("clean", "make clean"),
 371         ],
 372     },
 373
 374     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 375     "samba-mitkrb5": {
 376         "sequence": [
 377             ("random-sleep", random_sleep(300, 900)),
 378             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 379             ("make", "make -j"),
 380             ("test", make_test(exclude_envs=[
 381             "none",
 382             "nt4_dc",
 383             "nt4_dc_smb1",
 384             "nt4_dc_smb1_done",
 385             "nt4_dc_schannel",
 386             "nt4_member",
 387             "ad_dc",
 388             "ad_dc_smb1",
 389             "ad_dc_smb1_done",
 390             "ad_dc_backup",
 391             "ad_dc_ntvfs",
 392             "ad_dc_default",
 393             "ad_dc_default_smb1",
 394             "ad_dc_default_smb1_done",
 395             "ad_dc_slowtests",
 396             "ad_dc_no_nss",
 397             "ad_dc_no_ntlm",
 398             "fl2003dc",
 399             "fl2008dc",
 400             "fl2008r2dc",
 401             "ad_member",
 402             "ad_member_idmap_rid",
 403             "admem_idmap_autorid",
 404             "ad_member_idmap_ad",
 405             "ad_member_rfc2307",
 406             "ad_member_oneway",
 407             "chgdcpass",
 408             "vampire_2000_dc",
 409             "fl2000dc",
 410             "fileserver",
 411             "fileserver_smb1",
 412             "fileserver_smb1_done",
 413             "maptoguest",
 414             "simpleserver",
 415             "backupfromdc",
 416             "restoredc",
 417             "renamedc",
 418             "offlinebackupdc",
 419             "labdc",
 420             "preforkrestartdc",
 421             "proclimitdc",
 422             "promoted_dc",
 423             "vampire_dc",
 424             "rodc",
 425             "ad_dc_default",
 426             "ad_dc_default_smb1",
 427             "ad_dc_default_smb1_done",
 428             "ad_dc_slowtests",
 429             "schema_pair_dc",
 430             "schema_dc",
 431             "clusteredmember",
 432             "ad_dc_fips",
 433             "ad_member_fips",
 434             ])),
 435             ("lcov", LCOV_CMD),
 436             ("install", "make install"),
 437             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 438             ("clean", "make clean"),
 439         ],
 440     },
 441
 442     "samba-nt4": {
 443         "dependency": "samba-nt4-build",
 444         "sequence": [
 445             ("random-sleep", random_sleep(300, 900)),
 446             ("test", make_test(include_envs=[
 447             "nt4_dc",
 448             "nt4_dc_smb1",
 449             "nt4_dc_smb1_done",
 450             "nt4_dc_schannel",
 451             "nt4_member",
 452             "simpleserver",
 453             ])),
 454             ("lcov", LCOV_CMD),
 455             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 456         ],
 457     },
 458
 459     "samba-fileserver": {
 460         "dependency": "samba-h5l-build",
 461         "sequence": [
 462             ("random-sleep", random_sleep(300, 900)),
 463             ("test", make_test(include_envs=[
 464             "fileserver",
 465             "fileserver_smb1",
 466             "fileserver_smb1_done",
 467             "maptoguest",
 468             "ktest", # ktest is also tested in samba-ktest-mit samba
 469                      # and samba-mitkrb5 but is tested here against
 470                      # a system Heimdal
 471             ])),
 472             ("lcov", LCOV_CMD),
 473             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 474         ],
 475     },
 476
 477     "samba-fileserver-without-smb1": {
 478         "dependency": "samba-without-smb1-build",
 479         "sequence": [
 480             ("random-sleep", random_sleep(300, 900)),
 481             ("test", make_test(include_envs=["fileserver"])),
 482             ("lcov", LCOV_CMD),
 483             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 484         ],
 485     },
 486
 487     # This is a full build without the AD DC so we test the build with
 488     # MIT Kerberos from the current system.  Runtime behaviour is
 489     # confirmed via the ktest (static ccache and keytab) environment
 490
 491     # This environment also used to confirm we can still build with --with-libunwind
 492     "samba-ktest-mit": {
 493         "sequence": [
 494             ("random-sleep", random_sleep(300, 900)),
 495             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 496             ("make", "make -j"),
 497             ("test", make_test(include_envs=[
 498             "ktest", # ktest is also tested in fileserver, samba and
 499                      # samba-mitkrb5 but is tested here against a
 500                      # system MIT krb5
 501             ])),
 502             ("lcov", LCOV_CMD),
 503             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 504         ],
 505     },
 506
 507     "samba-admem": {
 508         "dependency": "samba-def-build",
 509         "sequence": [
 510             ("random-sleep", random_sleep(300, 900)),
 511             ("test", make_test(include_envs=[
 512             "ad_member",
 513             "ad_member_idmap_rid",
 514             "admem_idmap_autorid",
 515             "ad_member_idmap_ad",
 516             "ad_member_rfc2307",
 517             "ad_member_offlogon",
 518             ])),
 519             ("lcov", LCOV_CMD),
 520             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 521         ],
 522     },
 523
 524     "samba-no-opath1": {
 525         "dependency": "samba-no-opath-build",
 526         "sequence": [
 527             ("random-sleep", random_sleep(300, 900)),
 528             ("test", make_test(
 529                 cmd="make testonly DISABLE_OPATH=1",
 530                 include_envs=[
 531                 "nt4_dc",
 532                 "nt4_dc_smb1",
 533                 "nt4_dc_smb1_done",
 534                 "nt4_dc_schannel",
 535                 "nt4_member",
 536                 "simpleserver",
 537                 ])),
 538             ("lcov", LCOV_CMD),
 539             ("check-clean-tree", "script/clean-source-tree.sh"),
 540         ],
 541     },
 542
 543     "samba-no-opath2": {
 544         "dependency": "samba-no-opath-build",
 545         "sequence": [
 546             ("random-sleep", random_sleep(300, 900)),
 547             ("test", make_test(
 548                 cmd="make testonly DISABLE_OPATH=1",
 549                 include_envs=[
 550                 "fileserver",
 551                 "fileserver_smb1",
 552                 "fileserver_smb1_done",
 553                 ])),
 554             ("lcov", LCOV_CMD),
 555             ("check-clean-tree", "script/clean-source-tree.sh"),
 556         ],
 557     },
 558
 559     "samba-ad-dc-1": {
 560         "dependency": "samba-def-build",
 561         "sequence": [
 562             ("random-sleep", random_sleep(1, 1)),
 563             ("test", make_test(include_envs=[
 564             "ad_dc",
 565             "ad_dc_smb1",
 566             "ad_dc_smb1_done",
 567             "ad_dc_no_nss",
 568             "ad_dc_no_ntlm",
 569             ])),
 570             ("lcov", LCOV_CMD),
 571             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 572         ],
 573     },
 574
 575     "samba-ad-dc-2": {
 576         "dependency": "samba-def-build",
 577         "sequence": [
 578             ("random-sleep", random_sleep(1, 1)),
 579             ("test", make_test(include_envs=[
 580             "vampire_dc",
 581             "vampire_2000_dc",
 582             "rodc",
 583             ])),
 584             ("lcov", LCOV_CMD),
 585             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 586         ],
 587     },
 588
 589     "samba-ad-dc-3": {
 590         "dependency": "samba-def-build",
 591         "sequence": [
 592             ("random-sleep", random_sleep(1, 1)),
 593             ("test", make_test(include_envs=[
 594             "promoted_dc",
 595             "chgdcpass",
 596             "preforkrestartdc",
 597             "proclimitdc",
 598             ])),
 599             ("lcov", LCOV_CMD),
 600             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 601         ],
 602     },
 603
 604     "samba-ad-dc-4a": {
 605         "dependency": "samba-def-build",
 606         "sequence": [
 607             ("random-sleep", random_sleep(1, 1)),
 608             ("test", make_test(include_envs=[
 609             "fl2000dc",
 610             "ad_member_oneway",
 611             "fl2003dc",
 612             ])),
 613             ("lcov", LCOV_CMD),
 614             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 615         ],
 616     },
 617     "samba-ad-dc-4b": {
 618         "dependency": "samba-def-build",
 619         "sequence": [
 620             ("random-sleep", random_sleep(1, 1)),
 621             ("test", make_test(include_envs=[
 622             "fl2008dc",
 623             "fl2008r2dc",
 624             ])),
 625             ("lcov", LCOV_CMD),
 626             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 627         ],
 628     },
 629
 630     "samba-ad-dc-5": {
 631         "dependency": "samba-def-build",
 632         "sequence": [
 633             ("random-sleep", random_sleep(1, 1)),
 634             ("test", make_test(include_envs=[
 635             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 636             ("lcov", LCOV_CMD),
 637             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 638         ],
 639     },
 640
 641     "samba-ad-dc-6": {
 642         "dependency": "samba-def-build",
 643         "sequence": [
 644             ("random-sleep", random_sleep(1, 1)),
 645             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 646             ("lcov", LCOV_CMD),
 647             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 648         ],
 649     },
 650
 651     "samba-schemaupgrade": {
 652         "dependency": "samba-def-build",
 653         "sequence": [
 654             ("random-sleep", random_sleep(1, 1)),
 655             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 656             ("lcov", LCOV_CMD),
 657             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 658         ],
 659     },
 660
 661     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 662     # This is currently the longest task, so we don't randomly delay it.
 663     "samba-ad-dc-ntvfs": {
 664         "dependency": "samba-def-build",
 665         "sequence": [
 666             ("random-sleep", random_sleep(1, 1)),
 667             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 668             ("lcov", LCOV_CMD),
 669             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 670         ],
 671     },
 672
 673     # Test fips compliance
 674     "samba-fips": {
 675         "dependency": "samba-mit-build",
 676         "sequence": [
 677             ("random-sleep", random_sleep(1, 1)),
 678             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 679             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 680             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 681         ],
 682     },
 683
 684     # run the backup/restore testenvs separately as they're fairly standalone
 685     # (and CI seems to max out at ~3 different DCs running at once)
 686     "samba-ad-back1": {
 687         "dependency": "samba-def-build",
 688         "sequence": [
 689             ("random-sleep", random_sleep(300, 900)),
 690             ("test", make_test(include_envs=[
 691             "backupfromdc",
 692             "restoredc",
 693             "renamedc",
 694             ])),
 695             ("lcov", LCOV_CMD),
 696             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 697         ],
 698     },
 699     "samba-ad-back2": {
 700         "dependency": "samba-def-build",
 701         "sequence": [
 702             ("random-sleep", random_sleep(300, 900)),
 703             ("test", make_test(include_envs=[
 704             "backupfromdc",
 705             "offlinebackupdc",
 706             "labdc",
 707             ])),
 708             ("lcov", LCOV_CMD),
 709             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 710         ],
 711     },
 712
 713     "samba-admem-mit": {
 714         "dependency": "samba-mit-build",
 715         "sequence": [
 716             ("random-sleep", random_sleep(1, 1)),
 717             ("test", make_test(include_envs=[
 718             "ad_member",
 719             "ad_member_idmap_rid",
 720             "admem_idmap_autorid",
 721             "ad_member_idmap_ad",
 722             "ad_member_rfc2307",
 723             "ad_member_offlogon",
 724             ])),
 725             ("lcov", LCOV_CMD),
 726             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 727         ],
 728     },
 729
 730     "samba-addc-mit-1": {
 731         "dependency": "samba-mit-build",
 732         "sequence": [
 733             ("random-sleep", random_sleep(1, 1)),
 734             ("test", make_test(include_envs=[
 735             "ad_dc",
 736             "ad_dc_smb1",
 737             "ad_dc_smb1_done",
 738             "ad_dc_no_nss",
 739             "ad_dc_no_ntlm",
 740             ])),
 741             ("lcov", LCOV_CMD),
 742             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 743         ],
 744     },
 745
 746     "samba-addc-mit-4a": {
 747         "dependency": "samba-mit-build",
 748         "sequence": [
 749             ("random-sleep", random_sleep(1, 1)),
 750             ("test", make_test(include_envs=[
 751             "fl2000dc",
 752             "ad_member_oneway",
 753             "fl2003dc",
 754             ])),
 755             ("lcov", LCOV_CMD),
 756             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 757         ],
 758     },
 759     "samba-addc-mit-4b": {
 760         "dependency": "samba-mit-build",
 761         "sequence": [
 762             ("random-sleep", random_sleep(1, 1)),
 763             ("test", make_test(include_envs=[
 764             "fl2008dc",
 765             "fl2008r2dc",
 766             ])),
 767             ("lcov", LCOV_CMD),
 768             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 769         ],
 770     },
 771
 772     "samba-test-only": {
 773         "sequence": [
 774             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 775             ("make", "make -j"),
 776             ("test", make_test(TESTS="${TESTS}")),
 777             ("lcov", LCOV_CMD),
 778         ],
 779     },
 780
 781     # Test cross-compile infrastructure
 782     "samba-xc": {
 783         "sequence": [
 784             ("random-sleep", random_sleep(900, 1500)),
 785             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 786             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 787             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 788             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 789             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 790             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 791             ("compare-results", "script/compare_cc_results.py "
 792             "./bin/c4che/default{} "
 793             "./bin-xe/c4che/default{} "
 794             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 795             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 796             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 797             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 798             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 799             " = \"'1234'\"".format(CACHE_SUFFIX)),
 800             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 801             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 802             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 803             " ; test $? -ne 0"),
 804         ],
 805     },
 806
 807     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 808     "samba-o3": {
 809         "sequence": [
 810             ("random-sleep", random_sleep(300, 900)),
 811             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 812             ("make", "make -j"),
 813             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 814             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 815             ("lcov", LCOV_CMD),
 816             ("install", "make install"),
 817             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 818             ("clean", "make clean"),
 819         ],
 820     },
 821
 822     "samba-32bit": {
 823         "sequence": [
 824             ("random-sleep", random_sleep(300, 900)),
 825             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 826             ("make", "make -j"),
 827             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 828             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 829             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 830             ("install", "make install"),
 831             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 832             ("clean", "make clean"),
 833         ],
 834     },
 835
 836     "samba-ctdb": {
 837         "sequence": [
 838             ("random-sleep", random_sleep(900, 1500)),
 839
 840         # make sure we have tdb around:
 841             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 842             ("tdb-make", "cd lib/tdb && make"),
 843             ("tdb-install", "cd lib/tdb && make install"),
 844
 845         # build samba with cluster support (also building ctdb):
 846             ("samba-configure",
 847          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 848          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 849          "./configure.developer ${PREFIX} "
 850          "--with-selftest-prefix=./bin/ab "
 851          "--with-cluster-support "
 852          "--without-ad-dc "
 853          "--bundled-libraries=!tdb"),
 854             ("samba-make", "make"),
 855             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 856             ("samba-install", "make install"),
 857             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 858
 859             ("test", make_test(
 860                 cmd='make test',
 861                 INJECT_SELFTEST_PREFIX=0,
 862                 include_envs=["clusteredmember"])
 863             ),
 864
 865         # clean up:
 866             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 867             ("clean", "make clean"),
 868             ("ctdb-clean", "cd ./ctdb && make clean"),
 869         ],
 870     },
 871
 872     "samba-libs": {
 873         "sequence": [
 874             ("random-sleep", random_sleep(300, 900)),
 875             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 876             ("talloc-make", "cd lib/talloc && make"),
 877             ("talloc-install", "cd lib/talloc && make install"),
 878
 879             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 880             ("tdb-make", "cd lib/tdb && make"),
 881             ("tdb-install", "cd lib/tdb && make install"),
 882
 883             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 884             ("tevent-make", "cd lib/tevent && make"),
 885             ("tevent-install", "cd lib/tevent && make install"),
 886
 887             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 888             ("ldb-make", "cd lib/ldb && make"),
 889             ("ldb-install", "cd lib/ldb && make install"),
 890
 891             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 892             ("nondevel-make", "make -j"),
 893             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 894             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 895             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 896             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 897             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 898             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 899             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 900             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 901             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 902             ("nondevel-no-public-nss_winbind",
 903                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 904             ("nondevel-no-public-nss_wins",
 905                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 906             ("nondevel-no-public-libwbclient",
 907                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 908             ("nondevel-no-public-pam_winbind",
 909                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 910             ("nondevel-no-public-winbind_krb5_locator",
 911                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 912             ("nondevel-no-public-async_dns_krb5_locator",
 913                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 914             ("nondevel-install", "make -j install"),
 915             ("nondevel-dist", "make dist"),
 916
 917             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 918             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 919             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 920             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 921             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 922             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 923             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 924             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 925             ("prefix-no-public-nss_winbind",
 926                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 927             ("prefix-no-public-nss_wins",
 928                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 929             ("prefix-no-public-libwbclient",
 930                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 931             ("prefix-no-public-pam_winbind",
 932                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 933             ("prefix-no-public-winbind_krb5_locator",
 934                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 935                               "service_locator")),
 936             ("prefix-no-public-async_dns_krb5_locator",
 937                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 938                               "service_locator")),
 939
 940             # retry with all modules shared
 941             ("allshared-distclean", "make distclean"),
 942             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 943             ("allshared-make", "make -j"),
 944             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 945             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 946             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 947             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 948             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 949             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 950             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 951             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 952             ("allshared-no-public-nss_winbind",
 953                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 954             ("allshared-no-public-nss_wins",
 955                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 956             ("allshared-no-public-libwbclient",
 957                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 958             ("allshared-no-public-pam_winbind",
 959                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 960             ("allshared-no-public-winbind_krb5_locator",
 961                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 962             ("allshared-no-public-async_dns_krb5_locator",
 963                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 964         ],
 965     },
 966
 967     "samba-fuzz": {
 968         "sequence": [
 969         # build the fuzzers (static) via the oss-fuzz script
 970             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 971             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 972         ],
 973     },
 974
 975     # * Test smbd and smbtorture can build semi-static
 976     #
 977     # * Test Samba without python still builds.
 978     #
 979     # When this test fails due to more use of Python, the expectations
 980     # is that the newly failing part of the code should be disabled
 981     # when --disable-python is set (rather than major work being done
 982     # to support this environment).
 983     #
 984     # The target here is for vendors shipping a minimal smbd.
 985     "samba-minimal-smbd": {
 986         "sequence": [
 987             ("random-sleep", random_sleep(300, 900)),
 988
 989         # build with all modules static
 990             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 991             ("allstatic-make", "make -j"),
 992             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 993             ("allstatic-lcov", LCOV_CMD),
 994
 995         # retry with nonshared smbd and smbtorture
 996             ("nonshared-distclean", "make distclean"),
 997             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 998             ("nonshared-make", "make -j"),
 999             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1000             # TODO ("nonshared-lcov", LCOV_CMD),
1001
1002             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1003             ("clean", "make clean"),
1004         ],
1005     },
1006
1007     "samba-nopython": {
1008         "sequence": [
1009             ("random-sleep", random_sleep(300, 900)),
1010
1011             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1012             ("make", "make -j"),
1013             ("find-python", "script/find_python.sh ${PREFIX}"),
1014             ("test", "make test-nopython"),
1015             ("lcov", LCOV_CMD),
1016             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1017             ("clean", "make clean"),
1018
1019             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1020             ("talloc-make", "cd lib/talloc && make"),
1021             ("talloc-install", "cd lib/talloc && make install"),
1022
1023             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1024             ("tdb-make", "cd lib/tdb && make"),
1025             ("tdb-install", "cd lib/tdb && make install"),
1026
1027             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1028             ("tevent-make", "cd lib/tevent && make"),
1029             ("tevent-install", "cd lib/tevent && make install"),
1030
1031             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1032             ("ldb-make", "cd lib/ldb && make"),
1033             ("ldb-install", "cd lib/ldb && make install"),
1034
1035         # retry against installed library packages, but no required modules
1036             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1037             ("libs-make", "make -j"),
1038             ("libs-install", "make install"),
1039             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1040             ("libs-clean", "make clean"),
1041
1042         ],
1043     },
1044
1045     "samba-codecheck": {
1046         "sequence": [
1047             ("run", "script/check-shell-scripts.sh ."),
1048             ("run", "script/codespell.sh ."),
1049         ],
1050     },
1051
1052     "ldb": {
1053         "sequence": [
1054             ("random-sleep", random_sleep(60, 600)),
1055             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1056             ("make", "make"),
1057             ("install", "make install"),
1058             ("test", "make test"),
1059             ("lcov", LCOV_CMD),
1060             ("clean", "make clean"),
1061             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1062             ("make-no-lmdb", "make"),
1063             ("test-no-lmdb", "make test"),
1064             ("lcov-no-lmdb", LCOV_CMD),
1065             ("install-no-lmdb", "make install"),
1066             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1067             ("distcheck", "make distcheck"),
1068             ("clean", "make clean"),
1069         ],
1070     },
1071
1072     "tdb": {
1073         "sequence": [
1074             ("random-sleep", random_sleep(60, 600)),
1075             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1076             ("make", "make"),
1077             ("install", "make install"),
1078             ("test", "make test"),
1079             ("lcov", LCOV_CMD),
1080             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1081             ("distcheck", "make distcheck"),
1082             ("clean", "make clean"),
1083         ],
1084     },
1085
1086     "talloc": {
1087         "sequence": [
1088             ("random-sleep", random_sleep(60, 600)),
1089             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1090             ("make", "make"),
1091             ("install", "make install"),
1092             ("test", "make test"),
1093             ("lcov", LCOV_CMD),
1094             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1095             ("distcheck", "make distcheck"),
1096             ("clean", "make clean"),
1097         ],
1098     },
1099
1100     "replace": {
1101         "sequence": [
1102             ("random-sleep", random_sleep(60, 600)),
1103             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1104             ("make", "make"),
1105             ("install", "make install"),
1106             ("test", "make test"),
1107             ("lcov", LCOV_CMD),
1108             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1109             ("distcheck", "make distcheck"),
1110             ("clean", "make clean"),
1111         ],
1112     },
1113
1114     "tevent": {
1115         "sequence": [
1116             ("random-sleep", random_sleep(60, 600)),
1117             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1118             ("make", "make"),
1119             ("install", "make install"),
1120             ("test", "make test"),
1121             ("lcov", LCOV_CMD),
1122             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1123             ("distcheck", "make distcheck"),
1124             ("clean", "make clean"),
1125         ],
1126     },
1127
1128     "pidl": {
1129         "git-clone-required": True,
1130         "sequence": [
1131             ("random-sleep", random_sleep(60, 600)),
1132             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1133             ("touch", "touch *.yp"),
1134             ("make", "make"),
1135             ("test", "make test"),
1136             ("install", "make install"),
1137             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1138             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1139             ("clean", "make clean"),
1140         ],
1141     },
1142
1143     # these are useful for debugging autobuild
1144     "pass": {
1145         "sequence": [
1146             ("pass", 'echo passing && /bin/true'),
1147         ],
1148     },
1149     "fail": {
1150         "sequence": [
1151             ("fail", 'echo failing && /bin/false'),
1152         ],
1153     },
1154 }
1155
1156 defaulttasks = list(tasks.keys())
1157
1158 defaulttasks.remove("pass")
1159 defaulttasks.remove("fail")
1160
1161 # The build tasks will be brought in by the test tasks as needed
1162 defaulttasks.remove("samba-def-build")
1163 defaulttasks.remove("samba-nt4-build")
1164 defaulttasks.remove("samba-mit-build")
1165 defaulttasks.remove("samba-h5l-build")
1166 defaulttasks.remove("samba-no-opath-build")
1167
1168 # This is not a normal test, but a task to support manually running
1169 # one test under autobuild
1170 defaulttasks.remove("samba-test-only")
1171
1172 # Only built on GitLab CI and not in the default autobuild because it
1173 # uses too much space (4GB of semi-static binaries)
1174 defaulttasks.remove("samba-fuzz")
1175
1176 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1177 # container where a simulated FIPS mode is possible.
1178 defaulttasks.remove("samba-fips")
1179
1180 # The MIT build runs on a current Fedora where an up to date MIT KDC
1181 # is already packaged.  This avoids needing to backport a current MIT
1182 # to the default Ubuntu 18.04, particularly during development, and
1183 # the need to install on the shared sn-devel-184.
1184
1185 defaulttasks.remove("samba-mitkrb5")
1186 defaulttasks.remove("samba-admem-mit")
1187 defaulttasks.remove("samba-addc-mit-1")
1188 defaulttasks.remove("samba-addc-mit-4a")
1189 defaulttasks.remove("samba-addc-mit-4b")
1190
1191 defaulttasks.remove("samba-32bit")
1192
1193 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1194     defaulttasks.remove("samba-o3")
1195
1196
1197 def do_print(msg):
1198     logger.info(msg)
1199     sys.stdout.flush()
1200     sys.stderr.flush()
1201
1202 def do_debug(msg):
1203     logger.debug(msg)
1204     sys.stdout.flush()
1205     sys.stderr.flush()
1206
1207
1208 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1209     if show is None:
1210         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1211     elif show:
1212         do_print("Running: '%s' in '%s'" % (cmd, dir))
1213
1214     if output:
1215         out = check_output([cmd], shell=True, cwd=dir)
1216         return out.decode(encoding='utf-8', errors='backslashreplace')
1217     elif checkfail:
1218         return check_call(cmd, shell=True, cwd=dir)
1219     else:
1220         return call(cmd, shell=True, cwd=dir)
1221
1222 def rmdir_force(dirname, re_raise=True):
1223     try:
1224         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1225                 dirname, dirname, dirname), output=True, show=True)
1226     except CalledProcessError as e:
1227         do_print("Failed: '%s'" % (str(e)))
1228         run_cmd("tree %s" % dirname, output=True, show=True)
1229         if re_raise:
1230             raise
1231         return False
1232     return True
1233
1234 class builder(object):
1235     '''handle build of one directory'''
1236
1237     def __init__(self, name, definition):
1238         self.name = name
1239         self.dir = builddirs.get(name, '.')
1240         self.tag = self.name.replace('/', '_')
1241         self.definition = definition
1242         self.sequence = definition["sequence"]
1243         self.git_clone_required = False
1244         if "git-clone-required" in definition:
1245             self.git_clone_required = bool(definition["git-clone-required"])
1246         self.proc = None
1247         self.done = False
1248         self.next = 0
1249         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1250         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1251         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1252         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1253         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1254         self.stdout = open(self.stdout_path, 'w')
1255         self.stderr = open(self.stderr_path, 'w')
1256         self.stdin  = open("/dev/null", 'r')
1257         self.builder_dir = "%s/%s" % (testbase, self.tag)
1258         self.test_source_dir = self.builder_dir
1259         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1260         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1261         self.prefix = "%s/%s" % (test_prefix, self.tag)
1262         self.consumers = []
1263         self.producer = None
1264
1265         if self.git_clone_required:
1266             assert "dependency" not in definition
1267
1268     def mark_existing(self):
1269         do_debug('%s: Mark as existing dependency' % self.name)
1270         self.next = len(self.sequence)
1271         self.done = True
1272
1273     def add_consumer(self, consumer):
1274         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1275         consumer.producer = self
1276         consumer.test_source_dir = self.test_source_dir
1277         self.consumers.append(consumer)
1278
1279     def start_next(self):
1280         if self.producer is not None:
1281             if not self.producer.done:
1282                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1283                 return
1284
1285         if self.next == 0:
1286             rmdir_force(self.builder_dir)
1287             rmdir_force(self.prefix)
1288             if self.producer is not None:
1289                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1290             elif not self.git_clone_required:
1291                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1292             else:
1293                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1294
1295         if self.next == len(self.sequence):
1296             if not self.done:
1297                 do_print('%s: Completed OK' % self.name)
1298                 self.done = True
1299             if not options.nocleanup and len(self.consumers) == 0:
1300                 do_print('%s: Cleaning up' % self.name)
1301                 rmdir_force(self.builder_dir)
1302                 rmdir_force(self.prefix)
1303             for consumer in self.consumers:
1304                 if consumer.next != 0:
1305                     continue
1306                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1307                 consumer.start_next()
1308             if self.producer is not None:
1309                 self.producer.consumers.remove(self)
1310                 assert self.producer.done
1311                 self.producer.start_next()
1312             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1313             return
1314         (self.stage, self.cmd) = self.sequence[self.next]
1315         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1316                                     get_path(name='platlib',
1317                                              scheme="posix_prefix",
1318                                              vars={"base": self.prefix,
1319                                                    "platbase": self.prefix}))
1320         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1321         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1322         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1323         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1324         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1325         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1326         self.cmd = self.cmd.replace("${NAME}", self.name)
1327         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1328         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1329         self.proc = Popen(self.cmd, shell=True,
1330                           close_fds=True, cwd=self.cwd,
1331                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1332         self.next += 1
1333
1334 def expand_dependencies(n):
1335     deps = list()
1336     if "dependency" in tasks[n]:
1337         depname = tasks[n]["dependency"]
1338         assert depname in tasks
1339         sdeps = expand_dependencies(depname)
1340         assert n not in sdeps
1341         for sdep in sdeps:
1342             deps.append(sdep)
1343         deps.append(depname)
1344     return deps
1345
1346
1347 class buildlist(object):
1348     '''handle build of multiple directories'''
1349
1350     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1351         self.tail_proc = None
1352         self.retry = None
1353         if not tasknames:
1354             if options.restrict_tests:
1355                 tasknames = ["samba-test-only"]
1356             else:
1357                 tasknames = defaulttasks
1358
1359         given_tasknames = tasknames.copy()
1360         implicit_tasknames = []
1361         for n in given_tasknames:
1362             deps = expand_dependencies(n)
1363             for dep in deps:
1364                 if dep in given_tasknames:
1365                     continue
1366                 if dep in implicit_tasknames:
1367                     continue
1368                 implicit_tasknames.append(dep)
1369
1370         tasknames = implicit_tasknames.copy()
1371         tasknames.extend(given_tasknames)
1372         do_debug("given_tasknames: %s" % given_tasknames)
1373         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1374         do_debug("tasknames: %s" % tasknames)
1375         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1376
1377         if options.retry:
1378             rebase_remote = "rebaseon"
1379             retry_task = {
1380                     "git-clone-required": True,
1381                     "sequence": [
1382                             ("retry",
1383                             '''set -e
1384                             git remote add -t %s %s %s
1385                             git fetch %s
1386                             while :; do
1387                               sleep 60
1388                               git describe %s/%s > old_remote_branch.desc
1389                               git fetch %s
1390                               git describe %s/%s > remote_branch.desc
1391                               diff old_remote_branch.desc remote_branch.desc
1392                             done
1393                            ''' % (
1394                                rebase_branch, rebase_remote, rebase_url,
1395                                rebase_remote,
1396                                rebase_remote, rebase_branch,
1397                                rebase_remote,
1398                                rebase_remote, rebase_branch
1399                             ))]}
1400
1401             self.retry = builder('retry', retry_task)
1402             self.need_retry = False
1403
1404         if options.skip_dependencies:
1405             for b in self.tlist:
1406                 if b.name in implicit_tasknames:
1407                     b.mark_existing()
1408
1409         for b in self.tlist:
1410             do_debug("b.name=%s" % b.name)
1411             if "dependency" not in b.definition:
1412                 continue
1413             depname = b.definition["dependency"]
1414             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1415             for p in self.tlist:
1416                 if p.name == depname:
1417                     p.add_consumer(b)
1418
1419     def kill_kids(self):
1420         if self.tail_proc is not None:
1421             self.tail_proc.terminate()
1422             self.tail_proc.wait()
1423             self.tail_proc = None
1424         if self.retry is not None:
1425             self.retry.proc.terminate()
1426             self.retry.proc.wait()
1427             self.retry = None
1428         for b in self.tlist:
1429             if b.proc is not None:
1430                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1431                 b.proc.terminate()
1432                 b.proc.wait()
1433                 b.proc = None
1434
1435     def wait_one(self):
1436         while True:
1437             none_running = True
1438             for b in self.tlist:
1439                 if b.proc is None:
1440                     continue
1441                 none_running = False
1442                 b.status = b.proc.poll()
1443                 if b.status is None:
1444                     continue
1445                 b.proc = None
1446                 return b
1447             if options.retry:
1448                 ret = self.retry.proc.poll()
1449                 if ret is not None:
1450                     self.need_retry = True
1451                     self.retry = None
1452                     return None
1453             if none_running:
1454                 return None
1455             time.sleep(0.1)
1456
1457     def run(self):
1458         for b in self.tlist:
1459             b.start_next()
1460         if options.retry:
1461             self.retry.start_next()
1462         while True:
1463             b = self.wait_one()
1464             if options.retry and self.need_retry:
1465                 self.kill_kids()
1466                 do_print("retry needed")
1467                 return (0, None, None, None, "retry")
1468             if b is None:
1469                 break
1470             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1471                 self.kill_kids()
1472                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1473             b.start_next()
1474         self.kill_kids()
1475         return (0, None, None, None, "All OK")
1476
1477     def write_system_info(self, filename):
1478         with open(filename, 'w') as f:
1479             for cmd in ['uname -a',
1480                         'lsb_release -a',
1481                         'free',
1482                         'mount',
1483                         'cat /proc/cpuinfo',
1484                         'cc --version',
1485                         'df -m .',
1486                         'df -m %s' % testbase]:
1487                 try:
1488                     out = run_cmd(cmd, output=True, checkfail=False)
1489                 except CalledProcessError as e:
1490                     out = "<failed: %s>" % str(e)
1491                 print('### %s' % cmd, file=f)
1492                 print(out, file=f)
1493                 print(file=f)
1494
1495     def tarlogs(self, fname):
1496         with tarfile.open(fname, "w:gz") as tar:
1497             for b in self.tlist:
1498                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1499                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1500             if os.path.exists("autobuild.log"):
1501                 tar.add("autobuild.log")
1502             filename = 'system-info.txt'
1503             self.write_system_info(filename)
1504             tar.add(filename)
1505
1506     def remove_logs(self):
1507         for b in self.tlist:
1508             os.unlink(b.stdout_path)
1509             os.unlink(b.stderr_path)
1510
1511     def start_tail(self):
1512         cmd = ["tail", "-f"]
1513         for b in self.tlist:
1514             cmd.append(b.stdout_path)
1515             cmd.append(b.stderr_path)
1516         self.tail_proc = Popen(cmd, close_fds=True)
1517
1518
1519 def cleanup(do_raise=False):
1520     if options.nocleanup:
1521         return
1522     run_cmd("stat %s || true" % test_tmpdir, show=True)
1523     run_cmd("stat %s" % testbase, show=True)
1524     do_print("Cleaning up %r" % cleanup_list)
1525     for d in cleanup_list:
1526         ok = rmdir_force(d, re_raise=False)
1527         if ok:
1528             continue
1529         if os.path.isdir(d):
1530             do_print("Killing, waiting and retry")
1531             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1532         else:
1533             do_print("Waiting and retry")
1534         time.sleep(1)
1535         rmdir_force(d, re_raise=do_raise)
1536
1537
1538 def daemonize(logfile):
1539     pid = os.fork()
1540     if pid == 0:  # Parent
1541         os.setsid()
1542         pid = os.fork()
1543         if pid != 0:  # Actual daemon
1544             os._exit(0)
1545     else:  # Grandparent
1546         os._exit(0)
1547
1548     import resource      # Resource usage information.
1549     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1550     if maxfd == resource.RLIM_INFINITY:
1551         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1552     for fd in range(0, maxfd):
1553         try:
1554             os.close(fd)
1555         except OSError:
1556             pass
1557     os.open(logfile, os.O_RDWR | os.O_CREAT)
1558     os.dup2(0, 1)
1559     os.dup2(0, 2)
1560
1561
1562 def write_pidfile(fname):
1563     '''write a pid file, cleanup on exit'''
1564     with open(fname, mode='w') as f:
1565         f.write("%u\n" % os.getpid())
1566
1567
1568 def rebase_tree(rebase_url, rebase_branch="master"):
1569     rebase_remote = "rebaseon"
1570     do_print("Rebasing on %s" % rebase_url)
1571     run_cmd("git describe HEAD", show=True, dir=test_master)
1572     run_cmd("git remote add -t %s %s %s" %
1573             (rebase_branch, rebase_remote, rebase_url),
1574             show=True, dir=test_master)
1575     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1576     if options.fix_whitespace:
1577         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1578                 (rebase_remote, rebase_branch),
1579                 show=True, dir=test_master)
1580     else:
1581         run_cmd("git rebase --force-rebase %s/%s" %
1582                 (rebase_remote, rebase_branch),
1583                 show=True, dir=test_master)
1584     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1585                    (rebase_remote, rebase_branch),
1586                    dir=test_master, output=True)
1587     if diff == '':
1588         do_print("No differences between HEAD and %s/%s - exiting" %
1589                  (rebase_remote, rebase_branch))
1590         sys.exit(0)
1591     run_cmd("git describe %s/%s" %
1592             (rebase_remote, rebase_branch),
1593             show=True, dir=test_master)
1594     run_cmd("git describe HEAD", show=True, dir=test_master)
1595     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1596             (rebase_remote, rebase_branch),
1597             show=True, dir=test_master)
1598
1599
1600 def push_to(push_url, push_branch="master"):
1601     push_remote = "pushto"
1602     do_print("Pushing to %s" % push_url)
1603     if options.mark:
1604         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1605         run_cmd("git commit --amend -c HEAD", dir=test_master)
1606         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1607         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1608     run_cmd("git remote add -t %s %s %s" %
1609             (push_branch, push_remote, push_url),
1610             show=True, dir=test_master)
1611     run_cmd("git push %s +HEAD:%s" %
1612             (push_remote, push_branch),
1613             show=True, dir=test_master)
1614
1615
1616 def send_email(subject, text, log_tar):
1617     if options.email is None:
1618         do_print("not sending email because the recipient is not set")
1619         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1620                  (subject, text))
1621         return
1622     outer = MIMEMultipart()
1623     outer['Subject'] = subject
1624     outer['To'] = options.email
1625     outer['From'] = options.email_from
1626     outer['Date'] = email.utils.formatdate(localtime=True)
1627     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1628     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1629     if options.attach_logs:
1630         with open(log_tar, 'rb') as fp:
1631             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1632         # Set the filename parameter
1633         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1634         outer.attach(msg)
1635     content = outer.as_string()
1636     s = smtplib.SMTP(options.email_server)
1637     email_user = os.getenv('SMTP_USERNAME')
1638     email_password = os.getenv('SMTP_PASSWORD')
1639     if email_user is not None:
1640         s.starttls()
1641         s.login(email_user, email_password)
1642
1643     s.sendmail(options.email_from, [options.email], content)
1644     s.set_debuglevel(1)
1645     s.quit()
1646
1647
1648 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1649                   elapsed_time, log_base=None, add_log_tail=True):
1650     '''send an email to options.email about the failure'''
1651     elapsed_minutes = elapsed_time / 60.0
1652     if log_base is None:
1653         log_base = gitroot
1654     text = '''
1655 Dear Developer,
1656
1657 Your autobuild on %s failed after %.1f minutes
1658 when trying to test %s with the following error:
1659
1660    %s
1661
1662 the autobuild has been abandoned. Please fix the error and resubmit.
1663
1664 A summary of the autobuild process is here:
1665
1666   %s/autobuild.log
1667 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1668
1669     if options.restrict_tests:
1670         text += """
1671 The build was restricted to tests matching %s\n""" % options.restrict_tests
1672
1673     if failed_task != 'rebase':
1674         text += '''
1675 You can see logs of the failed task here:
1676
1677   %s/%s.stdout
1678   %s/%s.stderr
1679
1680 or you can get full logs of all tasks in this job here:
1681
1682   %s/logs.tar.gz
1683
1684 The top commit for the tree that was built was:
1685
1686 %s
1687
1688 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1689
1690     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1691     if add_log_tail and os.access(log_stdout, os.R_OK):
1692         f = open(log_stdout, 'r')
1693         lines = f.readlines()
1694         log_tail = "".join(lines[-50:])
1695         num_lines = len(lines)
1696         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1697         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1698             # Also include stderr (compile failures) if < 50 lines of stdout
1699             f = open(log_stderr, 'r')
1700             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1701
1702         text += '''
1703 The last 50 lines of log messages:
1704
1705 %s
1706     ''' % log_tail
1707         f.close()
1708
1709     logs = os.path.join(gitroot, 'logs.tar.gz')
1710     send_email('autobuild[%s] failure on %s for task %s during %s'
1711                % (options.branch, platform.node(), failed_task, failed_stage),
1712                text, logs)
1713
1714
1715 def email_success(elapsed_time, log_base=None):
1716     '''send an email to options.email about a successful build'''
1717     if log_base is None:
1718         log_base = gitroot
1719     text = '''
1720 Dear Developer,
1721
1722 Your autobuild on %s has succeeded after %.1f minutes.
1723
1724 ''' % (platform.node(), elapsed_time / 60.)
1725
1726     if options.restrict_tests:
1727         text += """
1728 The build was restricted to tests matching %s\n""" % options.restrict_tests
1729
1730     if options.keeplogs:
1731         text += '''
1732
1733 you can get full logs of all tasks in this job here:
1734
1735   %s/logs.tar.gz
1736
1737 ''' % log_base
1738
1739     text += '''
1740 The top commit for the tree that was built was:
1741
1742 %s
1743 ''' % top_commit_msg
1744
1745     logs = os.path.join(gitroot, 'logs.tar.gz')
1746     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1747                text, logs)
1748
1749
1750 # get the top commit message, for emails
1751 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1752
1753 try:
1754     if options.skip_dependencies:
1755         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1756     else:
1757         os.makedirs(testbase)
1758 except Exception as reason:
1759     raise Exception("Unable to create %s : %s" % (testbase, reason))
1760 cleanup_list.append(testbase)
1761
1762 if options.daemon:
1763     logfile = os.path.join(testbase, "log")
1764     do_print("Forking into the background, writing progress to %s" % logfile)
1765     daemonize(logfile)
1766
1767 write_pidfile(gitroot + "/autobuild.pid")
1768
1769 start_time = time.time()
1770
1771 while True:
1772     try:
1773         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1774         os.makedirs(test_tmpdir)
1775         # The waf uninstall code removes empty directories all the way
1776         # up the tree.  Creating a file in test_tmpdir stops it from
1777         # being removed.
1778         run_cmd("touch %s" % os.path.join(test_tmpdir,
1779                                           ".directory-is-not-empty"), show=True)
1780         run_cmd("stat %s" % test_tmpdir, show=True)
1781         run_cmd("stat %s" % testbase, show=True)
1782         if options.skip_dependencies:
1783             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1784         else:
1785             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1786     except Exception:
1787         cleanup()
1788         raise
1789
1790     try:
1791         if options.rebase is not None:
1792             rebase_tree(options.rebase, rebase_branch=options.branch)
1793     except Exception:
1794         cleanup_list.append(gitroot + "/autobuild.pid")
1795         cleanup()
1796         elapsed_time = time.time() - start_time
1797         email_failure(-1, 'rebase', 'rebase', 'rebase',
1798                       'rebase on %s failed' % options.branch,
1799                       elapsed_time, log_base=options.log_base)
1800         sys.exit(1)
1801
1802     try:
1803         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1804         if options.tail:
1805             blist.start_tail()
1806         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1807         if status != 0 or errstr != "retry":
1808             break
1809         cleanup(do_raise=True)
1810     except Exception:
1811         cleanup()
1812         raise
1813
1814 cleanup_list.append(gitroot + "/autobuild.pid")
1815
1816 do_print(errstr)
1817
1818 blist.kill_kids()
1819 if options.tail:
1820     do_print("waiting for tail to flush")
1821     time.sleep(1)
1822
1823 elapsed_time = time.time() - start_time
1824 if status == 0:
1825     if options.passcmd is not None:
1826         do_print("Running passcmd: %s" % options.passcmd)
1827         run_cmd(options.passcmd, dir=test_master)
1828     if options.pushto is not None:
1829         push_to(options.pushto, push_branch=options.branch)
1830     if options.keeplogs or options.attach_logs:
1831         blist.tarlogs("logs.tar.gz")
1832         do_print("Logs in logs.tar.gz")
1833     if options.always_email:
1834         email_success(elapsed_time, log_base=options.log_base)
1835     blist.remove_logs()
1836     cleanup()
1837     do_print(errstr)
1838     sys.exit(0)
1839
1840 # something failed, gather a tar of the logs
1841 blist.tarlogs("logs.tar.gz")
1842
1843 if options.email is not None:
1844     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1845                   elapsed_time, log_base=options.log_base)
1846 else:
1847     elapsed_minutes = elapsed_time / 60.0
1848     print('''
1849
1850 ####################################################################
1851
1852 AUTOBUILD FAILURE
1853
1854 Your autobuild[%s] on %s failed after %.1f minutes
1855 when trying to test %s with the following error:
1856
1857    %s
1858
1859 the autobuild has been abandoned. Please fix the error and resubmit.
1860
1861 ####################################################################
1862
1863 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1864
1865 cleanup()
1866 do_print(errstr)
1867 do_print("Logs in logs.tar.gz")
1868 sys.exit(status)