script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 # allow autobuild to run within git rebase -i
  35 if "GIT_DIR" in os.environ:
  36     del os.environ["GIT_DIR"]
  37 if "GIT_WORK_TREE" in os.environ:
  38     del os.environ["GIT_WORK_TREE"]
  39
  40 def find_git_root():
  41     '''get to the top of the git repo'''
  42     p = os.getcwd()
  43     while p != '/':
  44         if os.path.exists(os.path.join(p, ".git")):
  45             return p
  46         p = os.path.abspath(os.path.join(p, '..'))
  47     return None
  48
  49
  50 gitroot = find_git_root()
  51 if gitroot is None:
  52     raise Exception("Failed to find git root")
  53
  54
  55 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  56
  57 parser = OptionParser()
  58 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  59 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  60 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  61 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  62 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  63                   default=def_testbase)
  64 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  65                   default=None)
  66 parser.add_option("--passcmd", help="command to run on success", default=None)
  67 parser.add_option("--verbose", help="show all commands as they are run",
  68                   default=False, action="store_true")
  69 parser.add_option("--rebase", help="rebase on the given tree before testing",
  70                   default=None, type='str')
  71 parser.add_option("--pushto", help="push to a git url on success",
  72                   default=None, type='str')
  73 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  74                   default=False, action="store_true")
  75 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  76                   default=False, action="store_true")
  77 parser.add_option("--retry", help="automatically retry if master changes",
  78                   default=False, action="store_true")
  79 parser.add_option("--email", help="send email to the given address on failure",
  80                   type='str', default=None)
  81 parser.add_option("--email-from", help="send email from the given address",
  82                   type='str', default="autobuild@samba.org")
  83 parser.add_option("--email-server", help="send email via the given server",
  84                   type='str', default='localhost')
  85 parser.add_option("--always-email", help="always send email, even on success",
  86                   action="store_true")
  87 parser.add_option("--daemon", help="daemonize after initial setup",
  88                   action="store_true")
  89 parser.add_option("--branch", help="the branch to work on (default=master)",
  90                   default="master", type='str')
  91 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  92                   default=gitroot, type='str')
  93 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  94                   default=False, action="store_true")
  95 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  96                   default='')
  97 parser.add_option("--enable-coverage", dest='enable_coverage',
  98                   action="store_const", const='--enable-coverage', default='',
  99                   help="Add --enable-coverage option while configure")
 100
 101 (options, args) = parser.parse_args()
 102
 103 if options.retry:
 104     if options.rebase is None:
 105         raise Exception('You can only use --retry if you also rebase')
 106
 107 if options.full_testbase is not None:
 108     testbase = options.full_testbase
 109 else:
 110     testbase = "%s/b%u" % (options.testbase, os.getpid())
 111 test_master = "%s/master" % testbase
 112 test_prefix = "%s/prefix" % testbase
 113 test_tmpdir = "%s/tmp" % testbase
 114 os.environ['TMPDIR'] = test_tmpdir
 115
 116 if options.enable_coverage:
 117     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 118 else:
 119     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 120
 121 if options.enable_coverage:
 122     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 123 else:
 124     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 125
 126 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 127
 128
 129 def check_symbols(sofile, expected_symbols=""):
 130     return "objdump --dynamic-syms " + sofile + " | " + \
 131            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 132
 133 if args:
 134     # If we are only running specific test,
 135     # do not sleep randomly to wait for it to start
 136     def random_sleep(low, high):
 137         return 'sleep 1'
 138 else:
 139     def random_sleep(low, high):
 140         return 'sleep {}'.format(random.randint(low, high))
 141
 142 cleanup_list = []
 143
 144 builddirs = {
 145     "ctdb": "ctdb",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl",
 152     "docs-xml": "docs-xml"
 153 }
 154
 155 ctdb_configure_params = " --enable-developer ${PREFIX}"
 156 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 157
 158 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 159 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 160 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 161 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 162 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 163 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 164 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 165
 166
 167 def format_option(name, value=None):
 168     """Format option as str list."""
 169     if value is None:  # boolean option
 170         return [name]
 171     if not isinstance(value, list):  # single value option
 172         value = [value]
 173     # repeatable option
 174     return ['{}={}'.format(name, item) for item in value]
 175
 176
 177 def make_test(
 178         cmd='make testonly',
 179         INJECT_SELFTEST_PREFIX=1,
 180         TESTS='',
 181         include_envs=None,
 182         exclude_envs=None):
 183
 184     test_options = []
 185     if include_envs:
 186         test_options = format_option('--include-env', include_envs)
 187     if exclude_envs:
 188         test_options = format_option('--exclude-env', exclude_envs)
 189     if test_options:
 190         # join envs options to original test options
 191         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 192
 193     _options = []
 194
 195     # Allow getting a full CI with
 196     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 197
 198     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 199
 200     if int(FAIL_IMMEDIATELY):
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     if INJECT_SELFTEST_PREFIX:
 206         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 207         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 208
 209     return ' '.join([cmd] + _options)
 210
 211
 212 # When updating this list, also update .gitlab-ci.yml to add the job
 213 # and to make it a dependency of 'page' for the coverage report.
 214
 215 tasks = {
 216     "ctdb": {
 217         "sequence": [
 218             ("random-sleep", random_sleep(300, 900)),
 219             ("configure", "./configure " + ctdb_configure_params),
 220             ("make", "make all"),
 221             ("install", "make install"),
 222             ("test", "make autotest"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("clean", "make clean"),
 225         ],
 226     },
 227     "docs-xml": {
 228         "sequence": [
 229             ("random-sleep", random_sleep(300, 900)),
 230             ("autoconf", "autoconf"),
 231             ("configure", "./configure"),
 232             ("make", "make html htmlman"),
 233             ("publish-docs", PUBLISH_DOCS),
 234             ("clean", "make clean"),
 235         ],
 236     },
 237
 238     "samba-def-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-mit-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-nt4-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     "samba-h5l-build": {
 269         "git-clone-required": True,
 270         "sequence": [
 271             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 272             ("make", "make -j"),
 273             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 274             ("chmod-R-a-w", "chmod -R a-w ."),
 275         ],
 276     },
 277
 278     "samba-without-smb1-build": {
 279         "git-clone-required": True,
 280         "sequence": [
 281             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 282             ("make", "make -j"),
 283             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 284             ("chmod-R-a-w", "chmod -R a-w ."),
 285         ],
 286     },
 287
 288     "samba-no-opath-build": {
 289         "git-clone-required": True,
 290         "sequence": [
 291             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 292             ("make", "make -j"),
 293             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 294             ("chmod-R-a-w", "chmod -R a-w ."),
 295         ],
 296     },
 297
 298     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 299     "samba": {
 300         "sequence": [
 301             ("random-sleep", random_sleep(300, 900)),
 302             ("configure", "./configure.developer" + samba_configure_params),
 303             ("make", "make -j"),
 304             ("test", make_test(exclude_envs=[
 305             "none",
 306             "nt4_dc",
 307             "nt4_dc_smb1",
 308             "nt4_dc_smb1_done",
 309             "nt4_dc_schannel",
 310             "nt4_member",
 311             "ad_dc",
 312             "ad_dc_smb1",
 313             "ad_dc_smb1_done",
 314             "ad_dc_backup",
 315             "ad_dc_ntvfs",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_slowtests",
 319             "ad_dc_no_nss",
 320             "ad_dc_no_ntlm",
 321             "fl2003dc",
 322             "fl2008dc",
 323             "fl2008r2dc",
 324             "ad_member",
 325             "ad_member_idmap_rid",
 326             "admem_idmap_autorid",
 327             "ad_member_idmap_ad",
 328             "ad_member_rfc2307",
 329             "ad_member_oneway",
 330             "chgdcpass",
 331             "vampire_2000_dc",
 332             "fl2000dc",
 333             "fileserver",
 334             "fileserver_smb1",
 335             "fileserver_smb1_done",
 336             "maptoguest",
 337             "simpleserver",
 338             "backupfromdc",
 339             "restoredc",
 340             "renamedc",
 341             "offlinebackupdc",
 342             "labdc",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             "promoted_dc",
 346             "vampire_dc",
 347             "rodc",
 348             "ad_dc_default",
 349             "ad_dc_default_smb1",
 350             "ad_dc_default_smb1_done",
 351             "ad_dc_slowtests",
 352             "schema_pair_dc",
 353             "schema_dc",
 354             "clusteredmember",
 355             "ad_dc_fips",
 356             "ad_member_fips",
 357             ])),
 358             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 359             ("lcov", LCOV_CMD),
 360             ("install", "make install"),
 361             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 362             ("clean", "make clean"),
 363         ],
 364     },
 365
 366     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 367     "samba-mitkrb5": {
 368         "sequence": [
 369             ("random-sleep", random_sleep(300, 900)),
 370             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 371             ("make", "make -j"),
 372             ("test", make_test(exclude_envs=[
 373             "none",
 374             "nt4_dc",
 375             "nt4_dc_smb1",
 376             "nt4_dc_smb1_done",
 377             "nt4_dc_schannel",
 378             "nt4_member",
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_backup",
 383             "ad_dc_ntvfs",
 384             "ad_dc_default",
 385             "ad_dc_default_smb1",
 386             "ad_dc_default_smb1_done",
 387             "ad_dc_slowtests",
 388             "ad_dc_no_nss",
 389             "ad_dc_no_ntlm",
 390             "fl2003dc",
 391             "fl2008dc",
 392             "fl2008r2dc",
 393             "ad_member",
 394             "ad_member_idmap_rid",
 395             "admem_idmap_autorid",
 396             "ad_member_idmap_ad",
 397             "ad_member_rfc2307",
 398             "ad_member_oneway",
 399             "chgdcpass",
 400             "vampire_2000_dc",
 401             "fl2000dc",
 402             "fileserver",
 403             "fileserver_smb1",
 404             "fileserver_smb1_done",
 405             "maptoguest",
 406             "simpleserver",
 407             "backupfromdc",
 408             "restoredc",
 409             "renamedc",
 410             "offlinebackupdc",
 411             "labdc",
 412             "preforkrestartdc",
 413             "proclimitdc",
 414             "promoted_dc",
 415             "vampire_dc",
 416             "rodc",
 417             "ad_dc_default",
 418             "ad_dc_default_smb1",
 419             "ad_dc_default_smb1_done",
 420             "ad_dc_slowtests",
 421             "schema_pair_dc",
 422             "schema_dc",
 423             "clusteredmember",
 424             "ad_dc_fips",
 425             "ad_member_fips",
 426             ])),
 427             ("lcov", LCOV_CMD),
 428             ("install", "make install"),
 429             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 430             ("clean", "make clean"),
 431         ],
 432     },
 433
 434     "samba-nt4": {
 435         "dependency": "samba-nt4-build",
 436         "sequence": [
 437             ("random-sleep", random_sleep(300, 900)),
 438             ("test", make_test(include_envs=[
 439             "nt4_dc",
 440             "nt4_dc_smb1",
 441             "nt4_dc_smb1_done",
 442             "nt4_dc_schannel",
 443             "nt4_member",
 444             "simpleserver",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-fileserver": {
 452         "dependency": "samba-h5l-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(include_envs=[
 456             "fileserver",
 457             "fileserver_smb1",
 458             "fileserver_smb1_done",
 459             "maptoguest",
 460             "ktest", # ktest is also tested in samba-ktest-mit samba
 461                      # and samba-mitkrb5 but is tested here against
 462                      # a system Heimdal
 463             ])),
 464             ("lcov", LCOV_CMD),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466         ],
 467     },
 468
 469     "samba-fileserver-without-smb1": {
 470         "dependency": "samba-without-smb1-build",
 471         "sequence": [
 472             ("random-sleep", random_sleep(300, 900)),
 473             ("test", make_test(include_envs=["fileserver"])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     # This is a full build without the AD DC so we test the build with
 480     # MIT Kerberos from the current system.  Runtime behaviour is
 481     # confirmed via the ktest (static ccache and keytab) environment
 482
 483     # This environment also used to confirm we can still build with --with-libunwind
 484     "samba-ktest-mit": {
 485         "sequence": [
 486             ("random-sleep", random_sleep(300, 900)),
 487             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 488             ("make", "make -j"),
 489             ("test", make_test(include_envs=[
 490             "ktest", # ktest is also tested in fileserver, samba and
 491                      # samba-mitkrb5 but is tested here against a
 492                      # system MIT krb5
 493             ])),
 494             ("lcov", LCOV_CMD),
 495             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 496         ],
 497     },
 498
 499     "samba-admem": {
 500         "dependency": "samba-def-build",
 501         "sequence": [
 502             ("random-sleep", random_sleep(300, 900)),
 503             ("test", make_test(include_envs=[
 504             "ad_member",
 505             "ad_member_idmap_rid",
 506             "admem_idmap_autorid",
 507             "ad_member_idmap_ad",
 508             "ad_member_rfc2307",
 509             "ad_member_offlogon",
 510             ])),
 511             ("lcov", LCOV_CMD),
 512             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 513         ],
 514     },
 515
 516     "samba-no-opath1": {
 517         "dependency": "samba-no-opath-build",
 518         "sequence": [
 519             ("random-sleep", random_sleep(300, 900)),
 520             ("test", make_test(
 521                 cmd="make testonly DISABLE_OPATH=1",
 522                 include_envs=[
 523                 "nt4_dc",
 524                 "nt4_dc_smb1",
 525                 "nt4_dc_smb1_done",
 526                 "nt4_dc_schannel",
 527                 "nt4_member",
 528                 "simpleserver",
 529                 ])),
 530             ("lcov", LCOV_CMD),
 531             ("check-clean-tree", "script/clean-source-tree.sh"),
 532         ],
 533     },
 534
 535     "samba-no-opath2": {
 536         "dependency": "samba-no-opath-build",
 537         "sequence": [
 538             ("random-sleep", random_sleep(300, 900)),
 539             ("test", make_test(
 540                 cmd="make testonly DISABLE_OPATH=1",
 541                 include_envs=[
 542                 "fileserver",
 543                 "fileserver_smb1",
 544                 "fileserver_smb1_done",
 545                 ])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", "script/clean-source-tree.sh"),
 548         ],
 549     },
 550
 551     "samba-ad-dc-1": {
 552         "dependency": "samba-def-build",
 553         "sequence": [
 554             ("random-sleep", random_sleep(1, 1)),
 555             ("test", make_test(include_envs=[
 556             "ad_dc",
 557             "ad_dc_smb1",
 558             "ad_dc_smb1_done",
 559             "ad_dc_no_nss",
 560             "ad_dc_no_ntlm",
 561             ])),
 562             ("lcov", LCOV_CMD),
 563             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 564         ],
 565     },
 566
 567     "samba-ad-dc-2": {
 568         "dependency": "samba-def-build",
 569         "sequence": [
 570             ("random-sleep", random_sleep(1, 1)),
 571             ("test", make_test(include_envs=[
 572             "vampire_dc",
 573             "vampire_2000_dc",
 574             "rodc",
 575             ])),
 576             ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     "samba-ad-dc-3": {
 582         "dependency": "samba-def-build",
 583         "sequence": [
 584             ("random-sleep", random_sleep(1, 1)),
 585             ("test", make_test(include_envs=[
 586             "promoted_dc",
 587             "chgdcpass",
 588             "preforkrestartdc",
 589             "proclimitdc",
 590             ])),
 591             ("lcov", LCOV_CMD),
 592             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 593         ],
 594     },
 595
 596     "samba-ad-dc-4a": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=[
 601             "fl2000dc",
 602             "ad_member_oneway",
 603             "fl2003dc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609     "samba-ad-dc-4b": {
 610         "dependency": "samba-def-build",
 611         "sequence": [
 612             ("random-sleep", random_sleep(1, 1)),
 613             ("test", make_test(include_envs=[
 614             "fl2008dc",
 615             "fl2008r2dc",
 616             ])),
 617             ("lcov", LCOV_CMD),
 618             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 619         ],
 620     },
 621
 622     "samba-ad-dc-5": {
 623         "dependency": "samba-def-build",
 624         "sequence": [
 625             ("random-sleep", random_sleep(1, 1)),
 626             ("test", make_test(include_envs=[
 627             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-6": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-schemaupgrade": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 654     # This is currently the longest task, so we don't randomly delay it.
 655     "samba-ad-dc-ntvfs": {
 656         "dependency": "samba-def-build",
 657         "sequence": [
 658             ("random-sleep", random_sleep(1, 1)),
 659             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 660             ("lcov", LCOV_CMD),
 661             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 662         ],
 663     },
 664
 665     # Test fips compliance
 666     "samba-fips": {
 667         "dependency": "samba-mit-build",
 668         "sequence": [
 669             ("random-sleep", random_sleep(1, 1)),
 670             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 671             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 672             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 673         ],
 674     },
 675
 676     # run the backup/restore testenvs separately as they're fairly standalone
 677     # (and CI seems to max out at ~3 different DCs running at once)
 678     "samba-ad-back1": {
 679         "dependency": "samba-def-build",
 680         "sequence": [
 681             ("random-sleep", random_sleep(300, 900)),
 682             ("test", make_test(include_envs=[
 683             "backupfromdc",
 684             "restoredc",
 685             "renamedc",
 686             ])),
 687             ("lcov", LCOV_CMD),
 688             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 689         ],
 690     },
 691     "samba-ad-back2": {
 692         "dependency": "samba-def-build",
 693         "sequence": [
 694             ("random-sleep", random_sleep(300, 900)),
 695             ("test", make_test(include_envs=[
 696             "backupfromdc",
 697             "offlinebackupdc",
 698             "labdc",
 699             ])),
 700             ("lcov", LCOV_CMD),
 701             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 702         ],
 703     },
 704
 705     "samba-admem-mit": {
 706         "dependency": "samba-mit-build",
 707         "sequence": [
 708             ("random-sleep", random_sleep(1, 1)),
 709             ("test", make_test(include_envs=[
 710             "ad_member",
 711             "ad_member_idmap_rid",
 712             "admem_idmap_autorid",
 713             "ad_member_idmap_ad",
 714             "ad_member_rfc2307",
 715             "ad_member_offlogon",
 716             ])),
 717             ("lcov", LCOV_CMD),
 718             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 719         ],
 720     },
 721
 722     "samba-addc-mit-1": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "ad_dc",
 728             "ad_dc_smb1",
 729             "ad_dc_smb1_done",
 730             "ad_dc_no_nss",
 731             "ad_dc_no_ntlm",
 732             ])),
 733             ("lcov", LCOV_CMD),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735         ],
 736     },
 737
 738     "samba-addc-mit-4a": {
 739         "dependency": "samba-mit-build",
 740         "sequence": [
 741             ("random-sleep", random_sleep(1, 1)),
 742             ("test", make_test(include_envs=[
 743             "fl2000dc",
 744             "ad_member_oneway",
 745             "fl2003dc",
 746             ])),
 747             ("lcov", LCOV_CMD),
 748             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 749         ],
 750     },
 751     "samba-addc-mit-4b": {
 752         "dependency": "samba-mit-build",
 753         "sequence": [
 754             ("random-sleep", random_sleep(1, 1)),
 755             ("test", make_test(include_envs=[
 756             "fl2008dc",
 757             "fl2008r2dc",
 758             ])),
 759             ("lcov", LCOV_CMD),
 760             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 761         ],
 762     },
 763
 764     "samba-test-only": {
 765         "sequence": [
 766             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 767             ("make", "make -j"),
 768             ("test", make_test(TESTS="${TESTS}")),
 769             ("lcov", LCOV_CMD),
 770         ],
 771     },
 772
 773     # Test cross-compile infrastructure
 774     "samba-xc": {
 775         "sequence": [
 776             ("random-sleep", random_sleep(900, 1500)),
 777             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 778             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 779             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 780             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 781             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 782             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 783             ("compare-results", "script/compare_cc_results.py "
 784             "./bin/c4che/default{} "
 785             "./bin-xe/c4che/default{} "
 786             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 787             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 788             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 789             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 790             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 791             " = \"'1234'\"".format(CACHE_SUFFIX)),
 792             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 793             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 794             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 795             " ; test $? -ne 0"),
 796         ],
 797     },
 798
 799     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 800     "samba-o3": {
 801         "sequence": [
 802             ("random-sleep", random_sleep(300, 900)),
 803             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 804             ("make", "make -j"),
 805             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 806             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 807             ("lcov", LCOV_CMD),
 808             ("install", "make install"),
 809             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 810             ("clean", "make clean"),
 811         ],
 812     },
 813
 814     "samba-32bit": {
 815         "sequence": [
 816             ("random-sleep", random_sleep(300, 900)),
 817             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 818             ("make", "make -j"),
 819             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 820             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 821             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 822             ("install", "make install"),
 823             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 824             ("clean", "make clean"),
 825         ],
 826     },
 827
 828     "samba-ctdb": {
 829         "sequence": [
 830             ("random-sleep", random_sleep(900, 1500)),
 831
 832         # make sure we have tdb around:
 833             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 834             ("tdb-make", "cd lib/tdb && make"),
 835             ("tdb-install", "cd lib/tdb && make install"),
 836
 837         # build samba with cluster support (also building ctdb):
 838             ("samba-configure",
 839          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 840          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 841          "./configure.developer ${PREFIX} "
 842          "--with-selftest-prefix=./bin/ab "
 843          "--with-cluster-support "
 844          "--without-ad-dc "
 845          "--bundled-libraries=!tdb"),
 846             ("samba-make", "make"),
 847             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 848             ("samba-install", "make install"),
 849             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 850
 851             ("test", make_test(
 852                 cmd='make test',
 853                 INJECT_SELFTEST_PREFIX=0,
 854                 include_envs=["clusteredmember"])
 855             ),
 856
 857         # clean up:
 858             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 859             ("clean", "make clean"),
 860             ("ctdb-clean", "cd ./ctdb && make clean"),
 861         ],
 862     },
 863
 864     "samba-libs": {
 865         "sequence": [
 866             ("random-sleep", random_sleep(300, 900)),
 867             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 868             ("talloc-make", "cd lib/talloc && make"),
 869             ("talloc-install", "cd lib/talloc && make install"),
 870
 871             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 872             ("tdb-make", "cd lib/tdb && make"),
 873             ("tdb-install", "cd lib/tdb && make install"),
 874
 875             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 876             ("tevent-make", "cd lib/tevent && make"),
 877             ("tevent-install", "cd lib/tevent && make install"),
 878
 879             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 880             ("ldb-make", "cd lib/ldb && make"),
 881             ("ldb-install", "cd lib/ldb && make install"),
 882
 883             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 884             ("nondevel-make", "make -j"),
 885             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 886             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 887             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 888             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 889             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 890             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 891             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 892             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 893             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 894             ("nondevel-no-public-nss_winbind",
 895                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 896             ("nondevel-no-public-nss_wins",
 897                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 898             ("nondevel-no-public-libwbclient",
 899                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 900             ("nondevel-no-public-pam_winbind",
 901                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 902             ("nondevel-no-public-winbind_krb5_locator",
 903                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 904             ("nondevel-no-public-async_dns_krb5_locator",
 905                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 906             ("nondevel-install", "make -j install"),
 907             ("nondevel-dist", "make dist"),
 908
 909             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 910             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 911             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 912             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 913             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 914             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 915             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 916             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 917             ("prefix-no-public-nss_winbind",
 918                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 919             ("prefix-no-public-nss_wins",
 920                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 921             ("prefix-no-public-libwbclient",
 922                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 923             ("prefix-no-public-pam_winbind",
 924                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 925             ("prefix-no-public-winbind_krb5_locator",
 926                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 927                               "service_locator")),
 928             ("prefix-no-public-async_dns_krb5_locator",
 929                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 930                               "service_locator")),
 931
 932             # retry with all modules shared
 933             ("allshared-distclean", "make distclean"),
 934             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 935             ("allshared-make", "make -j"),
 936             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 937             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 938             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 939             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 940             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 941             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 942             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 943             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 944             ("allshared-no-public-nss_winbind",
 945                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 946             ("allshared-no-public-nss_wins",
 947                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 948             ("allshared-no-public-libwbclient",
 949                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 950             ("allshared-no-public-pam_winbind",
 951                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 952             ("allshared-no-public-winbind_krb5_locator",
 953                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 954             ("allshared-no-public-async_dns_krb5_locator",
 955                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 956         ],
 957     },
 958
 959     "samba-fuzz": {
 960         "sequence": [
 961         # build the fuzzers (static) via the oss-fuzz script
 962             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 963             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 964         ],
 965     },
 966
 967     # * Test smbd and smbtorture can build semi-static
 968     #
 969     # * Test Samba without python still builds.
 970     #
 971     # When this test fails due to more use of Python, the expectations
 972     # is that the newly failing part of the code should be disabled
 973     # when --disable-python is set (rather than major work being done
 974     # to support this environment).
 975     #
 976     # The target here is for vendors shipping a minimal smbd.
 977     "samba-minimal-smbd": {
 978         "sequence": [
 979             ("random-sleep", random_sleep(300, 900)),
 980
 981         # build with all modules static
 982             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 983             ("allstatic-make", "make -j"),
 984             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 985             ("allstatic-lcov", LCOV_CMD),
 986
 987         # retry with nonshared smbd and smbtorture
 988             ("nonshared-distclean", "make distclean"),
 989             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 990             ("nonshared-make", "make -j"),
 991             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 992             # TODO ("nonshared-lcov", LCOV_CMD),
 993
 994             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 995             ("clean", "make clean"),
 996         ],
 997     },
 998
 999     "samba-nopython": {
1000         "sequence": [
1001             ("random-sleep", random_sleep(300, 900)),
1002
1003             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1004             ("make", "make -j"),
1005             ("find-python", "script/find_python.sh ${PREFIX}"),
1006             ("test", "make test-nopython"),
1007             ("lcov", LCOV_CMD),
1008             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1009             ("clean", "make clean"),
1010
1011             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1012             ("talloc-make", "cd lib/talloc && make"),
1013             ("talloc-install", "cd lib/talloc && make install"),
1014
1015             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1016             ("tdb-make", "cd lib/tdb && make"),
1017             ("tdb-install", "cd lib/tdb && make install"),
1018
1019             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1020             ("tevent-make", "cd lib/tevent && make"),
1021             ("tevent-install", "cd lib/tevent && make install"),
1022
1023             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1024             ("ldb-make", "cd lib/ldb && make"),
1025             ("ldb-install", "cd lib/ldb && make install"),
1026
1027         # retry against installed library packages, but no required modules
1028             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1029             ("libs-make", "make -j"),
1030             ("libs-install", "make install"),
1031             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1032             ("libs-clean", "make clean"),
1033
1034         ],
1035     },
1036
1037     "samba-shellcheck": {
1038         "sequence": [
1039             ("run", "script/check-shell-scripts.sh ."),
1040         ],
1041     },
1042
1043     "ldb": {
1044         "sequence": [
1045             ("random-sleep", random_sleep(60, 600)),
1046             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1047             ("make", "make"),
1048             ("install", "make install"),
1049             ("test", "make test"),
1050             ("lcov", LCOV_CMD),
1051             ("clean", "make clean"),
1052             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1053             ("make-no-lmdb", "make"),
1054             ("test-no-lmdb", "make test"),
1055             ("lcov-no-lmdb", LCOV_CMD),
1056             ("install-no-lmdb", "make install"),
1057             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1058             ("distcheck", "make distcheck"),
1059             ("clean", "make clean"),
1060         ],
1061     },
1062
1063     "tdb": {
1064         "sequence": [
1065             ("random-sleep", random_sleep(60, 600)),
1066             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1067             ("make", "make"),
1068             ("install", "make install"),
1069             ("test", "make test"),
1070             ("lcov", LCOV_CMD),
1071             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1072             ("distcheck", "make distcheck"),
1073             ("clean", "make clean"),
1074         ],
1075     },
1076
1077     "talloc": {
1078         "sequence": [
1079             ("random-sleep", random_sleep(60, 600)),
1080             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1081             ("make", "make"),
1082             ("install", "make install"),
1083             ("test", "make test"),
1084             ("lcov", LCOV_CMD),
1085             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1086             ("distcheck", "make distcheck"),
1087             ("clean", "make clean"),
1088         ],
1089     },
1090
1091     "replace": {
1092         "sequence": [
1093             ("random-sleep", random_sleep(60, 600)),
1094             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1095             ("make", "make"),
1096             ("install", "make install"),
1097             ("test", "make test"),
1098             ("lcov", LCOV_CMD),
1099             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1100             ("distcheck", "make distcheck"),
1101             ("clean", "make clean"),
1102         ],
1103     },
1104
1105     "tevent": {
1106         "sequence": [
1107             ("random-sleep", random_sleep(60, 600)),
1108             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1109             ("make", "make"),
1110             ("install", "make install"),
1111             ("test", "make test"),
1112             ("lcov", LCOV_CMD),
1113             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1114             ("distcheck", "make distcheck"),
1115             ("clean", "make clean"),
1116         ],
1117     },
1118
1119     "pidl": {
1120         "git-clone-required": True,
1121         "sequence": [
1122             ("random-sleep", random_sleep(60, 600)),
1123             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1124             ("touch", "touch *.yp"),
1125             ("make", "make"),
1126             ("test", "make test"),
1127             ("install", "make install"),
1128             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1129             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1130             ("clean", "make clean"),
1131         ],
1132     },
1133
1134     # these are useful for debugging autobuild
1135     "pass": {
1136         "sequence": [
1137             ("pass", 'echo passing && /bin/true'),
1138         ],
1139     },
1140     "fail": {
1141         "sequence": [
1142             ("fail", 'echo failing && /bin/false'),
1143         ],
1144     },
1145 }
1146
1147 defaulttasks = list(tasks.keys())
1148
1149 defaulttasks.remove("pass")
1150 defaulttasks.remove("fail")
1151
1152 # The build tasks will be brought in by the test tasks as needed
1153 defaulttasks.remove("samba-def-build")
1154 defaulttasks.remove("samba-nt4-build")
1155 defaulttasks.remove("samba-mit-build")
1156 defaulttasks.remove("samba-h5l-build")
1157 defaulttasks.remove("samba-no-opath-build")
1158
1159 # This is not a normal test, but a task to support manually running
1160 # one test under autobuild
1161 defaulttasks.remove("samba-test-only")
1162
1163 # Only built on GitLab CI and not in the default autobuild because it
1164 # uses too much space (4GB of semi-static binaries)
1165 defaulttasks.remove("samba-fuzz")
1166
1167 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1168 # container where a simulated FIPS mode is possible.
1169 defaulttasks.remove("samba-fips")
1170
1171 # The MIT build runs on a current Fedora where an up to date MIT KDC
1172 # is already packaged.  This avoids needing to backport a current MIT
1173 # to the default Ubuntu 18.04, particularly during development, and
1174 # the need to install on the shared sn-devel-184.
1175
1176 defaulttasks.remove("samba-mitkrb5")
1177 defaulttasks.remove("samba-admem-mit")
1178 defaulttasks.remove("samba-addc-mit-1")
1179 defaulttasks.remove("samba-addc-mit-4a")
1180 defaulttasks.remove("samba-addc-mit-4b")
1181
1182 defaulttasks.remove("samba-32bit")
1183
1184 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1185     defaulttasks.remove("samba-o3")
1186
1187
1188 def do_print(msg):
1189     print("%s" % msg)
1190     sys.stdout.flush()
1191     sys.stderr.flush()
1192
1193
1194 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1195     if show is None:
1196         show = options.verbose
1197     if show:
1198         do_print("Running: '%s' in '%s'" % (cmd, dir))
1199     if output:
1200         out = check_output([cmd], shell=True, cwd=dir)
1201         return out.decode(encoding='utf-8', errors='backslashreplace')
1202     elif checkfail:
1203         return check_call(cmd, shell=True, cwd=dir)
1204     else:
1205         return call(cmd, shell=True, cwd=dir)
1206
1207 def rmdir_force(dirname, re_raise=True):
1208     try:
1209         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1210                 dirname, dirname, dirname), output=True, show=True)
1211     except CalledProcessError as e:
1212         do_print("Failed: '%s'" % (str(e)))
1213         run_cmd("tree %s" % dirname, output=True, show=True)
1214         if re_raise:
1215             raise
1216         return False
1217     return True
1218
1219 class builder(object):
1220     '''handle build of one directory'''
1221
1222     def __init__(self, name, definition):
1223         self.name = name
1224         self.dir = builddirs.get(name, '.')
1225         self.tag = self.name.replace('/', '_')
1226         self.definition = definition
1227         self.sequence = definition["sequence"]
1228         self.git_clone_required = False
1229         if "git-clone-required" in definition:
1230             self.git_clone_required = bool(definition["git-clone-required"])
1231         self.proc = None
1232         self.done = False
1233         self.next = 0
1234         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1235         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1236         if options.verbose:
1237             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1238             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1239         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1240         self.stdout = open(self.stdout_path, 'w')
1241         self.stderr = open(self.stderr_path, 'w')
1242         self.stdin  = open("/dev/null", 'r')
1243         self.builder_dir = "%s/%s" % (testbase, self.tag)
1244         self.test_source_dir = self.builder_dir
1245         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1246         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1247         self.prefix = "%s/%s" % (test_prefix, self.tag)
1248         self.consumers = []
1249         self.producer = None
1250
1251         if self.git_clone_required:
1252             assert "dependency" not in definition
1253
1254     def mark_existing(self):
1255         do_print('%s: Mark as existing dependency' % self.name)
1256         self.next = len(self.sequence)
1257         self.done = True
1258
1259     def add_consumer(self, consumer):
1260         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1261         consumer.producer = self
1262         consumer.test_source_dir = self.test_source_dir
1263         self.consumers.append(consumer)
1264
1265     def start_next(self):
1266         if self.producer is not None:
1267             if not self.producer.done:
1268                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1269                 return
1270
1271         if self.next == 0:
1272             rmdir_force(self.builder_dir)
1273             rmdir_force(self.prefix)
1274             if self.producer is not None:
1275                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1276             elif not self.git_clone_required:
1277                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1278             else:
1279                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1280
1281         if self.next == len(self.sequence):
1282             if not self.done:
1283                 do_print('%s: Completed OK' % self.name)
1284                 self.done = True
1285             if not options.nocleanup and len(self.consumers) == 0:
1286                 do_print('%s: Cleaning up' % self.name)
1287                 rmdir_force(self.builder_dir)
1288                 rmdir_force(self.prefix)
1289             for consumer in self.consumers:
1290                 if consumer.next != 0:
1291                     continue
1292                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1293                 consumer.start_next()
1294             if self.producer is not None:
1295                 self.producer.consumers.remove(self)
1296                 assert self.producer.done
1297                 self.producer.start_next()
1298             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1299             return
1300         (self.stage, self.cmd) = self.sequence[self.next]
1301         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1302                                     get_path(name='platlib',
1303                                              scheme="posix_prefix",
1304                                              vars={"base": self.prefix,
1305                                                    "platbase": self.prefix}))
1306         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1307         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1308         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1309         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1310         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1311         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1312         self.cmd = self.cmd.replace("${NAME}", self.name)
1313         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1314         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1315         self.proc = Popen(self.cmd, shell=True,
1316                           close_fds=True, cwd=self.cwd,
1317                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1318         self.next += 1
1319
1320 def expand_dependencies(n):
1321     deps = list()
1322     if "dependency" in tasks[n]:
1323         depname = tasks[n]["dependency"]
1324         assert depname in tasks
1325         sdeps = expand_dependencies(depname)
1326         assert n not in sdeps
1327         for sdep in sdeps:
1328             deps.append(sdep)
1329         deps.append(depname)
1330     return deps
1331
1332
1333 class buildlist(object):
1334     '''handle build of multiple directories'''
1335
1336     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1337         self.tail_proc = None
1338         self.retry = None
1339         if not tasknames:
1340             if options.restrict_tests:
1341                 tasknames = ["samba-test-only"]
1342             else:
1343                 tasknames = defaulttasks
1344
1345         given_tasknames = tasknames.copy()
1346         implicit_tasknames = []
1347         for n in given_tasknames:
1348             deps = expand_dependencies(n)
1349             for dep in deps:
1350                 if dep in given_tasknames:
1351                     continue
1352                 if dep in implicit_tasknames:
1353                     continue
1354                 implicit_tasknames.append(dep)
1355
1356         tasknames = implicit_tasknames.copy()
1357         tasknames.extend(given_tasknames)
1358         do_print("given_tasknames: %s" % given_tasknames)
1359         do_print("implicit_tasknames: %s" % implicit_tasknames)
1360         do_print("tasknames: %s" % tasknames)
1361         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1362
1363         if options.retry:
1364             rebase_remote = "rebaseon"
1365             retry_task = {
1366                     "git-clone-required": True,
1367                     "sequence": [
1368                             ("retry",
1369                             '''set -e
1370                             git remote add -t %s %s %s
1371                             git fetch %s
1372                             while :; do
1373                               sleep 60
1374                               git describe %s/%s > old_remote_branch.desc
1375                               git fetch %s
1376                               git describe %s/%s > remote_branch.desc
1377                               diff old_remote_branch.desc remote_branch.desc
1378                             done
1379                            ''' % (
1380                                rebase_branch, rebase_remote, rebase_url,
1381                                rebase_remote,
1382                                rebase_remote, rebase_branch,
1383                                rebase_remote,
1384                                rebase_remote, rebase_branch
1385                             ))]}
1386
1387             self.retry = builder('retry', retry_task)
1388             self.need_retry = False
1389
1390         if options.skip_dependencies:
1391             for b in self.tlist:
1392                 if b.name in implicit_tasknames:
1393                     b.mark_existing()
1394
1395         for b in self.tlist:
1396             do_print("b.name=%s" % b.name)
1397             if "dependency" not in b.definition:
1398                 continue
1399             depname = b.definition["dependency"]
1400             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1401             for p in self.tlist:
1402                 if p.name == depname:
1403                     p.add_consumer(b)
1404
1405     def kill_kids(self):
1406         if self.tail_proc is not None:
1407             self.tail_proc.terminate()
1408             self.tail_proc.wait()
1409             self.tail_proc = None
1410         if self.retry is not None:
1411             self.retry.proc.terminate()
1412             self.retry.proc.wait()
1413             self.retry = None
1414         for b in self.tlist:
1415             if b.proc is not None:
1416                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1417                 b.proc.terminate()
1418                 b.proc.wait()
1419                 b.proc = None
1420
1421     def wait_one(self):
1422         while True:
1423             none_running = True
1424             for b in self.tlist:
1425                 if b.proc is None:
1426                     continue
1427                 none_running = False
1428                 b.status = b.proc.poll()
1429                 if b.status is None:
1430                     continue
1431                 b.proc = None
1432                 return b
1433             if options.retry:
1434                 ret = self.retry.proc.poll()
1435                 if ret is not None:
1436                     self.need_retry = True
1437                     self.retry = None
1438                     return None
1439             if none_running:
1440                 return None
1441             time.sleep(0.1)
1442
1443     def run(self):
1444         for b in self.tlist:
1445             b.start_next()
1446         if options.retry:
1447             self.retry.start_next()
1448         while True:
1449             b = self.wait_one()
1450             if options.retry and self.need_retry:
1451                 self.kill_kids()
1452                 do_print("retry needed")
1453                 return (0, None, None, None, "retry")
1454             if b is None:
1455                 break
1456             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1457                 self.kill_kids()
1458                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1459             b.start_next()
1460         self.kill_kids()
1461         return (0, None, None, None, "All OK")
1462
1463     def write_system_info(self, filename):
1464         with open(filename, 'w') as f:
1465             for cmd in ['uname -a',
1466                         'lsb_release -a',
1467                         'free',
1468                         'mount',
1469                         'cat /proc/cpuinfo',
1470                         'cc --version',
1471                         'df -m .',
1472                         'df -m %s' % testbase]:
1473                 try:
1474                     out = run_cmd(cmd, output=True, checkfail=False)
1475                 except CalledProcessError as e:
1476                     out = "<failed: %s>" % str(e)
1477                 print('### %s' % cmd, file=f)
1478                 print(out, file=f)
1479                 print(file=f)
1480
1481     def tarlogs(self, fname):
1482         with tarfile.open(fname, "w:gz") as tar:
1483             for b in self.tlist:
1484                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1485                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1486             if os.path.exists("autobuild.log"):
1487                 tar.add("autobuild.log")
1488             filename = 'system-info.txt'
1489             self.write_system_info(filename)
1490             tar.add(filename)
1491
1492     def remove_logs(self):
1493         for b in self.tlist:
1494             os.unlink(b.stdout_path)
1495             os.unlink(b.stderr_path)
1496
1497     def start_tail(self):
1498         cmd = ["tail", "-f"]
1499         for b in self.tlist:
1500             cmd.append(b.stdout_path)
1501             cmd.append(b.stderr_path)
1502         self.tail_proc = Popen(cmd, close_fds=True)
1503
1504
1505 def cleanup(do_raise=False):
1506     if options.nocleanup:
1507         return
1508     run_cmd("stat %s || true" % test_tmpdir, show=True)
1509     run_cmd("stat %s" % testbase, show=True)
1510     do_print("Cleaning up %r" % cleanup_list)
1511     for d in cleanup_list:
1512         ok = rmdir_force(d, re_raise=False)
1513         if ok:
1514             continue
1515         if os.path.isdir(d):
1516             do_print("Killing, waiting and retry")
1517             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1518         else:
1519             do_print("Waiting and retry")
1520         time.sleep(1)
1521         rmdir_force(d, re_raise=do_raise)
1522
1523
1524 def daemonize(logfile):
1525     pid = os.fork()
1526     if pid == 0:  # Parent
1527         os.setsid()
1528         pid = os.fork()
1529         if pid != 0:  # Actual daemon
1530             os._exit(0)
1531     else:  # Grandparent
1532         os._exit(0)
1533
1534     import resource      # Resource usage information.
1535     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1536     if maxfd == resource.RLIM_INFINITY:
1537         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1538     for fd in range(0, maxfd):
1539         try:
1540             os.close(fd)
1541         except OSError:
1542             pass
1543     os.open(logfile, os.O_RDWR | os.O_CREAT)
1544     os.dup2(0, 1)
1545     os.dup2(0, 2)
1546
1547
1548 def write_pidfile(fname):
1549     '''write a pid file, cleanup on exit'''
1550     with open(fname, mode='w') as f:
1551         f.write("%u\n" % os.getpid())
1552
1553
1554 def rebase_tree(rebase_url, rebase_branch="master"):
1555     rebase_remote = "rebaseon"
1556     do_print("Rebasing on %s" % rebase_url)
1557     run_cmd("git describe HEAD", show=True, dir=test_master)
1558     run_cmd("git remote add -t %s %s %s" %
1559             (rebase_branch, rebase_remote, rebase_url),
1560             show=True, dir=test_master)
1561     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1562     if options.fix_whitespace:
1563         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1564                 (rebase_remote, rebase_branch),
1565                 show=True, dir=test_master)
1566     else:
1567         run_cmd("git rebase --force-rebase %s/%s" %
1568                 (rebase_remote, rebase_branch),
1569                 show=True, dir=test_master)
1570     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1571                    (rebase_remote, rebase_branch),
1572                    dir=test_master, output=True)
1573     if diff == '':
1574         do_print("No differences between HEAD and %s/%s - exiting" %
1575                  (rebase_remote, rebase_branch))
1576         sys.exit(0)
1577     run_cmd("git describe %s/%s" %
1578             (rebase_remote, rebase_branch),
1579             show=True, dir=test_master)
1580     run_cmd("git describe HEAD", show=True, dir=test_master)
1581     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1582             (rebase_remote, rebase_branch),
1583             show=True, dir=test_master)
1584
1585
1586 def push_to(push_url, push_branch="master"):
1587     push_remote = "pushto"
1588     do_print("Pushing to %s" % push_url)
1589     if options.mark:
1590         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1591         run_cmd("git commit --amend -c HEAD", dir=test_master)
1592         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1593         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1594     run_cmd("git remote add -t %s %s %s" %
1595             (push_branch, push_remote, push_url),
1596             show=True, dir=test_master)
1597     run_cmd("git push %s +HEAD:%s" %
1598             (push_remote, push_branch),
1599             show=True, dir=test_master)
1600
1601
1602 def send_email(subject, text, log_tar):
1603     if options.email is None:
1604         do_print("not sending email because the recipient is not set")
1605         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1606                  (subject, text))
1607         return
1608     outer = MIMEMultipart()
1609     outer['Subject'] = subject
1610     outer['To'] = options.email
1611     outer['From'] = options.email_from
1612     outer['Date'] = email.utils.formatdate(localtime=True)
1613     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1614     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1615     if options.attach_logs:
1616         with open(log_tar, 'rb') as fp:
1617             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1618         # Set the filename parameter
1619         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1620         outer.attach(msg)
1621     content = outer.as_string()
1622     s = smtplib.SMTP(options.email_server)
1623     email_user = os.getenv('SMTP_USERNAME')
1624     email_password = os.getenv('SMTP_PASSWORD')
1625     if email_user is not None:
1626         s.starttls()
1627         s.login(email_user, email_password)
1628
1629     s.sendmail(options.email_from, [options.email], content)
1630     s.set_debuglevel(1)
1631     s.quit()
1632
1633
1634 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1635                   elapsed_time, log_base=None, add_log_tail=True):
1636     '''send an email to options.email about the failure'''
1637     elapsed_minutes = elapsed_time / 60.0
1638     if log_base is None:
1639         log_base = gitroot
1640     text = '''
1641 Dear Developer,
1642
1643 Your autobuild on %s failed after %.1f minutes
1644 when trying to test %s with the following error:
1645
1646    %s
1647
1648 the autobuild has been abandoned. Please fix the error and resubmit.
1649
1650 A summary of the autobuild process is here:
1651
1652   %s/autobuild.log
1653 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1654
1655     if options.restrict_tests:
1656         text += """
1657 The build was restricted to tests matching %s\n""" % options.restrict_tests
1658
1659     if failed_task != 'rebase':
1660         text += '''
1661 You can see logs of the failed task here:
1662
1663   %s/%s.stdout
1664   %s/%s.stderr
1665
1666 or you can get full logs of all tasks in this job here:
1667
1668   %s/logs.tar.gz
1669
1670 The top commit for the tree that was built was:
1671
1672 %s
1673
1674 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1675
1676     if add_log_tail:
1677         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1678         lines = f.readlines()
1679         log_tail = "".join(lines[-50:])
1680         num_lines = len(lines)
1681         if num_lines < 50:
1682             # Also include stderr (compile failures) if < 50 lines of stdout
1683             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1684             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1685
1686         text += '''
1687 The last 50 lines of log messages:
1688
1689 %s
1690     ''' % log_tail
1691         f.close()
1692
1693     logs = os.path.join(gitroot, 'logs.tar.gz')
1694     send_email('autobuild[%s] failure on %s for task %s during %s'
1695                % (options.branch, platform.node(), failed_task, failed_stage),
1696                text, logs)
1697
1698
1699 def email_success(elapsed_time, log_base=None):
1700     '''send an email to options.email about a successful build'''
1701     if log_base is None:
1702         log_base = gitroot
1703     text = '''
1704 Dear Developer,
1705
1706 Your autobuild on %s has succeeded after %.1f minutes.
1707
1708 ''' % (platform.node(), elapsed_time / 60.)
1709
1710     if options.restrict_tests:
1711         text += """
1712 The build was restricted to tests matching %s\n""" % options.restrict_tests
1713
1714     if options.keeplogs:
1715         text += '''
1716
1717 you can get full logs of all tasks in this job here:
1718
1719   %s/logs.tar.gz
1720
1721 ''' % log_base
1722
1723     text += '''
1724 The top commit for the tree that was built was:
1725
1726 %s
1727 ''' % top_commit_msg
1728
1729     logs = os.path.join(gitroot, 'logs.tar.gz')
1730     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1731                text, logs)
1732
1733
1734 # get the top commit message, for emails
1735 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1736
1737 try:
1738     if options.skip_dependencies:
1739         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1740     else:
1741         os.makedirs(testbase)
1742 except Exception as reason:
1743     raise Exception("Unable to create %s : %s" % (testbase, reason))
1744 cleanup_list.append(testbase)
1745
1746 if options.daemon:
1747     logfile = os.path.join(testbase, "log")
1748     do_print("Forking into the background, writing progress to %s" % logfile)
1749     daemonize(logfile)
1750
1751 write_pidfile(gitroot + "/autobuild.pid")
1752
1753 start_time = time.time()
1754
1755 while True:
1756     try:
1757         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1758         os.makedirs(test_tmpdir)
1759         # The waf uninstall code removes empty directories all the way
1760         # up the tree.  Creating a file in test_tmpdir stops it from
1761         # being removed.
1762         run_cmd("touch %s" % os.path.join(test_tmpdir,
1763                                           ".directory-is-not-empty"), show=True)
1764         run_cmd("stat %s" % test_tmpdir, show=True)
1765         run_cmd("stat %s" % testbase, show=True)
1766         if options.skip_dependencies:
1767             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1768         else:
1769             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1770     except Exception:
1771         cleanup()
1772         raise
1773
1774     try:
1775         if options.rebase is not None:
1776             rebase_tree(options.rebase, rebase_branch=options.branch)
1777     except Exception:
1778         cleanup_list.append(gitroot + "/autobuild.pid")
1779         cleanup()
1780         elapsed_time = time.time() - start_time
1781         email_failure(-1, 'rebase', 'rebase', 'rebase',
1782                       'rebase on %s failed' % options.branch,
1783                       elapsed_time, log_base=options.log_base)
1784         sys.exit(1)
1785
1786     try:
1787         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1788         if options.tail:
1789             blist.start_tail()
1790         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1791         if status != 0 or errstr != "retry":
1792             break
1793         cleanup(do_raise=True)
1794     except Exception:
1795         cleanup()
1796         raise
1797
1798 cleanup_list.append(gitroot + "/autobuild.pid")
1799
1800 do_print(errstr)
1801
1802 blist.kill_kids()
1803 if options.tail:
1804     do_print("waiting for tail to flush")
1805     time.sleep(1)
1806
1807 elapsed_time = time.time() - start_time
1808 if status == 0:
1809     if options.passcmd is not None:
1810         do_print("Running passcmd: %s" % options.passcmd)
1811         run_cmd(options.passcmd, dir=test_master)
1812     if options.pushto is not None:
1813         push_to(options.pushto, push_branch=options.branch)
1814     if options.keeplogs or options.attach_logs:
1815         blist.tarlogs("logs.tar.gz")
1816         do_print("Logs in logs.tar.gz")
1817     if options.always_email:
1818         email_success(elapsed_time, log_base=options.log_base)
1819     blist.remove_logs()
1820     cleanup()
1821     do_print(errstr)
1822     sys.exit(0)
1823
1824 # something failed, gather a tar of the logs
1825 blist.tarlogs("logs.tar.gz")
1826
1827 if options.email is not None:
1828     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1829                   elapsed_time, log_base=options.log_base)
1830 else:
1831     elapsed_minutes = elapsed_time / 60.0
1832     print('''
1833
1834 ####################################################################
1835
1836 AUTOBUILD FAILURE
1837
1838 Your autobuild[%s] on %s failed after %.1f minutes
1839 when trying to test %s with the following error:
1840
1841    %s
1842
1843 the autobuild has been abandoned. Please fix the error and resubmit.
1844
1845 ####################################################################
1846
1847 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1848
1849 cleanup()
1850 do_print(errstr)
1851 do_print("Logs in logs.tar.gz")
1852 sys.exit(status)