script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 # allow autobuild to run within git rebase -i
  35 if "GIT_DIR" in os.environ:
  36     del os.environ["GIT_DIR"]
  37 if "GIT_WORK_TREE" in os.environ:
  38     del os.environ["GIT_WORK_TREE"]
  39
  40 def find_git_root():
  41     '''get to the top of the git repo'''
  42     p = os.getcwd()
  43     while p != '/':
  44         if os.path.exists(os.path.join(p, ".git")):
  45             return p
  46         p = os.path.abspath(os.path.join(p, '..'))
  47     return None
  48
  49
  50 gitroot = find_git_root()
  51 if gitroot is None:
  52     raise Exception("Failed to find git root")
  53
  54
  55 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  56
  57 parser = OptionParser()
  58 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  59 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  60 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  61 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  62 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  63                   default=def_testbase)
  64 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  65                   default=None)
  66 parser.add_option("--passcmd", help="command to run on success", default=None)
  67 parser.add_option("--verbose", help="show all commands as they are run",
  68                   default=False, action="store_true")
  69 parser.add_option("--rebase", help="rebase on the given tree before testing",
  70                   default=None, type='str')
  71 parser.add_option("--pushto", help="push to a git url on success",
  72                   default=None, type='str')
  73 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  74                   default=False, action="store_true")
  75 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  76                   default=False, action="store_true")
  77 parser.add_option("--retry", help="automatically retry if master changes",
  78                   default=False, action="store_true")
  79 parser.add_option("--email", help="send email to the given address on failure",
  80                   type='str', default=None)
  81 parser.add_option("--email-from", help="send email from the given address",
  82                   type='str', default="autobuild@samba.org")
  83 parser.add_option("--email-server", help="send email via the given server",
  84                   type='str', default='localhost')
  85 parser.add_option("--always-email", help="always send email, even on success",
  86                   action="store_true")
  87 parser.add_option("--daemon", help="daemonize after initial setup",
  88                   action="store_true")
  89 parser.add_option("--branch", help="the branch to work on (default=master)",
  90                   default="master", type='str')
  91 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  92                   default=gitroot, type='str')
  93 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  94                   default=False, action="store_true")
  95 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  96                   default='')
  97 parser.add_option("--enable-coverage", dest='enable_coverage',
  98                   action="store_const", const='--enable-coverage', default='',
  99                   help="Add --enable-coverage option while configure")
 100
 101 (options, args) = parser.parse_args()
 102
 103 if options.retry:
 104     if options.rebase is None:
 105         raise Exception('You can only use --retry if you also rebase')
 106
 107 if options.full_testbase is not None:
 108     testbase = options.full_testbase
 109 else:
 110     testbase = "%s/b%u" % (options.testbase, os.getpid())
 111 test_master = "%s/master" % testbase
 112 test_prefix = "%s/prefix" % testbase
 113 test_tmpdir = "%s/tmp" % testbase
 114 os.environ['TMPDIR'] = test_tmpdir
 115
 116 if options.enable_coverage:
 117     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 118 else:
 119     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 120
 121 if options.enable_coverage:
 122     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 123 else:
 124     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 125
 126 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 127
 128
 129 def check_symbols(sofile, expected_symbols=""):
 130     return "objdump --dynamic-syms " + sofile + " | " + \
 131            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 132
 133 if args:
 134     # If we are only running specific test,
 135     # do not sleep randomly to wait for it to start
 136     def random_sleep(low, high):
 137         return 'sleep 1'
 138 else:
 139     def random_sleep(low, high):
 140         return 'sleep {}'.format(random.randint(low, high))
 141
 142 cleanup_list = []
 143
 144 builddirs = {
 145     "ctdb": "ctdb",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl",
 152     "docs-xml": "docs-xml"
 153 }
 154
 155 ctdb_configure_params = " --enable-developer ${PREFIX}"
 156 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 157
 158 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 159 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 160 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 161 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 162 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 163 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 164 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 165
 166
 167 def format_option(name, value=None):
 168     """Format option as str list."""
 169     if value is None:  # boolean option
 170         return [name]
 171     if not isinstance(value, list):  # single value option
 172         value = [value]
 173     # repeatable option
 174     return ['{}={}'.format(name, item) for item in value]
 175
 176
 177 def make_test(
 178         cmd='make testonly',
 179         INJECT_SELFTEST_PREFIX=1,
 180         TESTS='',
 181         include_envs=None,
 182         exclude_envs=None):
 183
 184     test_options = []
 185     if include_envs:
 186         test_options = format_option('--include-env', include_envs)
 187     if exclude_envs:
 188         test_options = format_option('--exclude-env', exclude_envs)
 189     if test_options:
 190         # join envs options to original test options
 191         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 192
 193     _options = []
 194
 195     # Allow getting a full CI with
 196     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 197
 198     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 199
 200     if int(FAIL_IMMEDIATELY):
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     if INJECT_SELFTEST_PREFIX:
 206         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 207         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 208
 209     return ' '.join([cmd] + _options)
 210
 211
 212 # When updating this list, also update .gitlab-ci.yml to add the job
 213 # and to make it a dependency of 'page' for the coverage report.
 214
 215 tasks = {
 216     "ctdb": {
 217         "sequence": [
 218             ("random-sleep", random_sleep(300, 900)),
 219             ("configure", "./configure " + ctdb_configure_params),
 220             ("make", "make all"),
 221             ("install", "make install"),
 222             ("test", "make autotest"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("clean", "make clean"),
 225         ],
 226     },
 227     "docs-xml": {
 228         "sequence": [
 229             ("random-sleep", random_sleep(300, 900)),
 230             ("autoconf", "autoconf"),
 231             ("configure", "./configure"),
 232             ("make", "make html htmlman"),
 233             ("publish-docs", PUBLISH_DOCS),
 234             ("clean", "make clean"),
 235         ],
 236     },
 237
 238     "samba-def-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-mit-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-nt4-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     "samba-h5l-build": {
 269         "git-clone-required": True,
 270         "sequence": [
 271             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 272             ("make", "make -j"),
 273             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 274             ("chmod-R-a-w", "chmod -R a-w ."),
 275         ],
 276     },
 277
 278     "samba-without-smb1-build": {
 279         "git-clone-required": True,
 280         "sequence": [
 281             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 282             ("make", "make -j"),
 283             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 284             ("chmod-R-a-w", "chmod -R a-w ."),
 285         ],
 286     },
 287
 288     "samba-no-opath-build": {
 289         "git-clone-required": True,
 290         "sequence": [
 291             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 292             ("make", "make -j"),
 293             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 294             ("chmod-R-a-w", "chmod -R a-w ."),
 295         ],
 296     },
 297
 298     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 299     "samba": {
 300         "sequence": [
 301             ("random-sleep", random_sleep(300, 900)),
 302             ("configure", "./configure.developer" + samba_configure_params),
 303             ("make", "make -j"),
 304             ("test", make_test(exclude_envs=[
 305             "none",
 306             "nt4_dc",
 307             "nt4_dc_smb1",
 308             "nt4_dc_smb1_done",
 309             "nt4_dc_schannel",
 310             "nt4_member",
 311             "ad_dc",
 312             "ad_dc_smb1",
 313             "ad_dc_smb1_done",
 314             "ad_dc_backup",
 315             "ad_dc_ntvfs",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_slowtests",
 319             "ad_dc_no_nss",
 320             "ad_dc_no_ntlm",
 321             "fl2003dc",
 322             "fl2008dc",
 323             "fl2008r2dc",
 324             "ad_member",
 325             "ad_member_idmap_rid",
 326             "admem_idmap_autorid",
 327             "ad_member_idmap_ad",
 328             "ad_member_rfc2307",
 329             "ad_member_oneway",
 330             "chgdcpass",
 331             "vampire_2000_dc",
 332             "fl2000dc",
 333             "fileserver",
 334             "fileserver_smb1",
 335             "fileserver_smb1_done",
 336             "maptoguest",
 337             "simpleserver",
 338             "backupfromdc",
 339             "restoredc",
 340             "renamedc",
 341             "offlinebackupdc",
 342             "labdc",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             "promoted_dc",
 346             "vampire_dc",
 347             "rodc",
 348             "ad_dc_default",
 349             "ad_dc_default_smb1",
 350             "ad_dc_default_smb1_done",
 351             "ad_dc_slowtests",
 352             "schema_pair_dc",
 353             "schema_dc",
 354             "clusteredmember",
 355             "ad_dc_fips",
 356             "ad_member_fips",
 357             ])),
 358             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 359             ("lcov", LCOV_CMD),
 360             ("install", "make install"),
 361             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 362             ("clean", "make clean"),
 363         ],
 364     },
 365
 366     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 367     "samba-mitkrb5": {
 368         "sequence": [
 369             ("random-sleep", random_sleep(300, 900)),
 370             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 371             ("make", "make -j"),
 372             ("test", make_test(exclude_envs=[
 373             "none",
 374             "nt4_dc",
 375             "nt4_dc_smb1",
 376             "nt4_dc_smb1_done",
 377             "nt4_dc_schannel",
 378             "nt4_member",
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_backup",
 383             "ad_dc_ntvfs",
 384             "ad_dc_default",
 385             "ad_dc_default_smb1",
 386             "ad_dc_default_smb1_done",
 387             "ad_dc_slowtests",
 388             "ad_dc_no_nss",
 389             "ad_dc_no_ntlm",
 390             "fl2003dc",
 391             "fl2008dc",
 392             "fl2008r2dc",
 393             "ad_member",
 394             "ad_member_idmap_rid",
 395             "admem_idmap_autorid",
 396             "ad_member_idmap_ad",
 397             "ad_member_rfc2307",
 398             "ad_member_oneway",
 399             "chgdcpass",
 400             "vampire_2000_dc",
 401             "fl2000dc",
 402             "fileserver",
 403             "fileserver_smb1",
 404             "fileserver_smb1_done",
 405             "maptoguest",
 406             "simpleserver",
 407             "backupfromdc",
 408             "restoredc",
 409             "renamedc",
 410             "offlinebackupdc",
 411             "labdc",
 412             "preforkrestartdc",
 413             "proclimitdc",
 414             "promoted_dc",
 415             "vampire_dc",
 416             "rodc",
 417             "ad_dc_default",
 418             "ad_dc_default_smb1",
 419             "ad_dc_default_smb1_done",
 420             "ad_dc_slowtests",
 421             "schema_pair_dc",
 422             "schema_dc",
 423             "clusteredmember",
 424             "ad_dc_fips",
 425             "ad_member_fips",
 426             ])),
 427             ("lcov", LCOV_CMD),
 428             ("install", "make install"),
 429             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 430             ("clean", "make clean"),
 431         ],
 432     },
 433
 434     "samba-nt4": {
 435         "dependency": "samba-nt4-build",
 436         "sequence": [
 437             ("random-sleep", random_sleep(300, 900)),
 438             ("test", make_test(include_envs=[
 439             "nt4_dc",
 440             "nt4_dc_smb1",
 441             "nt4_dc_smb1_done",
 442             "nt4_dc_schannel",
 443             "nt4_member",
 444             "simpleserver",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-fileserver": {
 452         "dependency": "samba-h5l-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(include_envs=[
 456             "fileserver",
 457             "fileserver_smb1",
 458             "fileserver_smb1_done",
 459             "maptoguest",
 460             "ktest", # ktest is also tested in samba-ktest-mit samba
 461                      # and samba-mitkrb5 but is tested here against
 462                      # a system Heimdal
 463             ])),
 464             ("lcov", LCOV_CMD),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466         ],
 467     },
 468
 469     "samba-fileserver-without-smb1": {
 470         "dependency": "samba-without-smb1-build",
 471         "sequence": [
 472             ("random-sleep", random_sleep(300, 900)),
 473             ("test", make_test(include_envs=["fileserver"])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     # This is a full build without the AD DC so we test the build with
 480     # MIT Kerberos from the current system.  Runtime behaviour is
 481     # confirmed via the ktest (static ccache and keytab) environment
 482
 483     # This environment also used to confirm we can still build with --with-libunwind
 484     "samba-ktest-mit": {
 485         "sequence": [
 486             ("random-sleep", random_sleep(300, 900)),
 487             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 488             ("make", "make -j"),
 489             ("test", make_test(include_envs=[
 490             "ktest", # ktest is also tested in fileserver, samba and
 491                      # samba-mitkrb5 but is tested here against a
 492                      # system MIT krb5
 493             ])),
 494             ("lcov", LCOV_CMD),
 495             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 496         ],
 497     },
 498
 499     "samba-admem": {
 500         "dependency": "samba-def-build",
 501         "sequence": [
 502             ("random-sleep", random_sleep(300, 900)),
 503             ("test", make_test(include_envs=[
 504             "ad_member",
 505             "ad_member_idmap_rid",
 506             "admem_idmap_autorid",
 507             "ad_member_idmap_ad",
 508             "ad_member_rfc2307",
 509             "ad_member_offlogon",
 510             ])),
 511             ("lcov", LCOV_CMD),
 512             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 513         ],
 514     },
 515
 516     "samba-no-opath1": {
 517         "dependency": "samba-no-opath-build",
 518         "sequence": [
 519             ("random-sleep", random_sleep(300, 900)),
 520             ("test", make_test(
 521                 cmd="make testonly DISABLE_OPATH=1",
 522                 include_envs=[
 523                 "nt4_dc",
 524                 "nt4_dc_smb1",
 525                 "nt4_dc_smb1_done",
 526                 "nt4_dc_schannel",
 527                 "nt4_member",
 528                 "simpleserver",
 529                 ])),
 530             ("lcov", LCOV_CMD),
 531             ("check-clean-tree", "script/clean-source-tree.sh"),
 532         ],
 533     },
 534
 535     "samba-no-opath2": {
 536         "dependency": "samba-no-opath-build",
 537         "sequence": [
 538             ("random-sleep", random_sleep(300, 900)),
 539             ("test", make_test(
 540                 cmd="make testonly DISABLE_OPATH=1",
 541                 include_envs=[
 542                 "fileserver",
 543                 "fileserver_smb1",
 544                 "fileserver_smb1_done",
 545                 ])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", "script/clean-source-tree.sh"),
 548         ],
 549     },
 550
 551     "samba-ad-dc-1": {
 552         "dependency": "samba-def-build",
 553         "sequence": [
 554             ("random-sleep", random_sleep(1, 1)),
 555             ("test", make_test(include_envs=[
 556             "ad_dc",
 557             "ad_dc_smb1",
 558             "ad_dc_smb1_done",
 559             "ad_dc_no_nss",
 560             "ad_dc_no_ntlm",
 561             ])),
 562             ("lcov", LCOV_CMD),
 563             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 564         ],
 565     },
 566
 567     "samba-ad-dc-2": {
 568         "dependency": "samba-def-build",
 569         "sequence": [
 570             ("random-sleep", random_sleep(1, 1)),
 571             ("test", make_test(include_envs=[
 572             "vampire_dc",
 573             "vampire_2000_dc",
 574             "rodc",
 575             ])),
 576             ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     "samba-ad-dc-3": {
 582         "dependency": "samba-def-build",
 583         "sequence": [
 584             ("random-sleep", random_sleep(1, 1)),
 585             ("test", make_test(include_envs=[
 586             "promoted_dc",
 587             "chgdcpass",
 588             "preforkrestartdc",
 589             "proclimitdc",
 590             ])),
 591             ("lcov", LCOV_CMD),
 592             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 593         ],
 594     },
 595
 596     "samba-ad-dc-4a": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=[
 601             "fl2000dc",
 602             "ad_member_oneway",
 603             "fl2003dc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609     "samba-ad-dc-4b": {
 610         "dependency": "samba-def-build",
 611         "sequence": [
 612             ("random-sleep", random_sleep(1, 1)),
 613             ("test", make_test(include_envs=[
 614             "fl2008dc",
 615             "fl2008r2dc",
 616             ])),
 617             ("lcov", LCOV_CMD),
 618             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 619         ],
 620     },
 621
 622     "samba-ad-dc-5": {
 623         "dependency": "samba-def-build",
 624         "sequence": [
 625             ("random-sleep", random_sleep(1, 1)),
 626             ("test", make_test(include_envs=[
 627             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-6": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-schemaupgrade": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 654     # This is currently the longest task, so we don't randomly delay it.
 655     "samba-ad-dc-ntvfs": {
 656         "dependency": "samba-def-build",
 657         "sequence": [
 658             ("random-sleep", random_sleep(1, 1)),
 659             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 660             ("lcov", LCOV_CMD),
 661             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 662         ],
 663     },
 664
 665     # Test fips compliance
 666     "samba-fips": {
 667         "dependency": "samba-mit-build",
 668         "sequence": [
 669             ("random-sleep", random_sleep(1, 1)),
 670             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 671             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 672             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 673         ],
 674     },
 675
 676     # run the backup/restore testenvs separately as they're fairly standalone
 677     # (and CI seems to max out at ~3 different DCs running at once)
 678     "samba-ad-back1": {
 679         "dependency": "samba-def-build",
 680         "sequence": [
 681             ("random-sleep", random_sleep(300, 900)),
 682             ("test", make_test(include_envs=[
 683             "backupfromdc",
 684             "restoredc",
 685             "renamedc",
 686             ])),
 687             ("lcov", LCOV_CMD),
 688             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 689         ],
 690     },
 691     "samba-ad-back2": {
 692         "dependency": "samba-def-build",
 693         "sequence": [
 694             ("random-sleep", random_sleep(300, 900)),
 695             ("test", make_test(include_envs=[
 696             "backupfromdc",
 697             "offlinebackupdc",
 698             "labdc",
 699             ])),
 700             ("lcov", LCOV_CMD),
 701             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 702         ],
 703     },
 704
 705     "samba-admem-mit": {
 706         "dependency": "samba-mit-build",
 707         "sequence": [
 708             ("random-sleep", random_sleep(1, 1)),
 709             ("test", make_test(include_envs=[
 710             "ad_member",
 711             "ad_member_idmap_rid",
 712             "admem_idmap_autorid",
 713             "ad_member_idmap_ad",
 714             "ad_member_rfc2307",
 715             "ad_member_offlogon",
 716             ])),
 717             ("lcov", LCOV_CMD),
 718             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 719         ],
 720     },
 721
 722     "samba-addc-mit-1": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "ad_dc",
 728             "ad_dc_smb1",
 729             "ad_dc_smb1_done",
 730             "ad_dc_no_nss",
 731             "ad_dc_no_ntlm",
 732             ])),
 733             ("lcov", LCOV_CMD),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735         ],
 736     },
 737
 738     "samba-addc-mit-4a": {
 739         "dependency": "samba-mit-build",
 740         "sequence": [
 741             ("random-sleep", random_sleep(1, 1)),
 742             ("test", make_test(include_envs=[
 743             "fl2000dc",
 744             "ad_member_oneway",
 745             "fl2003dc",
 746             ])),
 747             ("lcov", LCOV_CMD),
 748             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 749         ],
 750     },
 751     "samba-addc-mit-4b": {
 752         "dependency": "samba-mit-build",
 753         "sequence": [
 754             ("random-sleep", random_sleep(1, 1)),
 755             ("test", make_test(include_envs=[
 756             "fl2008dc",
 757             "fl2008r2dc",
 758             ])),
 759             ("lcov", LCOV_CMD),
 760             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 761         ],
 762     },
 763
 764     "samba-test-only": {
 765         "sequence": [
 766             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 767             ("make", "make -j"),
 768             ("test", make_test(TESTS="${TESTS}")),
 769             ("lcov", LCOV_CMD),
 770         ],
 771     },
 772
 773     # Test cross-compile infrastructure
 774     "samba-xc": {
 775         "sequence": [
 776             ("random-sleep", random_sleep(900, 1500)),
 777             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 778             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 779             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 780             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 781             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 782             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 783             ("compare-results", "script/compare_cc_results.py "
 784             "./bin/c4che/default{} "
 785             "./bin-xe/c4che/default{} "
 786             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 787             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 788             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 789             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 790             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 791             " = \"'1234'\"".format(CACHE_SUFFIX)),
 792             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 793             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 794             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 795             " ; test $? -ne 0"),
 796         ],
 797     },
 798
 799     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 800     "samba-o3": {
 801         "sequence": [
 802             ("random-sleep", random_sleep(300, 900)),
 803             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 804             ("make", "make -j"),
 805             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 806             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 807             ("lcov", LCOV_CMD),
 808             ("install", "make install"),
 809             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 810             ("clean", "make clean"),
 811         ],
 812     },
 813
 814     "samba-ctdb": {
 815         "sequence": [
 816             ("random-sleep", random_sleep(900, 1500)),
 817
 818         # make sure we have tdb around:
 819             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 820             ("tdb-make", "cd lib/tdb && make"),
 821             ("tdb-install", "cd lib/tdb && make install"),
 822
 823         # build samba with cluster support (also building ctdb):
 824             ("samba-configure",
 825          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 826          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 827          "./configure.developer ${PREFIX} "
 828          "--with-selftest-prefix=./bin/ab "
 829          "--with-cluster-support "
 830          "--without-ad-dc "
 831          "--bundled-libraries=!tdb"),
 832             ("samba-make", "make"),
 833             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 834             ("samba-install", "make install"),
 835             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 836
 837             ("test", make_test(
 838                 cmd='make test',
 839                 INJECT_SELFTEST_PREFIX=0,
 840                 include_envs=["clusteredmember"])
 841             ),
 842
 843         # clean up:
 844             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 845             ("clean", "make clean"),
 846             ("ctdb-clean", "cd ./ctdb && make clean"),
 847         ],
 848     },
 849
 850     "samba-libs": {
 851         "sequence": [
 852             ("random-sleep", random_sleep(300, 900)),
 853             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 854             ("talloc-make", "cd lib/talloc && make"),
 855             ("talloc-install", "cd lib/talloc && make install"),
 856
 857             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 858             ("tdb-make", "cd lib/tdb && make"),
 859             ("tdb-install", "cd lib/tdb && make install"),
 860
 861             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 862             ("tevent-make", "cd lib/tevent && make"),
 863             ("tevent-install", "cd lib/tevent && make install"),
 864
 865             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 866             ("ldb-make", "cd lib/ldb && make"),
 867             ("ldb-install", "cd lib/ldb && make install"),
 868
 869             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 870             ("nondevel-make", "make -j"),
 871             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 872             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 873             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 874             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 875             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 876             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 877             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 878             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 879             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 880             ("nondevel-no-public-nss_winbind",
 881                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 882             ("nondevel-no-public-nss_wins",
 883                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 884             ("nondevel-no-public-libwbclient",
 885                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 886             ("nondevel-no-public-pam_winbind",
 887                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 888             ("nondevel-no-public-winbind_krb5_locator",
 889                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 890             ("nondevel-no-public-async_dns_krb5_locator",
 891                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 892             ("nondevel-install", "make -j install"),
 893             ("nondevel-dist", "make dist"),
 894
 895             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 896             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 897             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 898             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 899             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 900             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 901             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 902             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 903             ("prefix-no-public-nss_winbind",
 904                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 905             ("prefix-no-public-nss_wins",
 906                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 907             ("prefix-no-public-libwbclient",
 908                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 909             ("prefix-no-public-pam_winbind",
 910                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 911             ("prefix-no-public-winbind_krb5_locator",
 912                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 913                               "service_locator")),
 914             ("prefix-no-public-async_dns_krb5_locator",
 915                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 916                               "service_locator")),
 917
 918             # retry with all modules shared
 919             ("allshared-distclean", "make distclean"),
 920             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 921             ("allshared-make", "make -j"),
 922             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 923             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 924             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 925             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 926             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 927             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 928             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 929             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 930             ("allshared-no-public-nss_winbind",
 931                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 932             ("allshared-no-public-nss_wins",
 933                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 934             ("allshared-no-public-libwbclient",
 935                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 936             ("allshared-no-public-pam_winbind",
 937                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 938             ("allshared-no-public-winbind_krb5_locator",
 939                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 940             ("allshared-no-public-async_dns_krb5_locator",
 941                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 942         ],
 943     },
 944
 945     "samba-fuzz": {
 946         "sequence": [
 947         # build the fuzzers (static) via the oss-fuzz script
 948             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 949             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 950         ],
 951     },
 952
 953     # * Test smbd and smbtorture can build semi-static
 954     #
 955     # * Test Samba without python still builds.
 956     #
 957     # When this test fails due to more use of Python, the expectations
 958     # is that the newly failing part of the code should be disabled
 959     # when --disable-python is set (rather than major work being done
 960     # to support this environment).
 961     #
 962     # The target here is for vendors shipping a minimal smbd.
 963     "samba-minimal-smbd": {
 964         "sequence": [
 965             ("random-sleep", random_sleep(300, 900)),
 966
 967         # build with all modules static
 968             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 969             ("allstatic-make", "make -j"),
 970             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 971             ("allstatic-lcov", LCOV_CMD),
 972
 973         # retry with nonshared smbd and smbtorture
 974             ("nonshared-distclean", "make distclean"),
 975             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 976             ("nonshared-make", "make -j"),
 977             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 978             # TODO ("nonshared-lcov", LCOV_CMD),
 979
 980             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 981             ("clean", "make clean"),
 982         ],
 983     },
 984
 985     "samba-nopython": {
 986         "sequence": [
 987             ("random-sleep", random_sleep(300, 900)),
 988
 989             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 990             ("make", "make -j"),
 991             ("find-python", "script/find_python.sh ${PREFIX}"),
 992             ("test", "make test-nopython"),
 993             ("lcov", LCOV_CMD),
 994             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 995             ("clean", "make clean"),
 996
 997             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 998             ("talloc-make", "cd lib/talloc && make"),
 999             ("talloc-install", "cd lib/talloc && make install"),
1000
1001             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1002             ("tdb-make", "cd lib/tdb && make"),
1003             ("tdb-install", "cd lib/tdb && make install"),
1004
1005             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1006             ("tevent-make", "cd lib/tevent && make"),
1007             ("tevent-install", "cd lib/tevent && make install"),
1008
1009             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1010             ("ldb-make", "cd lib/ldb && make"),
1011             ("ldb-install", "cd lib/ldb && make install"),
1012
1013         # retry against installed library packages, but no required modules
1014             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1015             ("libs-make", "make -j"),
1016             ("libs-install", "make install"),
1017             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1018             ("libs-clean", "make clean"),
1019
1020         ],
1021     },
1022
1023     "ldb": {
1024         "sequence": [
1025             ("random-sleep", random_sleep(60, 600)),
1026             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1027             ("make", "make"),
1028             ("install", "make install"),
1029             ("test", "make test"),
1030             ("lcov", LCOV_CMD),
1031             ("clean", "make clean"),
1032             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1033             ("make-no-lmdb", "make"),
1034             ("test-no-lmdb", "make test"),
1035             ("lcov-no-lmdb", LCOV_CMD),
1036             ("install-no-lmdb", "make install"),
1037             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1038             ("distcheck", "make distcheck"),
1039             ("clean", "make clean"),
1040         ],
1041     },
1042
1043     "tdb": {
1044         "sequence": [
1045             ("random-sleep", random_sleep(60, 600)),
1046             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1047             ("make", "make"),
1048             ("install", "make install"),
1049             ("test", "make test"),
1050             ("lcov", LCOV_CMD),
1051             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1052             ("distcheck", "make distcheck"),
1053             ("clean", "make clean"),
1054         ],
1055     },
1056
1057     "talloc": {
1058         "sequence": [
1059             ("random-sleep", random_sleep(60, 600)),
1060             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1061             ("make", "make"),
1062             ("install", "make install"),
1063             ("test", "make test"),
1064             ("lcov", LCOV_CMD),
1065             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1066             ("distcheck", "make distcheck"),
1067             ("clean", "make clean"),
1068         ],
1069     },
1070
1071     "replace": {
1072         "sequence": [
1073             ("random-sleep", random_sleep(60, 600)),
1074             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1075             ("make", "make"),
1076             ("install", "make install"),
1077             ("test", "make test"),
1078             ("lcov", LCOV_CMD),
1079             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1080             ("distcheck", "make distcheck"),
1081             ("clean", "make clean"),
1082         ],
1083     },
1084
1085     "tevent": {
1086         "sequence": [
1087             ("random-sleep", random_sleep(60, 600)),
1088             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1089             ("make", "make"),
1090             ("install", "make install"),
1091             ("test", "make test"),
1092             ("lcov", LCOV_CMD),
1093             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1094             ("distcheck", "make distcheck"),
1095             ("clean", "make clean"),
1096         ],
1097     },
1098
1099     "pidl": {
1100         "git-clone-required": True,
1101         "sequence": [
1102             ("random-sleep", random_sleep(60, 600)),
1103             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1104             ("touch", "touch *.yp"),
1105             ("make", "make"),
1106             ("test", "make test"),
1107             ("install", "make install"),
1108             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1109             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1110             ("clean", "make clean"),
1111         ],
1112     },
1113
1114     # these are useful for debugging autobuild
1115     "pass": {
1116         "sequence": [
1117             ("pass", 'echo passing && /bin/true'),
1118         ],
1119     },
1120     "fail": {
1121         "sequence": [
1122             ("fail", 'echo failing && /bin/false'),
1123         ],
1124     },
1125 }
1126
1127 defaulttasks = list(tasks.keys())
1128
1129 defaulttasks.remove("pass")
1130 defaulttasks.remove("fail")
1131
1132 # The build tasks will be brought in by the test tasks as needed
1133 defaulttasks.remove("samba-def-build")
1134 defaulttasks.remove("samba-nt4-build")
1135 defaulttasks.remove("samba-mit-build")
1136 defaulttasks.remove("samba-h5l-build")
1137 defaulttasks.remove("samba-no-opath-build")
1138
1139 # This is not a normal test, but a task to support manually running
1140 # one test under autobuild
1141 defaulttasks.remove("samba-test-only")
1142
1143 # Only built on GitLab CI and not in the default autobuild because it
1144 # uses too much space (4GB of semi-static binaries)
1145 defaulttasks.remove("samba-fuzz")
1146
1147 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1148 # container where a simulated FIPS mode is possible.
1149 defaulttasks.remove("samba-fips")
1150
1151 # The MIT build runs on a current Fedora where an up to date MIT KDC
1152 # is already packaged.  This avoids needing to backport a current MIT
1153 # to the default Ubuntu 18.04, particularly during development, and
1154 # the need to install on the shared sn-devel-184.
1155
1156 defaulttasks.remove("samba-mitkrb5")
1157 defaulttasks.remove("samba-admem-mit")
1158 defaulttasks.remove("samba-addc-mit-1")
1159 defaulttasks.remove("samba-addc-mit-4a")
1160 defaulttasks.remove("samba-addc-mit-4b")
1161
1162 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1163     defaulttasks.remove("samba-o3")
1164
1165
1166 def do_print(msg):
1167     print("%s" % msg)
1168     sys.stdout.flush()
1169     sys.stderr.flush()
1170
1171
1172 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1173     if show is None:
1174         show = options.verbose
1175     if show:
1176         do_print("Running: '%s' in '%s'" % (cmd, dir))
1177     if output:
1178         out = check_output([cmd], shell=True, cwd=dir)
1179         return out.decode(encoding='utf-8', errors='backslashreplace')
1180     elif checkfail:
1181         return check_call(cmd, shell=True, cwd=dir)
1182     else:
1183         return call(cmd, shell=True, cwd=dir)
1184
1185 def rmdir_force(dirname, re_raise=True):
1186     try:
1187         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1188                 dirname, dirname, dirname), output=True, show=True)
1189     except CalledProcessError as e:
1190         do_print("Failed: '%s'" % (str(e)))
1191         run_cmd("tree %s" % dirname, output=True, show=True)
1192         if re_raise:
1193             raise
1194         return False
1195     return True
1196
1197 class builder(object):
1198     '''handle build of one directory'''
1199
1200     def __init__(self, name, definition):
1201         self.name = name
1202         self.dir = builddirs.get(name, '.')
1203         self.tag = self.name.replace('/', '_')
1204         self.definition = definition
1205         self.sequence = definition["sequence"]
1206         self.git_clone_required = False
1207         if "git-clone-required" in definition:
1208             self.git_clone_required = bool(definition["git-clone-required"])
1209         self.proc = None
1210         self.done = False
1211         self.next = 0
1212         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1213         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1214         if options.verbose:
1215             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1216             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1217         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1218         self.stdout = open(self.stdout_path, 'w')
1219         self.stderr = open(self.stderr_path, 'w')
1220         self.stdin  = open("/dev/null", 'r')
1221         self.builder_dir = "%s/%s" % (testbase, self.tag)
1222         self.test_source_dir = self.builder_dir
1223         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1224         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1225         self.prefix = "%s/%s" % (test_prefix, self.tag)
1226         self.consumers = []
1227         self.producer = None
1228
1229         if self.git_clone_required:
1230             assert "dependency" not in definition
1231
1232     def mark_existing(self):
1233         do_print('%s: Mark as existing dependency' % self.name)
1234         self.next = len(self.sequence)
1235         self.done = True
1236
1237     def add_consumer(self, consumer):
1238         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1239         consumer.producer = self
1240         consumer.test_source_dir = self.test_source_dir
1241         self.consumers.append(consumer)
1242
1243     def start_next(self):
1244         if self.producer is not None:
1245             if not self.producer.done:
1246                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1247                 return
1248
1249         if self.next == 0:
1250             rmdir_force(self.builder_dir)
1251             rmdir_force(self.prefix)
1252             if self.producer is not None:
1253                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1254             elif not self.git_clone_required:
1255                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1256             else:
1257                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1258
1259         if self.next == len(self.sequence):
1260             if not self.done:
1261                 do_print('%s: Completed OK' % self.name)
1262                 self.done = True
1263             if not options.nocleanup and len(self.consumers) == 0:
1264                 do_print('%s: Cleaning up' % self.name)
1265                 rmdir_force(self.builder_dir)
1266                 rmdir_force(self.prefix)
1267             for consumer in self.consumers:
1268                 if consumer.next != 0:
1269                     continue
1270                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1271                 consumer.start_next()
1272             if self.producer is not None:
1273                 self.producer.consumers.remove(self)
1274                 assert self.producer.done
1275                 self.producer.start_next()
1276             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1277             return
1278         (self.stage, self.cmd) = self.sequence[self.next]
1279         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1280         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1281         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1282         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1283         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1284         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1285         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1286         self.cmd = self.cmd.replace("${NAME}", self.name)
1287         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1288         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1289         self.proc = Popen(self.cmd, shell=True,
1290                           close_fds=True, cwd=self.cwd,
1291                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1292         self.next += 1
1293
1294 def expand_dependencies(n):
1295     deps = list()
1296     if "dependency" in tasks[n]:
1297         depname = tasks[n]["dependency"]
1298         assert depname in tasks
1299         sdeps = expand_dependencies(depname)
1300         assert n not in sdeps
1301         for sdep in sdeps:
1302             deps.append(sdep)
1303         deps.append(depname)
1304     return deps
1305
1306
1307 class buildlist(object):
1308     '''handle build of multiple directories'''
1309
1310     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1311         self.tail_proc = None
1312         self.retry = None
1313         if not tasknames:
1314             if options.restrict_tests:
1315                 tasknames = ["samba-test-only"]
1316             else:
1317                 tasknames = defaulttasks
1318
1319         given_tasknames = tasknames.copy()
1320         implicit_tasknames = []
1321         for n in given_tasknames:
1322             deps = expand_dependencies(n)
1323             for dep in deps:
1324                 if dep in given_tasknames:
1325                     continue
1326                 if dep in implicit_tasknames:
1327                     continue
1328                 implicit_tasknames.append(dep)
1329
1330         tasknames = implicit_tasknames.copy()
1331         tasknames.extend(given_tasknames)
1332         do_print("given_tasknames: %s" % given_tasknames)
1333         do_print("implicit_tasknames: %s" % implicit_tasknames)
1334         do_print("tasknames: %s" % tasknames)
1335         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1336
1337         if options.retry:
1338             rebase_remote = "rebaseon"
1339             retry_task = {
1340                     "git-clone-required": True,
1341                     "sequence": [
1342                             ("retry",
1343                             '''set -e
1344                             git remote add -t %s %s %s
1345                             git fetch %s
1346                             while :; do
1347                               sleep 60
1348                               git describe %s/%s > old_remote_branch.desc
1349                               git fetch %s
1350                               git describe %s/%s > remote_branch.desc
1351                               diff old_remote_branch.desc remote_branch.desc
1352                             done
1353                            ''' % (
1354                                rebase_branch, rebase_remote, rebase_url,
1355                                rebase_remote,
1356                                rebase_remote, rebase_branch,
1357                                rebase_remote,
1358                                rebase_remote, rebase_branch
1359                             ))]}
1360
1361             self.retry = builder('retry', retry_task)
1362             self.need_retry = False
1363
1364         if options.skip_dependencies:
1365             for b in self.tlist:
1366                 if b.name in implicit_tasknames:
1367                     b.mark_existing()
1368
1369         for b in self.tlist:
1370             do_print("b.name=%s" % b.name)
1371             if "dependency" not in b.definition:
1372                 continue
1373             depname = b.definition["dependency"]
1374             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1375             for p in self.tlist:
1376                 if p.name == depname:
1377                     p.add_consumer(b)
1378
1379     def kill_kids(self):
1380         if self.tail_proc is not None:
1381             self.tail_proc.terminate()
1382             self.tail_proc.wait()
1383             self.tail_proc = None
1384         if self.retry is not None:
1385             self.retry.proc.terminate()
1386             self.retry.proc.wait()
1387             self.retry = None
1388         for b in self.tlist:
1389             if b.proc is not None:
1390                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1391                 b.proc.terminate()
1392                 b.proc.wait()
1393                 b.proc = None
1394
1395     def wait_one(self):
1396         while True:
1397             none_running = True
1398             for b in self.tlist:
1399                 if b.proc is None:
1400                     continue
1401                 none_running = False
1402                 b.status = b.proc.poll()
1403                 if b.status is None:
1404                     continue
1405                 b.proc = None
1406                 return b
1407             if options.retry:
1408                 ret = self.retry.proc.poll()
1409                 if ret is not None:
1410                     self.need_retry = True
1411                     self.retry = None
1412                     return None
1413             if none_running:
1414                 return None
1415             time.sleep(0.1)
1416
1417     def run(self):
1418         for b in self.tlist:
1419             b.start_next()
1420         if options.retry:
1421             self.retry.start_next()
1422         while True:
1423             b = self.wait_one()
1424             if options.retry and self.need_retry:
1425                 self.kill_kids()
1426                 do_print("retry needed")
1427                 return (0, None, None, None, "retry")
1428             if b is None:
1429                 break
1430             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1431                 self.kill_kids()
1432                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1433             b.start_next()
1434         self.kill_kids()
1435         return (0, None, None, None, "All OK")
1436
1437     def write_system_info(self, filename):
1438         with open(filename, 'w') as f:
1439             for cmd in ['uname -a',
1440                         'lsb_release -a',
1441                         'free',
1442                         'mount',
1443                         'cat /proc/cpuinfo',
1444                         'cc --version',
1445                         'df -m .',
1446                         'df -m %s' % testbase]:
1447                 try:
1448                     out = run_cmd(cmd, output=True, checkfail=False)
1449                 except CalledProcessError as e:
1450                     out = "<failed: %s>" % str(e)
1451                 print('### %s' % cmd, file=f)
1452                 print(out, file=f)
1453                 print(file=f)
1454
1455     def tarlogs(self, fname):
1456         with tarfile.open(fname, "w:gz") as tar:
1457             for b in self.tlist:
1458                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1459                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1460             if os.path.exists("autobuild.log"):
1461                 tar.add("autobuild.log")
1462             filename = 'system-info.txt'
1463             self.write_system_info(filename)
1464             tar.add(filename)
1465
1466     def remove_logs(self):
1467         for b in self.tlist:
1468             os.unlink(b.stdout_path)
1469             os.unlink(b.stderr_path)
1470
1471     def start_tail(self):
1472         cmd = ["tail", "-f"]
1473         for b in self.tlist:
1474             cmd.append(b.stdout_path)
1475             cmd.append(b.stderr_path)
1476         self.tail_proc = Popen(cmd, close_fds=True)
1477
1478
1479 def cleanup(do_raise=False):
1480     if options.nocleanup:
1481         return
1482     run_cmd("stat %s || true" % test_tmpdir, show=True)
1483     run_cmd("stat %s" % testbase, show=True)
1484     do_print("Cleaning up %r" % cleanup_list)
1485     for d in cleanup_list:
1486         ok = rmdir_force(d, re_raise=False)
1487         if ok:
1488             continue
1489         if os.path.isdir(d):
1490             do_print("Killing, waiting and retry")
1491             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1492         else:
1493             do_print("Waiting and retry")
1494         time.sleep(1)
1495         rmdir_force(d, re_raise=do_raise)
1496
1497
1498 def daemonize(logfile):
1499     pid = os.fork()
1500     if pid == 0:  # Parent
1501         os.setsid()
1502         pid = os.fork()
1503         if pid != 0:  # Actual daemon
1504             os._exit(0)
1505     else:  # Grandparent
1506         os._exit(0)
1507
1508     import resource      # Resource usage information.
1509     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1510     if maxfd == resource.RLIM_INFINITY:
1511         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1512     for fd in range(0, maxfd):
1513         try:
1514             os.close(fd)
1515         except OSError:
1516             pass
1517     os.open(logfile, os.O_RDWR | os.O_CREAT)
1518     os.dup2(0, 1)
1519     os.dup2(0, 2)
1520
1521
1522 def write_pidfile(fname):
1523     '''write a pid file, cleanup on exit'''
1524     with open(fname, mode='w') as f:
1525         f.write("%u\n" % os.getpid())
1526
1527
1528 def rebase_tree(rebase_url, rebase_branch="master"):
1529     rebase_remote = "rebaseon"
1530     do_print("Rebasing on %s" % rebase_url)
1531     run_cmd("git describe HEAD", show=True, dir=test_master)
1532     run_cmd("git remote add -t %s %s %s" %
1533             (rebase_branch, rebase_remote, rebase_url),
1534             show=True, dir=test_master)
1535     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1536     if options.fix_whitespace:
1537         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1538                 (rebase_remote, rebase_branch),
1539                 show=True, dir=test_master)
1540     else:
1541         run_cmd("git rebase --force-rebase %s/%s" %
1542                 (rebase_remote, rebase_branch),
1543                 show=True, dir=test_master)
1544     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1545                    (rebase_remote, rebase_branch),
1546                    dir=test_master, output=True)
1547     if diff == '':
1548         do_print("No differences between HEAD and %s/%s - exiting" %
1549                  (rebase_remote, rebase_branch))
1550         sys.exit(0)
1551     run_cmd("git describe %s/%s" %
1552             (rebase_remote, rebase_branch),
1553             show=True, dir=test_master)
1554     run_cmd("git describe HEAD", show=True, dir=test_master)
1555     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1556             (rebase_remote, rebase_branch),
1557             show=True, dir=test_master)
1558
1559
1560 def push_to(push_url, push_branch="master"):
1561     push_remote = "pushto"
1562     do_print("Pushing to %s" % push_url)
1563     if options.mark:
1564         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1565         run_cmd("git commit --amend -c HEAD", dir=test_master)
1566         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1567         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1568     run_cmd("git remote add -t %s %s %s" %
1569             (push_branch, push_remote, push_url),
1570             show=True, dir=test_master)
1571     run_cmd("git push %s +HEAD:%s" %
1572             (push_remote, push_branch),
1573             show=True, dir=test_master)
1574
1575
1576 def send_email(subject, text, log_tar):
1577     if options.email is None:
1578         do_print("not sending email because the recipient is not set")
1579         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1580                  (subject, text))
1581         return
1582     outer = MIMEMultipart()
1583     outer['Subject'] = subject
1584     outer['To'] = options.email
1585     outer['From'] = options.email_from
1586     outer['Date'] = email.utils.formatdate(localtime=True)
1587     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1588     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1589     if options.attach_logs:
1590         with open(log_tar, 'rb') as fp:
1591             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1592         # Set the filename parameter
1593         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1594         outer.attach(msg)
1595     content = outer.as_string()
1596     s = smtplib.SMTP(options.email_server)
1597     email_user = os.getenv('SMTP_USERNAME')
1598     email_password = os.getenv('SMTP_PASSWORD')
1599     if email_user is not None:
1600         s.starttls()
1601         s.login(email_user, email_password)
1602
1603     s.sendmail(options.email_from, [options.email], content)
1604     s.set_debuglevel(1)
1605     s.quit()
1606
1607
1608 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1609                   elapsed_time, log_base=None, add_log_tail=True):
1610     '''send an email to options.email about the failure'''
1611     elapsed_minutes = elapsed_time / 60.0
1612     if log_base is None:
1613         log_base = gitroot
1614     text = '''
1615 Dear Developer,
1616
1617 Your autobuild on %s failed after %.1f minutes
1618 when trying to test %s with the following error:
1619
1620    %s
1621
1622 the autobuild has been abandoned. Please fix the error and resubmit.
1623
1624 A summary of the autobuild process is here:
1625
1626   %s/autobuild.log
1627 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1628
1629     if options.restrict_tests:
1630         text += """
1631 The build was restricted to tests matching %s\n""" % options.restrict_tests
1632
1633     if failed_task != 'rebase':
1634         text += '''
1635 You can see logs of the failed task here:
1636
1637   %s/%s.stdout
1638   %s/%s.stderr
1639
1640 or you can get full logs of all tasks in this job here:
1641
1642   %s/logs.tar.gz
1643
1644 The top commit for the tree that was built was:
1645
1646 %s
1647
1648 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1649
1650     if add_log_tail:
1651         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1652         lines = f.readlines()
1653         log_tail = "".join(lines[-50:])
1654         num_lines = len(lines)
1655         if num_lines < 50:
1656             # Also include stderr (compile failures) if < 50 lines of stdout
1657             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1658             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1659
1660         text += '''
1661 The last 50 lines of log messages:
1662
1663 %s
1664     ''' % log_tail
1665         f.close()
1666
1667     logs = os.path.join(gitroot, 'logs.tar.gz')
1668     send_email('autobuild[%s] failure on %s for task %s during %s'
1669                % (options.branch, platform.node(), failed_task, failed_stage),
1670                text, logs)
1671
1672
1673 def email_success(elapsed_time, log_base=None):
1674     '''send an email to options.email about a successful build'''
1675     if log_base is None:
1676         log_base = gitroot
1677     text = '''
1678 Dear Developer,
1679
1680 Your autobuild on %s has succeeded after %.1f minutes.
1681
1682 ''' % (platform.node(), elapsed_time / 60.)
1683
1684     if options.restrict_tests:
1685         text += """
1686 The build was restricted to tests matching %s\n""" % options.restrict_tests
1687
1688     if options.keeplogs:
1689         text += '''
1690
1691 you can get full logs of all tasks in this job here:
1692
1693   %s/logs.tar.gz
1694
1695 ''' % log_base
1696
1697     text += '''
1698 The top commit for the tree that was built was:
1699
1700 %s
1701 ''' % top_commit_msg
1702
1703     logs = os.path.join(gitroot, 'logs.tar.gz')
1704     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1705                text, logs)
1706
1707
1708 # get the top commit message, for emails
1709 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1710
1711 try:
1712     if options.skip_dependencies:
1713         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1714     else:
1715         os.makedirs(testbase)
1716 except Exception as reason:
1717     raise Exception("Unable to create %s : %s" % (testbase, reason))
1718 cleanup_list.append(testbase)
1719
1720 if options.daemon:
1721     logfile = os.path.join(testbase, "log")
1722     do_print("Forking into the background, writing progress to %s" % logfile)
1723     daemonize(logfile)
1724
1725 write_pidfile(gitroot + "/autobuild.pid")
1726
1727 start_time = time.time()
1728
1729 while True:
1730     try:
1731         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1732         os.makedirs(test_tmpdir)
1733         # The waf uninstall code removes empty directories all the way
1734         # up the tree.  Creating a file in test_tmpdir stops it from
1735         # being removed.
1736         run_cmd("touch %s" % os.path.join(test_tmpdir,
1737                                           ".directory-is-not-empty"), show=True)
1738         run_cmd("stat %s" % test_tmpdir, show=True)
1739         run_cmd("stat %s" % testbase, show=True)
1740         if options.skip_dependencies:
1741             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1742         else:
1743             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1744     except Exception:
1745         cleanup()
1746         raise
1747
1748     try:
1749         if options.rebase is not None:
1750             rebase_tree(options.rebase, rebase_branch=options.branch)
1751     except Exception:
1752         cleanup_list.append(gitroot + "/autobuild.pid")
1753         cleanup()
1754         elapsed_time = time.time() - start_time
1755         email_failure(-1, 'rebase', 'rebase', 'rebase',
1756                       'rebase on %s failed' % options.branch,
1757                       elapsed_time, log_base=options.log_base)
1758         sys.exit(1)
1759
1760     try:
1761         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1762         if options.tail:
1763             blist.start_tail()
1764         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1765         if status != 0 or errstr != "retry":
1766             break
1767         cleanup(do_raise=True)
1768     except Exception:
1769         cleanup()
1770         raise
1771
1772 cleanup_list.append(gitroot + "/autobuild.pid")
1773
1774 do_print(errstr)
1775
1776 blist.kill_kids()
1777 if options.tail:
1778     do_print("waiting for tail to flush")
1779     time.sleep(1)
1780
1781 elapsed_time = time.time() - start_time
1782 if status == 0:
1783     if options.passcmd is not None:
1784         do_print("Running passcmd: %s" % options.passcmd)
1785         run_cmd(options.passcmd, dir=test_master)
1786     if options.pushto is not None:
1787         push_to(options.pushto, push_branch=options.branch)
1788     if options.keeplogs or options.attach_logs:
1789         blist.tarlogs("logs.tar.gz")
1790         do_print("Logs in logs.tar.gz")
1791     if options.always_email:
1792         email_success(elapsed_time, log_base=options.log_base)
1793     blist.remove_logs()
1794     cleanup()
1795     do_print(errstr)
1796     sys.exit(0)
1797
1798 # something failed, gather a tar of the logs
1799 blist.tarlogs("logs.tar.gz")
1800
1801 if options.email is not None:
1802     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1803                   elapsed_time, log_base=options.log_base)
1804 else:
1805     elapsed_minutes = elapsed_time / 60.0
1806     print('''
1807
1808 ####################################################################
1809
1810 AUTOBUILD FAILURE
1811
1812 Your autobuild[%s] on %s failed after %.1f minutes
1813 when trying to test %s with the following error:
1814
1815    %s
1816
1817 the autobuild has been abandoned. Please fix the error and resubmit.
1818
1819 ####################################################################
1820
1821 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1822
1823 cleanup()
1824 do_print(errstr)
1825 do_print("Logs in logs.tar.gz")
1826 sys.exit(status)