script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 # allow autobuild to run within git rebase -i
  35 if "GIT_DIR" in os.environ:
  36     del os.environ["GIT_DIR"]
  37 if "GIT_WORK_TREE" in os.environ:
  38     del os.environ["GIT_WORK_TREE"]
  39
  40 def find_git_root():
  41     '''get to the top of the git repo'''
  42     p = os.getcwd()
  43     while p != '/':
  44         if os.path.exists(os.path.join(p, ".git")):
  45             return p
  46         p = os.path.abspath(os.path.join(p, '..'))
  47     return None
  48
  49
  50 gitroot = find_git_root()
  51 if gitroot is None:
  52     raise Exception("Failed to find git root")
  53
  54
  55 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  56
  57 parser = OptionParser()
  58 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  59 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  60 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  61 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  62 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  63                   default=def_testbase)
  64 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  65                   default=None)
  66 parser.add_option("--passcmd", help="command to run on success", default=None)
  67 parser.add_option("--verbose", help="show all commands as they are run",
  68                   default=False, action="store_true")
  69 parser.add_option("--rebase", help="rebase on the given tree before testing",
  70                   default=None, type='str')
  71 parser.add_option("--pushto", help="push to a git url on success",
  72                   default=None, type='str')
  73 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  74                   default=False, action="store_true")
  75 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  76                   default=False, action="store_true")
  77 parser.add_option("--retry", help="automatically retry if master changes",
  78                   default=False, action="store_true")
  79 parser.add_option("--email", help="send email to the given address on failure",
  80                   type='str', default=None)
  81 parser.add_option("--email-from", help="send email from the given address",
  82                   type='str', default="autobuild@samba.org")
  83 parser.add_option("--email-server", help="send email via the given server",
  84                   type='str', default='localhost')
  85 parser.add_option("--always-email", help="always send email, even on success",
  86                   action="store_true")
  87 parser.add_option("--daemon", help="daemonize after initial setup",
  88                   action="store_true")
  89 parser.add_option("--branch", help="the branch to work on (default=master)",
  90                   default="master", type='str')
  91 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  92                   default=gitroot, type='str')
  93 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  94                   default=False, action="store_true")
  95 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  96                   default='')
  97 parser.add_option("--enable-coverage", dest='enable_coverage',
  98                   action="store_const", const='--enable-coverage', default='',
  99                   help="Add --enable-coverage option while configure")
 100
 101 (options, args) = parser.parse_args()
 102
 103 if options.retry:
 104     if options.rebase is None:
 105         raise Exception('You can only use --retry if you also rebase')
 106
 107 if options.full_testbase is not None:
 108     testbase = options.full_testbase
 109 else:
 110     testbase = "%s/b%u" % (options.testbase, os.getpid())
 111 test_master = "%s/master" % testbase
 112 test_prefix = "%s/prefix" % testbase
 113 test_tmpdir = "%s/tmp" % testbase
 114 os.environ['TMPDIR'] = test_tmpdir
 115
 116 if options.enable_coverage:
 117     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 118 else:
 119     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 120
 121 if options.enable_coverage:
 122     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 123 else:
 124     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 125
 126 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 127
 128
 129 def check_symbols(sofile, expected_symbols=""):
 130     return "objdump --dynamic-syms " + sofile + " | " + \
 131            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 132
 133 if args:
 134     # If we are only running specific test,
 135     # do not sleep randomly to wait for it to start
 136     def random_sleep(low, high):
 137         return 'sleep 1'
 138 else:
 139     def random_sleep(low, high):
 140         return 'sleep {}'.format(random.randint(low, high))
 141
 142 cleanup_list = []
 143
 144 builddirs = {
 145     "ctdb": "ctdb",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl",
 152     "docs-xml": "docs-xml"
 153 }
 154
 155 ctdb_configure_params = " --enable-developer ${PREFIX}"
 156 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 157
 158 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 159 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 160 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 161 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 162 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 163 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 164 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 165
 166
 167 def format_option(name, value=None):
 168     """Format option as str list."""
 169     if value is None:  # boolean option
 170         return [name]
 171     if not isinstance(value, list):  # single value option
 172         value = [value]
 173     # repeatable option
 174     return ['{}={}'.format(name, item) for item in value]
 175
 176
 177 def make_test(
 178         cmd='make testonly',
 179         INJECT_SELFTEST_PREFIX=1,
 180         TESTS='',
 181         include_envs=None,
 182         exclude_envs=None):
 183
 184     test_options = []
 185     if include_envs:
 186         test_options = format_option('--include-env', include_envs)
 187     if exclude_envs:
 188         test_options = format_option('--exclude-env', exclude_envs)
 189     if test_options:
 190         # join envs options to original test options
 191         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 192
 193     _options = []
 194
 195     # Allow getting a full CI with
 196     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 197
 198     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 199
 200     if int(FAIL_IMMEDIATELY):
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     if INJECT_SELFTEST_PREFIX:
 206         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 207         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 208
 209     return ' '.join([cmd] + _options)
 210
 211
 212 # When updating this list, also update .gitlab-ci.yml to add the job
 213 # and to make it a dependency of 'page' for the coverage report.
 214
 215 tasks = {
 216     "ctdb": {
 217         "sequence": [
 218             ("random-sleep", random_sleep(300, 900)),
 219             ("configure", "./configure " + ctdb_configure_params),
 220             ("make", "make all"),
 221             ("install", "make install"),
 222             ("test", "make autotest"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("clean", "make clean"),
 225         ],
 226     },
 227     "docs-xml": {
 228         "sequence": [
 229             ("random-sleep", random_sleep(300, 900)),
 230             ("autoconf", "autoconf"),
 231             ("configure", "./configure"),
 232             ("make", "make html htmlman"),
 233             ("publish-docs", PUBLISH_DOCS),
 234             ("clean", "make clean"),
 235         ],
 236     },
 237
 238     "samba-def-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-mit-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-nt4-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     "samba-h5l-build": {
 269         "git-clone-required": True,
 270         "sequence": [
 271             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 272             ("make", "make -j"),
 273             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 274             ("chmod-R-a-w", "chmod -R a-w ."),
 275         ],
 276     },
 277
 278     "samba-without-smb1-build": {
 279         "git-clone-required": True,
 280         "sequence": [
 281             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 282             ("make", "make -j"),
 283             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 284             ("chmod-R-a-w", "chmod -R a-w ."),
 285         ],
 286     },
 287
 288     "samba-no-opath-build": {
 289         "git-clone-required": True,
 290         "sequence": [
 291             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 292             ("make", "make -j"),
 293             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 294             ("chmod-R-a-w", "chmod -R a-w ."),
 295         ],
 296     },
 297
 298     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 299     "samba": {
 300         "sequence": [
 301             ("random-sleep", random_sleep(300, 900)),
 302             ("configure", "./configure.developer" + samba_configure_params),
 303             ("make", "make -j"),
 304             ("test", make_test(exclude_envs=[
 305             "none",
 306             "nt4_dc",
 307             "nt4_dc_smb1",
 308             "nt4_dc_smb1_done",
 309             "nt4_dc_schannel",
 310             "nt4_member",
 311             "ad_dc",
 312             "ad_dc_smb1",
 313             "ad_dc_smb1_done",
 314             "ad_dc_backup",
 315             "ad_dc_ntvfs",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_slowtests",
 319             "ad_dc_no_nss",
 320             "ad_dc_no_ntlm",
 321             "fl2003dc",
 322             "fl2008dc",
 323             "fl2008r2dc",
 324             "ad_member",
 325             "ad_member_idmap_rid",
 326             "admem_idmap_autorid",
 327             "ad_member_idmap_ad",
 328             "ad_member_rfc2307",
 329             "ad_member_oneway",
 330             "chgdcpass",
 331             "vampire_2000_dc",
 332             "fl2000dc",
 333             "fileserver",
 334             "fileserver_smb1",
 335             "fileserver_smb1_done",
 336             "maptoguest",
 337             "simpleserver",
 338             "backupfromdc",
 339             "restoredc",
 340             "renamedc",
 341             "offlinebackupdc",
 342             "labdc",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             "promoted_dc",
 346             "vampire_dc",
 347             "rodc",
 348             "ad_dc_default",
 349             "ad_dc_default_smb1",
 350             "ad_dc_default_smb1_done",
 351             "ad_dc_slowtests",
 352             "schema_pair_dc",
 353             "schema_dc",
 354             "clusteredmember",
 355             "ad_dc_fips",
 356             "ad_member_fips",
 357             ])),
 358             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 359             ("lcov", LCOV_CMD),
 360             ("install", "make install"),
 361             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 362             ("clean", "make clean"),
 363         ],
 364     },
 365
 366     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 367     "samba-mitkrb5": {
 368         "sequence": [
 369             ("random-sleep", random_sleep(300, 900)),
 370             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 371             ("make", "make -j"),
 372             ("test", make_test(exclude_envs=[
 373             "none",
 374             "nt4_dc",
 375             "nt4_dc_smb1",
 376             "nt4_dc_smb1_done",
 377             "nt4_dc_schannel",
 378             "nt4_member",
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_backup",
 383             "ad_dc_ntvfs",
 384             "ad_dc_default",
 385             "ad_dc_default_smb1",
 386             "ad_dc_default_smb1_done",
 387             "ad_dc_slowtests",
 388             "ad_dc_no_nss",
 389             "ad_dc_no_ntlm",
 390             "fl2003dc",
 391             "fl2008dc",
 392             "fl2008r2dc",
 393             "ad_member",
 394             "ad_member_idmap_rid",
 395             "admem_idmap_autorid",
 396             "ad_member_idmap_ad",
 397             "ad_member_rfc2307",
 398             "ad_member_oneway",
 399             "chgdcpass",
 400             "vampire_2000_dc",
 401             "fl2000dc",
 402             "fileserver",
 403             "fileserver_smb1",
 404             "fileserver_smb1_done",
 405             "maptoguest",
 406             "simpleserver",
 407             "backupfromdc",
 408             "restoredc",
 409             "renamedc",
 410             "offlinebackupdc",
 411             "labdc",
 412             "preforkrestartdc",
 413             "proclimitdc",
 414             "promoted_dc",
 415             "vampire_dc",
 416             "rodc",
 417             "ad_dc_default",
 418             "ad_dc_default_smb1",
 419             "ad_dc_default_smb1_done",
 420             "ad_dc_slowtests",
 421             "schema_pair_dc",
 422             "schema_dc",
 423             "clusteredmember",
 424             "ad_dc_fips",
 425             "ad_member_fips",
 426             ])),
 427             ("lcov", LCOV_CMD),
 428             ("install", "make install"),
 429             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 430             ("clean", "make clean"),
 431         ],
 432     },
 433
 434     "samba-nt4": {
 435         "dependency": "samba-nt4-build",
 436         "sequence": [
 437             ("random-sleep", random_sleep(300, 900)),
 438             ("test", make_test(include_envs=[
 439             "nt4_dc",
 440             "nt4_dc_smb1",
 441             "nt4_dc_smb1_done",
 442             "nt4_dc_schannel",
 443             "nt4_member",
 444             "simpleserver",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-fileserver": {
 452         "dependency": "samba-h5l-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(include_envs=[
 456             "fileserver",
 457             "fileserver_smb1",
 458             "fileserver_smb1_done",
 459             "maptoguest",
 460             "ktest", # ktest is also tested in samba-ktest-mit samba
 461                      # and samba-mitkrb5 but is tested here against
 462                      # a system Heimdal
 463             ])),
 464             ("lcov", LCOV_CMD),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466         ],
 467     },
 468
 469     "samba-fileserver-without-smb1": {
 470         "dependency": "samba-without-smb1-build",
 471         "sequence": [
 472             ("random-sleep", random_sleep(300, 900)),
 473             ("test", make_test(include_envs=["fileserver"])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     # This is a full build without the AD DC so we test the build with
 480     # MIT Kerberos from the current system.  Runtime behaviour is
 481     # confirmed via the ktest (static ccache and keytab) environment
 482
 483     # This environment also used to confirm we can still build with --with-libunwind
 484     "samba-ktest-mit": {
 485         "sequence": [
 486             ("random-sleep", random_sleep(300, 900)),
 487             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 488             ("make", "make -j"),
 489             ("test", make_test(include_envs=[
 490             "ktest", # ktest is also tested in fileserver, samba and
 491                      # samba-mitkrb5 but is tested here against a
 492                      # system MIT krb5
 493             ])),
 494             ("lcov", LCOV_CMD),
 495             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 496         ],
 497     },
 498
 499     "samba-admem": {
 500         "dependency": "samba-def-build",
 501         "sequence": [
 502             ("random-sleep", random_sleep(300, 900)),
 503             ("test", make_test(include_envs=[
 504             "ad_member",
 505             "ad_member_idmap_rid",
 506             "admem_idmap_autorid",
 507             "ad_member_idmap_ad",
 508             "ad_member_rfc2307",
 509             "ad_member_offlogon",
 510             ])),
 511             ("lcov", LCOV_CMD),
 512             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 513         ],
 514     },
 515
 516     "samba-no-opath1": {
 517         "dependency": "samba-no-opath-build",
 518         "sequence": [
 519             ("random-sleep", random_sleep(300, 900)),
 520             ("test", make_test(
 521                 cmd="make testonly DISABLE_OPATH=1",
 522                 include_envs=[
 523                 "nt4_dc",
 524                 "nt4_dc_smb1",
 525                 "nt4_dc_smb1_done",
 526                 "nt4_dc_schannel",
 527                 "nt4_member",
 528                 "simpleserver",
 529                 ])),
 530             ("lcov", LCOV_CMD),
 531             ("check-clean-tree", "script/clean-source-tree.sh"),
 532         ],
 533     },
 534
 535     "samba-no-opath2": {
 536         "dependency": "samba-no-opath-build",
 537         "sequence": [
 538             ("random-sleep", random_sleep(300, 900)),
 539             ("test", make_test(
 540                 cmd="make testonly DISABLE_OPATH=1",
 541                 include_envs=[
 542                 "fileserver",
 543                 "fileserver_smb1",
 544                 "fileserver_smb1_done",
 545                 ])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", "script/clean-source-tree.sh"),
 548         ],
 549     },
 550
 551     "samba-ad-dc-1": {
 552         "dependency": "samba-def-build",
 553         "sequence": [
 554             ("random-sleep", random_sleep(1, 1)),
 555             ("test", make_test(include_envs=[
 556             "ad_dc",
 557             "ad_dc_smb1",
 558             "ad_dc_smb1_done",
 559             "ad_dc_no_nss",
 560             "ad_dc_no_ntlm",
 561             ])),
 562             ("lcov", LCOV_CMD),
 563             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 564         ],
 565     },
 566
 567     "samba-ad-dc-2": {
 568         "dependency": "samba-def-build",
 569         "sequence": [
 570             ("random-sleep", random_sleep(1, 1)),
 571             ("test", make_test(include_envs=[
 572             "vampire_dc",
 573             "vampire_2000_dc",
 574             "rodc",
 575             ])),
 576             ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     "samba-ad-dc-3": {
 582         "dependency": "samba-def-build",
 583         "sequence": [
 584             ("random-sleep", random_sleep(1, 1)),
 585             ("test", make_test(include_envs=[
 586             "promoted_dc",
 587             "chgdcpass",
 588             "preforkrestartdc",
 589             "proclimitdc",
 590             ])),
 591             ("lcov", LCOV_CMD),
 592             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 593         ],
 594     },
 595
 596     "samba-ad-dc-4a": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=[
 601             "fl2000dc",
 602             "ad_member_oneway",
 603             "fl2003dc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609     "samba-ad-dc-4b": {
 610         "dependency": "samba-def-build",
 611         "sequence": [
 612             ("random-sleep", random_sleep(1, 1)),
 613             ("test", make_test(include_envs=[
 614             "fl2008dc",
 615             "fl2008r2dc",
 616             ])),
 617             ("lcov", LCOV_CMD),
 618             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 619         ],
 620     },
 621
 622     "samba-ad-dc-5": {
 623         "dependency": "samba-def-build",
 624         "sequence": [
 625             ("random-sleep", random_sleep(1, 1)),
 626             ("test", make_test(include_envs=[
 627             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-6": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-schemaupgrade": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 654     # This is currently the longest task, so we don't randomly delay it.
 655     "samba-ad-dc-ntvfs": {
 656         "dependency": "samba-def-build",
 657         "sequence": [
 658             ("random-sleep", random_sleep(1, 1)),
 659             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 660             ("lcov", LCOV_CMD),
 661             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 662         ],
 663     },
 664
 665     # Test fips compliance
 666     "samba-fips": {
 667         "dependency": "samba-mit-build",
 668         "sequence": [
 669             ("random-sleep", random_sleep(1, 1)),
 670             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 671             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 672             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 673         ],
 674     },
 675
 676     # run the backup/restore testenvs separately as they're fairly standalone
 677     # (and CI seems to max out at ~3 different DCs running at once)
 678     "samba-ad-back1": {
 679         "dependency": "samba-def-build",
 680         "sequence": [
 681             ("random-sleep", random_sleep(300, 900)),
 682             ("test", make_test(include_envs=[
 683             "backupfromdc",
 684             "restoredc",
 685             "renamedc",
 686             ])),
 687             ("lcov", LCOV_CMD),
 688             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 689         ],
 690     },
 691     "samba-ad-back2": {
 692         "dependency": "samba-def-build",
 693         "sequence": [
 694             ("random-sleep", random_sleep(300, 900)),
 695             ("test", make_test(include_envs=[
 696             "backupfromdc",
 697             "offlinebackupdc",
 698             "labdc",
 699             ])),
 700             ("lcov", LCOV_CMD),
 701             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 702         ],
 703     },
 704
 705     "samba-admem-mit": {
 706         "dependency": "samba-mit-build",
 707         "sequence": [
 708             ("random-sleep", random_sleep(1, 1)),
 709             ("test", make_test(include_envs=[
 710             "ad_member",
 711             "ad_member_idmap_rid",
 712             "admem_idmap_autorid",
 713             "ad_member_idmap_ad",
 714             "ad_member_rfc2307",
 715             "ad_member_offlogon",
 716             ])),
 717             ("lcov", LCOV_CMD),
 718             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 719         ],
 720     },
 721
 722     "samba-addc-mit-1": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "ad_dc",
 728             "ad_dc_smb1",
 729             "ad_dc_smb1_done",
 730             "ad_dc_no_nss",
 731             "ad_dc_no_ntlm",
 732             ])),
 733             ("lcov", LCOV_CMD),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735         ],
 736     },
 737
 738     "samba-addc-mit-4a": {
 739         "dependency": "samba-mit-build",
 740         "sequence": [
 741             ("random-sleep", random_sleep(1, 1)),
 742             ("test", make_test(include_envs=[
 743             "fl2000dc",
 744             "ad_member_oneway",
 745             "fl2003dc",
 746             ])),
 747             ("lcov", LCOV_CMD),
 748             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 749         ],
 750     },
 751     "samba-addc-mit-4b": {
 752         "dependency": "samba-mit-build",
 753         "sequence": [
 754             ("random-sleep", random_sleep(1, 1)),
 755             ("test", make_test(include_envs=[
 756             "fl2008dc",
 757             "fl2008r2dc",
 758             ])),
 759             ("lcov", LCOV_CMD),
 760             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 761         ],
 762     },
 763
 764     "samba-test-only": {
 765         "sequence": [
 766             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 767             ("make", "make -j"),
 768             ("test", make_test(TESTS="${TESTS}")),
 769             ("lcov", LCOV_CMD),
 770         ],
 771     },
 772
 773     # Test cross-compile infrastructure
 774     "samba-xc": {
 775         "sequence": [
 776             ("random-sleep", random_sleep(900, 1500)),
 777             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 778             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 779             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 780             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 781             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 782             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 783             ("compare-results", "script/compare_cc_results.py "
 784             "./bin/c4che/default{} "
 785             "./bin-xe/c4che/default{} "
 786             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 787             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 788             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 789             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 790             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 791             " = \"'1234'\"".format(CACHE_SUFFIX)),
 792             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 793             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 794             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 795             " ; test $? -ne 0"),
 796         ],
 797     },
 798
 799     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 800     "samba-o3": {
 801         "sequence": [
 802             ("random-sleep", random_sleep(300, 900)),
 803             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 804             ("make", "make -j"),
 805             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 806             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 807             ("lcov", LCOV_CMD),
 808             ("install", "make install"),
 809             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 810             ("clean", "make clean"),
 811         ],
 812     },
 813
 814     "samba-ctdb": {
 815         "sequence": [
 816             ("random-sleep", random_sleep(900, 1500)),
 817
 818         # make sure we have tdb around:
 819             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 820             ("tdb-make", "cd lib/tdb && make"),
 821             ("tdb-install", "cd lib/tdb && make install"),
 822
 823         # build samba with cluster support (also building ctdb):
 824             ("samba-configure",
 825          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 826          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 827          "./configure.developer ${PREFIX} "
 828          "--with-selftest-prefix=./bin/ab "
 829          "--with-cluster-support "
 830          "--without-ad-dc "
 831          "--bundled-libraries=!tdb"),
 832             ("samba-make", "make"),
 833             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 834             ("samba-install", "make install"),
 835             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 836
 837             ("test", make_test(
 838                 cmd='make test',
 839                 INJECT_SELFTEST_PREFIX=0,
 840                 include_envs=["clusteredmember"])
 841             ),
 842
 843         # clean up:
 844             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 845             ("clean", "make clean"),
 846             ("ctdb-clean", "cd ./ctdb && make clean"),
 847         ],
 848     },
 849
 850     "samba-libs": {
 851         "sequence": [
 852             ("random-sleep", random_sleep(300, 900)),
 853             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 854             ("talloc-make", "cd lib/talloc && make"),
 855             ("talloc-install", "cd lib/talloc && make install"),
 856
 857             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 858             ("tdb-make", "cd lib/tdb && make"),
 859             ("tdb-install", "cd lib/tdb && make install"),
 860
 861             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 862             ("tevent-make", "cd lib/tevent && make"),
 863             ("tevent-install", "cd lib/tevent && make install"),
 864
 865             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 866             ("ldb-make", "cd lib/ldb && make"),
 867             ("ldb-install", "cd lib/ldb && make install"),
 868
 869             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 870             ("nondevel-make", "make -j"),
 871             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 872             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 873             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 874             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 875             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 876             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 877             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 878             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 879             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 880             ("nondevel-no-public-nss_winbind",
 881                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 882             ("nondevel-no-public-nss_wins",
 883                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 884             ("nondevel-no-public-libwbclient",
 885                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 886             ("nondevel-no-public-pam_winbind",
 887                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 888             ("nondevel-no-public-winbind_krb5_locator",
 889                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 890             ("nondevel-no-public-async_dns_krb5_locator",
 891                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 892             ("nondevel-install", "make -j install"),
 893             ("nondevel-dist", "make dist"),
 894
 895             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 896             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 897             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 898             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 899             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 900             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 901             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 902             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 903             ("prefix-no-public-nss_winbind",
 904                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 905             ("prefix-no-public-nss_wins",
 906                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 907             ("prefix-no-public-libwbclient",
 908                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 909             ("prefix-no-public-pam_winbind",
 910                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 911             ("prefix-no-public-winbind_krb5_locator",
 912                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 913                               "service_locator")),
 914             ("prefix-no-public-async_dns_krb5_locator",
 915                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 916                               "service_locator")),
 917
 918             # retry with all modules shared
 919             ("allshared-distclean", "make distclean"),
 920             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 921             ("allshared-make", "make -j"),
 922             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 923             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 924             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 925             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 926             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 927             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 928             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 929             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 930             ("allshared-no-public-nss_winbind",
 931                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 932             ("allshared-no-public-nss_wins",
 933                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 934             ("allshared-no-public-libwbclient",
 935                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 936             ("allshared-no-public-pam_winbind",
 937                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 938             ("allshared-no-public-winbind_krb5_locator",
 939                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 940             ("allshared-no-public-async_dns_krb5_locator",
 941                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 942         ],
 943     },
 944
 945     "samba-fuzz": {
 946         "sequence": [
 947         # build the fuzzers (static) via the oss-fuzz script
 948             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 949             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 950         ],
 951     },
 952
 953     # * Test smbd and smbtorture can build semi-static
 954     #
 955     # * Test Samba without python still builds.
 956     #
 957     # When this test fails due to more use of Python, the expectations
 958     # is that the newly failing part of the code should be disabled
 959     # when --disable-python is set (rather than major work being done
 960     # to support this environment).
 961     #
 962     # The target here is for vendors shipping a minimal smbd.
 963     "samba-minimal-smbd": {
 964         "sequence": [
 965             ("random-sleep", random_sleep(300, 900)),
 966
 967         # build with all modules static
 968             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 969             ("allstatic-make", "make -j"),
 970             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 971             ("allstatic-lcov", LCOV_CMD),
 972
 973         # retry with nonshared smbd and smbtorture
 974             ("nonshared-distclean", "make distclean"),
 975             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 976             ("nonshared-make", "make -j"),
 977             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 978             # TODO ("nonshared-lcov", LCOV_CMD),
 979
 980             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 981             ("clean", "make clean"),
 982         ],
 983     },
 984
 985     "samba-nopython": {
 986         "sequence": [
 987             ("random-sleep", random_sleep(300, 900)),
 988
 989             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 990             ("make", "make -j"),
 991             ("find-python", "script/find_python.sh ${PREFIX}"),
 992             ("test", "make test-nopython"),
 993             ("lcov", LCOV_CMD),
 994             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 995             ("clean", "make clean"),
 996
 997             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 998             ("talloc-make", "cd lib/talloc && make"),
 999             ("talloc-install", "cd lib/talloc && make install"),
1000
1001             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1002             ("tdb-make", "cd lib/tdb && make"),
1003             ("tdb-install", "cd lib/tdb && make install"),
1004
1005             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1006             ("tevent-make", "cd lib/tevent && make"),
1007             ("tevent-install", "cd lib/tevent && make install"),
1008
1009             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1010             ("ldb-make", "cd lib/ldb && make"),
1011             ("ldb-install", "cd lib/ldb && make install"),
1012
1013         # retry against installed library packages, but no required modules
1014             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1015             ("libs-make", "make -j"),
1016             ("libs-install", "make install"),
1017             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1018             ("libs-clean", "make clean"),
1019
1020         ],
1021     },
1022
1023     "samba-shellcheck": {
1024         "sequence": [
1025             ("run", "script/check-shell-scripts.sh ."),
1026         ],
1027     },
1028
1029     "ldb": {
1030         "sequence": [
1031             ("random-sleep", random_sleep(60, 600)),
1032             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1033             ("make", "make"),
1034             ("install", "make install"),
1035             ("test", "make test"),
1036             ("lcov", LCOV_CMD),
1037             ("clean", "make clean"),
1038             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1039             ("make-no-lmdb", "make"),
1040             ("test-no-lmdb", "make test"),
1041             ("lcov-no-lmdb", LCOV_CMD),
1042             ("install-no-lmdb", "make install"),
1043             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1044             ("distcheck", "make distcheck"),
1045             ("clean", "make clean"),
1046         ],
1047     },
1048
1049     "tdb": {
1050         "sequence": [
1051             ("random-sleep", random_sleep(60, 600)),
1052             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1053             ("make", "make"),
1054             ("install", "make install"),
1055             ("test", "make test"),
1056             ("lcov", LCOV_CMD),
1057             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1058             ("distcheck", "make distcheck"),
1059             ("clean", "make clean"),
1060         ],
1061     },
1062
1063     "talloc": {
1064         "sequence": [
1065             ("random-sleep", random_sleep(60, 600)),
1066             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1067             ("make", "make"),
1068             ("install", "make install"),
1069             ("test", "make test"),
1070             ("lcov", LCOV_CMD),
1071             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1072             ("distcheck", "make distcheck"),
1073             ("clean", "make clean"),
1074         ],
1075     },
1076
1077     "replace": {
1078         "sequence": [
1079             ("random-sleep", random_sleep(60, 600)),
1080             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1081             ("make", "make"),
1082             ("install", "make install"),
1083             ("test", "make test"),
1084             ("lcov", LCOV_CMD),
1085             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1086             ("distcheck", "make distcheck"),
1087             ("clean", "make clean"),
1088         ],
1089     },
1090
1091     "tevent": {
1092         "sequence": [
1093             ("random-sleep", random_sleep(60, 600)),
1094             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1095             ("make", "make"),
1096             ("install", "make install"),
1097             ("test", "make test"),
1098             ("lcov", LCOV_CMD),
1099             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1100             ("distcheck", "make distcheck"),
1101             ("clean", "make clean"),
1102         ],
1103     },
1104
1105     "pidl": {
1106         "git-clone-required": True,
1107         "sequence": [
1108             ("random-sleep", random_sleep(60, 600)),
1109             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1110             ("touch", "touch *.yp"),
1111             ("make", "make"),
1112             ("test", "make test"),
1113             ("install", "make install"),
1114             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1115             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1116             ("clean", "make clean"),
1117         ],
1118     },
1119
1120     # these are useful for debugging autobuild
1121     "pass": {
1122         "sequence": [
1123             ("pass", 'echo passing && /bin/true'),
1124         ],
1125     },
1126     "fail": {
1127         "sequence": [
1128             ("fail", 'echo failing && /bin/false'),
1129         ],
1130     },
1131 }
1132
1133 defaulttasks = list(tasks.keys())
1134
1135 defaulttasks.remove("pass")
1136 defaulttasks.remove("fail")
1137
1138 # The build tasks will be brought in by the test tasks as needed
1139 defaulttasks.remove("samba-def-build")
1140 defaulttasks.remove("samba-nt4-build")
1141 defaulttasks.remove("samba-mit-build")
1142 defaulttasks.remove("samba-h5l-build")
1143 defaulttasks.remove("samba-no-opath-build")
1144
1145 # This is not a normal test, but a task to support manually running
1146 # one test under autobuild
1147 defaulttasks.remove("samba-test-only")
1148
1149 # Only built on GitLab CI and not in the default autobuild because it
1150 # uses too much space (4GB of semi-static binaries)
1151 defaulttasks.remove("samba-fuzz")
1152
1153 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1154 # container where a simulated FIPS mode is possible.
1155 defaulttasks.remove("samba-fips")
1156
1157 # The MIT build runs on a current Fedora where an up to date MIT KDC
1158 # is already packaged.  This avoids needing to backport a current MIT
1159 # to the default Ubuntu 18.04, particularly during development, and
1160 # the need to install on the shared sn-devel-184.
1161
1162 defaulttasks.remove("samba-mitkrb5")
1163 defaulttasks.remove("samba-admem-mit")
1164 defaulttasks.remove("samba-addc-mit-1")
1165 defaulttasks.remove("samba-addc-mit-4a")
1166 defaulttasks.remove("samba-addc-mit-4b")
1167
1168 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1169     defaulttasks.remove("samba-o3")
1170
1171
1172 def do_print(msg):
1173     print("%s" % msg)
1174     sys.stdout.flush()
1175     sys.stderr.flush()
1176
1177
1178 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1179     if show is None:
1180         show = options.verbose
1181     if show:
1182         do_print("Running: '%s' in '%s'" % (cmd, dir))
1183     if output:
1184         out = check_output([cmd], shell=True, cwd=dir)
1185         return out.decode(encoding='utf-8', errors='backslashreplace')
1186     elif checkfail:
1187         return check_call(cmd, shell=True, cwd=dir)
1188     else:
1189         return call(cmd, shell=True, cwd=dir)
1190
1191 def rmdir_force(dirname, re_raise=True):
1192     try:
1193         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1194                 dirname, dirname, dirname), output=True, show=True)
1195     except CalledProcessError as e:
1196         do_print("Failed: '%s'" % (str(e)))
1197         run_cmd("tree %s" % dirname, output=True, show=True)
1198         if re_raise:
1199             raise
1200         return False
1201     return True
1202
1203 class builder(object):
1204     '''handle build of one directory'''
1205
1206     def __init__(self, name, definition):
1207         self.name = name
1208         self.dir = builddirs.get(name, '.')
1209         self.tag = self.name.replace('/', '_')
1210         self.definition = definition
1211         self.sequence = definition["sequence"]
1212         self.git_clone_required = False
1213         if "git-clone-required" in definition:
1214             self.git_clone_required = bool(definition["git-clone-required"])
1215         self.proc = None
1216         self.done = False
1217         self.next = 0
1218         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1219         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1220         if options.verbose:
1221             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1222             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1223         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1224         self.stdout = open(self.stdout_path, 'w')
1225         self.stderr = open(self.stderr_path, 'w')
1226         self.stdin  = open("/dev/null", 'r')
1227         self.builder_dir = "%s/%s" % (testbase, self.tag)
1228         self.test_source_dir = self.builder_dir
1229         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1230         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1231         self.prefix = "%s/%s" % (test_prefix, self.tag)
1232         self.consumers = []
1233         self.producer = None
1234
1235         if self.git_clone_required:
1236             assert "dependency" not in definition
1237
1238     def mark_existing(self):
1239         do_print('%s: Mark as existing dependency' % self.name)
1240         self.next = len(self.sequence)
1241         self.done = True
1242
1243     def add_consumer(self, consumer):
1244         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1245         consumer.producer = self
1246         consumer.test_source_dir = self.test_source_dir
1247         self.consumers.append(consumer)
1248
1249     def start_next(self):
1250         if self.producer is not None:
1251             if not self.producer.done:
1252                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1253                 return
1254
1255         if self.next == 0:
1256             rmdir_force(self.builder_dir)
1257             rmdir_force(self.prefix)
1258             if self.producer is not None:
1259                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1260             elif not self.git_clone_required:
1261                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1262             else:
1263                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1264
1265         if self.next == len(self.sequence):
1266             if not self.done:
1267                 do_print('%s: Completed OK' % self.name)
1268                 self.done = True
1269             if not options.nocleanup and len(self.consumers) == 0:
1270                 do_print('%s: Cleaning up' % self.name)
1271                 rmdir_force(self.builder_dir)
1272                 rmdir_force(self.prefix)
1273             for consumer in self.consumers:
1274                 if consumer.next != 0:
1275                     continue
1276                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1277                 consumer.start_next()
1278             if self.producer is not None:
1279                 self.producer.consumers.remove(self)
1280                 assert self.producer.done
1281                 self.producer.start_next()
1282             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1283             return
1284         (self.stage, self.cmd) = self.sequence[self.next]
1285         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1286         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1287         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1288         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1289         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1290         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1291         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1292         self.cmd = self.cmd.replace("${NAME}", self.name)
1293         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1294         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1295         self.proc = Popen(self.cmd, shell=True,
1296                           close_fds=True, cwd=self.cwd,
1297                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1298         self.next += 1
1299
1300 def expand_dependencies(n):
1301     deps = list()
1302     if "dependency" in tasks[n]:
1303         depname = tasks[n]["dependency"]
1304         assert depname in tasks
1305         sdeps = expand_dependencies(depname)
1306         assert n not in sdeps
1307         for sdep in sdeps:
1308             deps.append(sdep)
1309         deps.append(depname)
1310     return deps
1311
1312
1313 class buildlist(object):
1314     '''handle build of multiple directories'''
1315
1316     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1317         self.tail_proc = None
1318         self.retry = None
1319         if not tasknames:
1320             if options.restrict_tests:
1321                 tasknames = ["samba-test-only"]
1322             else:
1323                 tasknames = defaulttasks
1324
1325         given_tasknames = tasknames.copy()
1326         implicit_tasknames = []
1327         for n in given_tasknames:
1328             deps = expand_dependencies(n)
1329             for dep in deps:
1330                 if dep in given_tasknames:
1331                     continue
1332                 if dep in implicit_tasknames:
1333                     continue
1334                 implicit_tasknames.append(dep)
1335
1336         tasknames = implicit_tasknames.copy()
1337         tasknames.extend(given_tasknames)
1338         do_print("given_tasknames: %s" % given_tasknames)
1339         do_print("implicit_tasknames: %s" % implicit_tasknames)
1340         do_print("tasknames: %s" % tasknames)
1341         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1342
1343         if options.retry:
1344             rebase_remote = "rebaseon"
1345             retry_task = {
1346                     "git-clone-required": True,
1347                     "sequence": [
1348                             ("retry",
1349                             '''set -e
1350                             git remote add -t %s %s %s
1351                             git fetch %s
1352                             while :; do
1353                               sleep 60
1354                               git describe %s/%s > old_remote_branch.desc
1355                               git fetch %s
1356                               git describe %s/%s > remote_branch.desc
1357                               diff old_remote_branch.desc remote_branch.desc
1358                             done
1359                            ''' % (
1360                                rebase_branch, rebase_remote, rebase_url,
1361                                rebase_remote,
1362                                rebase_remote, rebase_branch,
1363                                rebase_remote,
1364                                rebase_remote, rebase_branch
1365                             ))]}
1366
1367             self.retry = builder('retry', retry_task)
1368             self.need_retry = False
1369
1370         if options.skip_dependencies:
1371             for b in self.tlist:
1372                 if b.name in implicit_tasknames:
1373                     b.mark_existing()
1374
1375         for b in self.tlist:
1376             do_print("b.name=%s" % b.name)
1377             if "dependency" not in b.definition:
1378                 continue
1379             depname = b.definition["dependency"]
1380             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1381             for p in self.tlist:
1382                 if p.name == depname:
1383                     p.add_consumer(b)
1384
1385     def kill_kids(self):
1386         if self.tail_proc is not None:
1387             self.tail_proc.terminate()
1388             self.tail_proc.wait()
1389             self.tail_proc = None
1390         if self.retry is not None:
1391             self.retry.proc.terminate()
1392             self.retry.proc.wait()
1393             self.retry = None
1394         for b in self.tlist:
1395             if b.proc is not None:
1396                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1397                 b.proc.terminate()
1398                 b.proc.wait()
1399                 b.proc = None
1400
1401     def wait_one(self):
1402         while True:
1403             none_running = True
1404             for b in self.tlist:
1405                 if b.proc is None:
1406                     continue
1407                 none_running = False
1408                 b.status = b.proc.poll()
1409                 if b.status is None:
1410                     continue
1411                 b.proc = None
1412                 return b
1413             if options.retry:
1414                 ret = self.retry.proc.poll()
1415                 if ret is not None:
1416                     self.need_retry = True
1417                     self.retry = None
1418                     return None
1419             if none_running:
1420                 return None
1421             time.sleep(0.1)
1422
1423     def run(self):
1424         for b in self.tlist:
1425             b.start_next()
1426         if options.retry:
1427             self.retry.start_next()
1428         while True:
1429             b = self.wait_one()
1430             if options.retry and self.need_retry:
1431                 self.kill_kids()
1432                 do_print("retry needed")
1433                 return (0, None, None, None, "retry")
1434             if b is None:
1435                 break
1436             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1437                 self.kill_kids()
1438                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1439             b.start_next()
1440         self.kill_kids()
1441         return (0, None, None, None, "All OK")
1442
1443     def write_system_info(self, filename):
1444         with open(filename, 'w') as f:
1445             for cmd in ['uname -a',
1446                         'lsb_release -a',
1447                         'free',
1448                         'mount',
1449                         'cat /proc/cpuinfo',
1450                         'cc --version',
1451                         'df -m .',
1452                         'df -m %s' % testbase]:
1453                 try:
1454                     out = run_cmd(cmd, output=True, checkfail=False)
1455                 except CalledProcessError as e:
1456                     out = "<failed: %s>" % str(e)
1457                 print('### %s' % cmd, file=f)
1458                 print(out, file=f)
1459                 print(file=f)
1460
1461     def tarlogs(self, fname):
1462         with tarfile.open(fname, "w:gz") as tar:
1463             for b in self.tlist:
1464                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1465                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1466             if os.path.exists("autobuild.log"):
1467                 tar.add("autobuild.log")
1468             filename = 'system-info.txt'
1469             self.write_system_info(filename)
1470             tar.add(filename)
1471
1472     def remove_logs(self):
1473         for b in self.tlist:
1474             os.unlink(b.stdout_path)
1475             os.unlink(b.stderr_path)
1476
1477     def start_tail(self):
1478         cmd = ["tail", "-f"]
1479         for b in self.tlist:
1480             cmd.append(b.stdout_path)
1481             cmd.append(b.stderr_path)
1482         self.tail_proc = Popen(cmd, close_fds=True)
1483
1484
1485 def cleanup(do_raise=False):
1486     if options.nocleanup:
1487         return
1488     run_cmd("stat %s || true" % test_tmpdir, show=True)
1489     run_cmd("stat %s" % testbase, show=True)
1490     do_print("Cleaning up %r" % cleanup_list)
1491     for d in cleanup_list:
1492         ok = rmdir_force(d, re_raise=False)
1493         if ok:
1494             continue
1495         if os.path.isdir(d):
1496             do_print("Killing, waiting and retry")
1497             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1498         else:
1499             do_print("Waiting and retry")
1500         time.sleep(1)
1501         rmdir_force(d, re_raise=do_raise)
1502
1503
1504 def daemonize(logfile):
1505     pid = os.fork()
1506     if pid == 0:  # Parent
1507         os.setsid()
1508         pid = os.fork()
1509         if pid != 0:  # Actual daemon
1510             os._exit(0)
1511     else:  # Grandparent
1512         os._exit(0)
1513
1514     import resource      # Resource usage information.
1515     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1516     if maxfd == resource.RLIM_INFINITY:
1517         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1518     for fd in range(0, maxfd):
1519         try:
1520             os.close(fd)
1521         except OSError:
1522             pass
1523     os.open(logfile, os.O_RDWR | os.O_CREAT)
1524     os.dup2(0, 1)
1525     os.dup2(0, 2)
1526
1527
1528 def write_pidfile(fname):
1529     '''write a pid file, cleanup on exit'''
1530     with open(fname, mode='w') as f:
1531         f.write("%u\n" % os.getpid())
1532
1533
1534 def rebase_tree(rebase_url, rebase_branch="master"):
1535     rebase_remote = "rebaseon"
1536     do_print("Rebasing on %s" % rebase_url)
1537     run_cmd("git describe HEAD", show=True, dir=test_master)
1538     run_cmd("git remote add -t %s %s %s" %
1539             (rebase_branch, rebase_remote, rebase_url),
1540             show=True, dir=test_master)
1541     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1542     if options.fix_whitespace:
1543         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1544                 (rebase_remote, rebase_branch),
1545                 show=True, dir=test_master)
1546     else:
1547         run_cmd("git rebase --force-rebase %s/%s" %
1548                 (rebase_remote, rebase_branch),
1549                 show=True, dir=test_master)
1550     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1551                    (rebase_remote, rebase_branch),
1552                    dir=test_master, output=True)
1553     if diff == '':
1554         do_print("No differences between HEAD and %s/%s - exiting" %
1555                  (rebase_remote, rebase_branch))
1556         sys.exit(0)
1557     run_cmd("git describe %s/%s" %
1558             (rebase_remote, rebase_branch),
1559             show=True, dir=test_master)
1560     run_cmd("git describe HEAD", show=True, dir=test_master)
1561     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1562             (rebase_remote, rebase_branch),
1563             show=True, dir=test_master)
1564
1565
1566 def push_to(push_url, push_branch="master"):
1567     push_remote = "pushto"
1568     do_print("Pushing to %s" % push_url)
1569     if options.mark:
1570         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1571         run_cmd("git commit --amend -c HEAD", dir=test_master)
1572         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1573         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1574     run_cmd("git remote add -t %s %s %s" %
1575             (push_branch, push_remote, push_url),
1576             show=True, dir=test_master)
1577     run_cmd("git push %s +HEAD:%s" %
1578             (push_remote, push_branch),
1579             show=True, dir=test_master)
1580
1581
1582 def send_email(subject, text, log_tar):
1583     if options.email is None:
1584         do_print("not sending email because the recipient is not set")
1585         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1586                  (subject, text))
1587         return
1588     outer = MIMEMultipart()
1589     outer['Subject'] = subject
1590     outer['To'] = options.email
1591     outer['From'] = options.email_from
1592     outer['Date'] = email.utils.formatdate(localtime=True)
1593     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1594     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1595     if options.attach_logs:
1596         with open(log_tar, 'rb') as fp:
1597             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1598         # Set the filename parameter
1599         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1600         outer.attach(msg)
1601     content = outer.as_string()
1602     s = smtplib.SMTP(options.email_server)
1603     email_user = os.getenv('SMTP_USERNAME')
1604     email_password = os.getenv('SMTP_PASSWORD')
1605     if email_user is not None:
1606         s.starttls()
1607         s.login(email_user, email_password)
1608
1609     s.sendmail(options.email_from, [options.email], content)
1610     s.set_debuglevel(1)
1611     s.quit()
1612
1613
1614 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1615                   elapsed_time, log_base=None, add_log_tail=True):
1616     '''send an email to options.email about the failure'''
1617     elapsed_minutes = elapsed_time / 60.0
1618     if log_base is None:
1619         log_base = gitroot
1620     text = '''
1621 Dear Developer,
1622
1623 Your autobuild on %s failed after %.1f minutes
1624 when trying to test %s with the following error:
1625
1626    %s
1627
1628 the autobuild has been abandoned. Please fix the error and resubmit.
1629
1630 A summary of the autobuild process is here:
1631
1632   %s/autobuild.log
1633 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1634
1635     if options.restrict_tests:
1636         text += """
1637 The build was restricted to tests matching %s\n""" % options.restrict_tests
1638
1639     if failed_task != 'rebase':
1640         text += '''
1641 You can see logs of the failed task here:
1642
1643   %s/%s.stdout
1644   %s/%s.stderr
1645
1646 or you can get full logs of all tasks in this job here:
1647
1648   %s/logs.tar.gz
1649
1650 The top commit for the tree that was built was:
1651
1652 %s
1653
1654 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1655
1656     if add_log_tail:
1657         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1658         lines = f.readlines()
1659         log_tail = "".join(lines[-50:])
1660         num_lines = len(lines)
1661         if num_lines < 50:
1662             # Also include stderr (compile failures) if < 50 lines of stdout
1663             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1664             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1665
1666         text += '''
1667 The last 50 lines of log messages:
1668
1669 %s
1670     ''' % log_tail
1671         f.close()
1672
1673     logs = os.path.join(gitroot, 'logs.tar.gz')
1674     send_email('autobuild[%s] failure on %s for task %s during %s'
1675                % (options.branch, platform.node(), failed_task, failed_stage),
1676                text, logs)
1677
1678
1679 def email_success(elapsed_time, log_base=None):
1680     '''send an email to options.email about a successful build'''
1681     if log_base is None:
1682         log_base = gitroot
1683     text = '''
1684 Dear Developer,
1685
1686 Your autobuild on %s has succeeded after %.1f minutes.
1687
1688 ''' % (platform.node(), elapsed_time / 60.)
1689
1690     if options.restrict_tests:
1691         text += """
1692 The build was restricted to tests matching %s\n""" % options.restrict_tests
1693
1694     if options.keeplogs:
1695         text += '''
1696
1697 you can get full logs of all tasks in this job here:
1698
1699   %s/logs.tar.gz
1700
1701 ''' % log_base
1702
1703     text += '''
1704 The top commit for the tree that was built was:
1705
1706 %s
1707 ''' % top_commit_msg
1708
1709     logs = os.path.join(gitroot, 'logs.tar.gz')
1710     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1711                text, logs)
1712
1713
1714 # get the top commit message, for emails
1715 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1716
1717 try:
1718     if options.skip_dependencies:
1719         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1720     else:
1721         os.makedirs(testbase)
1722 except Exception as reason:
1723     raise Exception("Unable to create %s : %s" % (testbase, reason))
1724 cleanup_list.append(testbase)
1725
1726 if options.daemon:
1727     logfile = os.path.join(testbase, "log")
1728     do_print("Forking into the background, writing progress to %s" % logfile)
1729     daemonize(logfile)
1730
1731 write_pidfile(gitroot + "/autobuild.pid")
1732
1733 start_time = time.time()
1734
1735 while True:
1736     try:
1737         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1738         os.makedirs(test_tmpdir)
1739         # The waf uninstall code removes empty directories all the way
1740         # up the tree.  Creating a file in test_tmpdir stops it from
1741         # being removed.
1742         run_cmd("touch %s" % os.path.join(test_tmpdir,
1743                                           ".directory-is-not-empty"), show=True)
1744         run_cmd("stat %s" % test_tmpdir, show=True)
1745         run_cmd("stat %s" % testbase, show=True)
1746         if options.skip_dependencies:
1747             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1748         else:
1749             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1750     except Exception:
1751         cleanup()
1752         raise
1753
1754     try:
1755         if options.rebase is not None:
1756             rebase_tree(options.rebase, rebase_branch=options.branch)
1757     except Exception:
1758         cleanup_list.append(gitroot + "/autobuild.pid")
1759         cleanup()
1760         elapsed_time = time.time() - start_time
1761         email_failure(-1, 'rebase', 'rebase', 'rebase',
1762                       'rebase on %s failed' % options.branch,
1763                       elapsed_time, log_base=options.log_base)
1764         sys.exit(1)
1765
1766     try:
1767         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1768         if options.tail:
1769             blist.start_tail()
1770         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1771         if status != 0 or errstr != "retry":
1772             break
1773         cleanup(do_raise=True)
1774     except Exception:
1775         cleanup()
1776         raise
1777
1778 cleanup_list.append(gitroot + "/autobuild.pid")
1779
1780 do_print(errstr)
1781
1782 blist.kill_kids()
1783 if options.tail:
1784     do_print("waiting for tail to flush")
1785     time.sleep(1)
1786
1787 elapsed_time = time.time() - start_time
1788 if status == 0:
1789     if options.passcmd is not None:
1790         do_print("Running passcmd: %s" % options.passcmd)
1791         run_cmd(options.passcmd, dir=test_master)
1792     if options.pushto is not None:
1793         push_to(options.pushto, push_branch=options.branch)
1794     if options.keeplogs or options.attach_logs:
1795         blist.tarlogs("logs.tar.gz")
1796         do_print("Logs in logs.tar.gz")
1797     if options.always_email:
1798         email_success(elapsed_time, log_base=options.log_base)
1799     blist.remove_logs()
1800     cleanup()
1801     do_print(errstr)
1802     sys.exit(0)
1803
1804 # something failed, gather a tar of the logs
1805 blist.tarlogs("logs.tar.gz")
1806
1807 if options.email is not None:
1808     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1809                   elapsed_time, log_base=options.log_base)
1810 else:
1811     elapsed_minutes = elapsed_time / 60.0
1812     print('''
1813
1814 ####################################################################
1815
1816 AUTOBUILD FAILURE
1817
1818 Your autobuild[%s] on %s failed after %.1f minutes
1819 when trying to test %s with the following error:
1820
1821    %s
1822
1823 the autobuild has been abandoned. Please fix the error and resubmit.
1824
1825 ####################################################################
1826
1827 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1828
1829 cleanup()
1830 do_print(errstr)
1831 do_print("Logs in logs.tar.gz")
1832 sys.exit(status)