script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.exists(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if options.enable_coverage:
 117     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 118 else:
 119     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 120
 121 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 122
 123
 124 def check_symbols(sofile, expected_symbols=""):
 125     return "objdump --dynamic-syms " + sofile + " | " + \
 126            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 127
 128 if args:
 129     # If we are only running specific test,
 130     # do not sleep randomly to wait for it to start
 131     def random_sleep(low, high):
 132         return 'sleep 1'
 133 else:
 134     def random_sleep(low, high):
 135         return 'sleep {}'.format(random.randint(low, high))
 136
 137 cleanup_list = []
 138
 139 builddirs = {
 140     "ctdb": "ctdb",
 141     "ldb": "lib/ldb",
 142     "tdb": "lib/tdb",
 143     "talloc": "lib/talloc",
 144     "replace": "lib/replace",
 145     "tevent": "lib/tevent",
 146     "pidl": "pidl",
 147     "docs-xml": "docs-xml"
 148 }
 149
 150 ctdb_configure_params = " --enable-developer ${PREFIX}"
 151 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 152
 153 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 154 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 155 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 156 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 157 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 158 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 159 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 160
 161
 162 def format_option(name, value=None):
 163     """Format option as str list."""
 164     if value is None:  # boolean option
 165         return [name]
 166     if not isinstance(value, list):  # single value option
 167         value = [value]
 168     # repeatable option
 169     return ['{}={}'.format(name, item) for item in value]
 170
 171
 172 def make_test(
 173         cmd='make testonly',
 174         INJECT_SELFTEST_PREFIX=1,
 175         TESTS='',
 176         include_envs=None,
 177         exclude_envs=None):
 178
 179     test_options = []
 180     if include_envs:
 181         test_options = format_option('--include-env', include_envs)
 182     if exclude_envs:
 183         test_options = format_option('--exclude-env', exclude_envs)
 184     if test_options:
 185         # join envs options to original test options
 186         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 187
 188     _options = []
 189
 190     # Allow getting a full CI with
 191     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 192
 193     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 194
 195     if int(FAIL_IMMEDIATELY):
 196         _options.append('FAIL_IMMEDIATELY=1')
 197     if TESTS:
 198         _options.append("TESTS='{}'".format(TESTS))
 199
 200     if INJECT_SELFTEST_PREFIX:
 201         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 202         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 203
 204     return ' '.join([cmd] + _options)
 205
 206
 207 # When updating this list, also update .gitlab-ci.yml to add the job
 208 # and to make it a dependency of 'page' for the coverage report.
 209
 210 tasks = {
 211     "ctdb": {
 212         "sequence": [
 213             ("random-sleep", random_sleep(300, 900)),
 214             ("configure", "./configure " + ctdb_configure_params),
 215             ("make", "make all"),
 216             ("install", "make install"),
 217             ("test", "make autotest"),
 218             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 219             ("clean", "make clean"),
 220         ],
 221     },
 222     "docs-xml": {
 223         "sequence": [
 224             ("random-sleep", random_sleep(300, 900)),
 225             ("autoconf", "autoconf"),
 226             ("configure", "./configure"),
 227             ("make", "make html htmlman"),
 228             ("publish-docs", PUBLISH_DOCS),
 229             ("clean", "make clean"),
 230         ],
 231     },
 232
 233     "samba-def-build": {
 234         "git-clone-required": True,
 235         "sequence": [
 236             ("configure", "./configure.developer" + samba_configure_params),
 237             ("make", "make -j"),
 238             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 239             ("chmod-R-a-w", "chmod -R a-w ."),
 240         ],
 241     },
 242
 243     "samba-mit-build": {
 244         "git-clone-required": True,
 245         "sequence": [
 246             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 247             ("make", "make -j"),
 248             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 249             ("chmod-R-a-w", "chmod -R a-w ."),
 250         ],
 251     },
 252
 253     "samba-nt4-build": {
 254         "git-clone-required": True,
 255         "sequence": [
 256             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 257             ("make", "make -j"),
 258             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 259             ("chmod-R-a-w", "chmod -R a-w ."),
 260         ],
 261     },
 262
 263     "samba-h5l-build": {
 264         "git-clone-required": True,
 265         "sequence": [
 266             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 267             ("make", "make -j"),
 268             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 269             ("chmod-R-a-w", "chmod -R a-w ."),
 270         ],
 271     },
 272
 273     "samba-no-opath-build": {
 274         "git-clone-required": True,
 275         "sequence": [
 276             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 277             ("make", "make -j"),
 278             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 279             ("chmod-R-a-w", "chmod -R a-w ."),
 280         ],
 281     },
 282
 283     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 284     "samba": {
 285         "sequence": [
 286             ("random-sleep", random_sleep(300, 900)),
 287             ("configure", "./configure.developer" + samba_configure_params),
 288             ("make", "make -j"),
 289             ("test", make_test(exclude_envs=[
 290             "none",
 291             "nt4_dc",
 292             "nt4_dc_smb1",
 293             "nt4_dc_smb1_done",
 294             "nt4_dc_schannel",
 295             "nt4_member",
 296             "ad_dc",
 297             "ad_dc_smb1",
 298             "ad_dc_smb1_done",
 299             "ad_dc_backup",
 300             "ad_dc_ntvfs",
 301             "ad_dc_default",
 302             "ad_dc_default_smb1",
 303             "ad_dc_slowtests",
 304             "ad_dc_no_nss",
 305             "ad_dc_no_ntlm",
 306             "fl2003dc",
 307             "fl2008dc",
 308             "fl2008r2dc",
 309             "ad_member",
 310             "ad_member_idmap_rid",
 311             "admem_idmap_autorid",
 312             "ad_member_idmap_ad",
 313             "ad_member_rfc2307",
 314             "ad_member_oneway",
 315             "chgdcpass",
 316             "vampire_2000_dc",
 317             "fl2000dc",
 318             "fileserver",
 319             "fileserver_smb1",
 320             "fileserver_smb1_done",
 321             "maptoguest",
 322             "simpleserver",
 323             "backupfromdc",
 324             "restoredc",
 325             "renamedc",
 326             "offlinebackupdc",
 327             "labdc",
 328             "preforkrestartdc",
 329             "proclimitdc",
 330             "promoted_dc",
 331             "vampire_dc",
 332             "rodc",
 333             "ad_dc_default",
 334             "ad_dc_default_smb1",
 335             "ad_dc_default_smb1_done",
 336             "ad_dc_slowtests",
 337             "schema_pair_dc",
 338             "schema_dc",
 339             "clusteredmember",
 340             "ad_dc_fips",
 341             "ad_member_fips",
 342             ])),
 343             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 344             ("lcov", LCOV_CMD),
 345             ("install", "make install"),
 346             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 347             ("clean", "make clean"),
 348         ],
 349     },
 350
 351     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 352     "samba-mitkrb5": {
 353         "sequence": [
 354             ("random-sleep", random_sleep(300, 900)),
 355             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 356             ("make", "make -j"),
 357             ("test", make_test(exclude_envs=[
 358             "none",
 359             "nt4_dc",
 360             "nt4_dc_smb1",
 361             "nt4_dc_smb1_done",
 362             "nt4_dc_schannel",
 363             "nt4_member",
 364             "ad_dc",
 365             "ad_dc_smb1",
 366             "ad_dc_smb1_done",
 367             "ad_dc_backup",
 368             "ad_dc_ntvfs",
 369             "ad_dc_default",
 370             "ad_dc_default_smb1",
 371             "ad_dc_default_smb1_done",
 372             "ad_dc_slowtests",
 373             "ad_dc_no_nss",
 374             "ad_dc_no_ntlm",
 375             "fl2003dc",
 376             "fl2008dc",
 377             "fl2008r2dc",
 378             "ad_member",
 379             "ad_member_idmap_rid",
 380             "admem_idmap_autorid",
 381             "ad_member_idmap_ad",
 382             "ad_member_rfc2307",
 383             "ad_member_oneway",
 384             "chgdcpass",
 385             "vampire_2000_dc",
 386             "fl2000dc",
 387             "fileserver",
 388             "fileserver_smb1",
 389             "fileserver_smb1_done",
 390             "maptoguest",
 391             "simpleserver",
 392             "backupfromdc",
 393             "restoredc",
 394             "renamedc",
 395             "offlinebackupdc",
 396             "labdc",
 397             "preforkrestartdc",
 398             "proclimitdc",
 399             "promoted_dc",
 400             "vampire_dc",
 401             "rodc",
 402             "ad_dc_default",
 403             "ad_dc_default_smb1",
 404             "ad_dc_default_smb1_done",
 405             "ad_dc_slowtests",
 406             "schema_pair_dc",
 407             "schema_dc",
 408             "clusteredmember",
 409             "ad_dc_fips",
 410             "ad_member_fips",
 411             ])),
 412             ("lcov", LCOV_CMD),
 413             ("install", "make install"),
 414             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 415             ("clean", "make clean"),
 416         ],
 417     },
 418
 419     "samba-nt4": {
 420         "dependency": "samba-nt4-build",
 421         "sequence": [
 422             ("random-sleep", random_sleep(300, 900)),
 423             ("test", make_test(include_envs=[
 424             "nt4_dc",
 425             "nt4_dc_smb1",
 426             "nt4_dc_smb1_done",
 427             "nt4_dc_schannel",
 428             "nt4_member",
 429             "simpleserver",
 430             ])),
 431             ("lcov", LCOV_CMD),
 432             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 433         ],
 434     },
 435
 436     "samba-fileserver": {
 437         "dependency": "samba-h5l-build",
 438         "sequence": [
 439             ("random-sleep", random_sleep(300, 900)),
 440             ("test", make_test(include_envs=[
 441             "fileserver",
 442             "fileserver_smb1",
 443             "fileserver_smb1_done",
 444             "maptoguest",
 445             "ktest", # ktest is also tested in samba-ktest-mit samba
 446                      # and samba-mitkrb5 but is tested here against
 447                      # a system Heimdal
 448             ])),
 449             ("lcov", LCOV_CMD),
 450             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 451         ],
 452     },
 453
 454     # This is a full build without the AD DC so we test the build with
 455     # MIT Kerberos from the current system.  Runtime behaviour is
 456     # confirmed via the ktest (static ccache and keytab) environment
 457
 458     "samba-ktest-mit": {
 459         "sequence": [
 460             ("random-sleep", random_sleep(300, 900)),
 461             ("configure", "./configure.developer --without-ad-dc --with-system-mitkrb5 " + samba_configure_params),
 462             ("make", "make -j"),
 463             ("test", make_test(include_envs=[
 464             "ktest", # ktest is also tested in fileserver, samba and
 465                      # samba-mitkrb5 but is tested here against a
 466                      # system MIT krb5
 467             ])),
 468             ("lcov", LCOV_CMD),
 469             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 470         ],
 471     },
 472
 473     "samba-admem": {
 474         "dependency": "samba-def-build",
 475         "sequence": [
 476             ("random-sleep", random_sleep(300, 900)),
 477             ("test", make_test(include_envs=[
 478             "ad_member",
 479             "ad_member_idmap_rid",
 480             "admem_idmap_autorid",
 481             "ad_member_idmap_ad",
 482             "ad_member_rfc2307",
 483             "ad_member_offlogon",
 484             ])),
 485             ("lcov", LCOV_CMD),
 486             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 487         ],
 488     },
 489
 490     "samba-no-opath1": {
 491         "dependency": "samba-no-opath-build",
 492         "sequence": [
 493             ("random-sleep", random_sleep(300, 900)),
 494             ("test", make_test(
 495                 cmd="make testonly DISABLE_OPATH=1",
 496                 include_envs=[
 497                 "nt4_dc",
 498                 "nt4_dc_smb1",
 499                 "nt4_dc_smb1_done",
 500                 "nt4_dc_schannel",
 501                 "nt4_member",
 502                 "simpleserver",
 503                 ])),
 504             ("lcov", LCOV_CMD),
 505             ("check-clean-tree", "script/clean-source-tree.sh"),
 506         ],
 507     },
 508
 509     "samba-no-opath2": {
 510         "dependency": "samba-no-opath-build",
 511         "sequence": [
 512             ("random-sleep", random_sleep(300, 900)),
 513             ("test", make_test(
 514                 cmd="make testonly DISABLE_OPATH=1",
 515                 include_envs=[
 516                 "fileserver",
 517                 "fileserver_smb1",
 518                 "fileserver_smb1_done",
 519                 ])),
 520             ("lcov", LCOV_CMD),
 521             ("check-clean-tree", "script/clean-source-tree.sh"),
 522         ],
 523     },
 524
 525     "samba-ad-dc-1": {
 526         "dependency": "samba-def-build",
 527         "sequence": [
 528             ("random-sleep", random_sleep(1, 1)),
 529             ("test", make_test(include_envs=[
 530             "ad_dc",
 531             "ad_dc_smb1",
 532             "ad_dc_smb1_done",
 533             "ad_dc_no_nss",
 534             "ad_dc_no_ntlm",
 535             ])),
 536             ("lcov", LCOV_CMD),
 537             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 538         ],
 539     },
 540
 541     "samba-ad-dc-2": {
 542         "dependency": "samba-def-build",
 543         "sequence": [
 544             ("random-sleep", random_sleep(1, 1)),
 545             ("test", make_test(include_envs=[
 546             "vampire_dc",
 547             "vampire_2000_dc",
 548             "rodc",
 549             ])),
 550             ("lcov", LCOV_CMD),
 551             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 552         ],
 553     },
 554
 555     "samba-ad-dc-3": {
 556         "dependency": "samba-def-build",
 557         "sequence": [
 558             ("random-sleep", random_sleep(1, 1)),
 559             ("test", make_test(include_envs=[
 560             "promoted_dc",
 561             "chgdcpass",
 562             "preforkrestartdc",
 563             "proclimitdc",
 564             ])),
 565             ("lcov", LCOV_CMD),
 566             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 567         ],
 568     },
 569
 570     "samba-ad-dc-4a": {
 571         "dependency": "samba-def-build",
 572         "sequence": [
 573             ("random-sleep", random_sleep(1, 1)),
 574             ("test", make_test(include_envs=[
 575             "fl2000dc",
 576             "ad_member_oneway",
 577             "fl2003dc",
 578             ])),
 579             ("lcov", LCOV_CMD),
 580             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 581         ],
 582     },
 583     "samba-ad-dc-4b": {
 584         "dependency": "samba-def-build",
 585         "sequence": [
 586             ("random-sleep", random_sleep(1, 1)),
 587             ("test", make_test(include_envs=[
 588             "fl2008dc",
 589             "fl2008r2dc",
 590             ])),
 591             ("lcov", LCOV_CMD),
 592             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 593         ],
 594     },
 595
 596     "samba-ad-dc-5": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=[
 601             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 602             ("lcov", LCOV_CMD),
 603             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 604         ],
 605     },
 606
 607     "samba-ad-dc-6": {
 608         "dependency": "samba-def-build",
 609         "sequence": [
 610             ("random-sleep", random_sleep(1, 1)),
 611             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 612             ("lcov", LCOV_CMD),
 613             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 614         ],
 615     },
 616
 617     "samba-schemaupgrade": {
 618         "dependency": "samba-def-build",
 619         "sequence": [
 620             ("random-sleep", random_sleep(1, 1)),
 621             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 622             ("lcov", LCOV_CMD),
 623             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 624         ],
 625     },
 626
 627     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 628     # This is currently the longest task, so we don't randomly delay it.
 629     "samba-ad-dc-ntvfs": {
 630         "dependency": "samba-def-build",
 631         "sequence": [
 632             ("random-sleep", random_sleep(1, 1)),
 633             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 634             ("lcov", LCOV_CMD),
 635             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 636         ],
 637     },
 638
 639     # Test fips compliance
 640     "samba-fips": {
 641         "dependency": "samba-mit-build",
 642         "sequence": [
 643             ("random-sleep", random_sleep(1, 1)),
 644             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 645             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 646             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 647         ],
 648     },
 649
 650     # run the backup/restore testenvs separately as they're fairly standalone
 651     # (and CI seems to max out at ~3 different DCs running at once)
 652     "samba-ad-back1": {
 653         "dependency": "samba-def-build",
 654         "sequence": [
 655             ("random-sleep", random_sleep(300, 900)),
 656             ("test", make_test(include_envs=[
 657             "backupfromdc",
 658             "restoredc",
 659             "renamedc",
 660             ])),
 661             ("lcov", LCOV_CMD),
 662             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 663         ],
 664     },
 665     "samba-ad-back2": {
 666         "dependency": "samba-def-build",
 667         "sequence": [
 668             ("random-sleep", random_sleep(300, 900)),
 669             ("test", make_test(include_envs=[
 670             "backupfromdc",
 671             "offlinebackupdc",
 672             "labdc",
 673             ])),
 674             ("lcov", LCOV_CMD),
 675             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 676         ],
 677     },
 678
 679     "samba-admem-mit": {
 680         "dependency": "samba-mit-build",
 681         "sequence": [
 682             ("random-sleep", random_sleep(1, 1)),
 683             ("test", make_test(include_envs=[
 684             "ad_member",
 685             "ad_member_idmap_rid",
 686             "admem_idmap_autorid",
 687             "ad_member_idmap_ad",
 688             "ad_member_rfc2307",
 689             "ad_member_offlogon",
 690             ])),
 691             ("lcov", LCOV_CMD),
 692             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 693         ],
 694     },
 695
 696     "samba-addc-mit-1": {
 697         "dependency": "samba-mit-build",
 698         "sequence": [
 699             ("random-sleep", random_sleep(1, 1)),
 700             ("test", make_test(include_envs=[
 701             "ad_dc",
 702             "ad_dc_smb1",
 703             "ad_dc_smb1_done",
 704             "ad_dc_no_nss",
 705             "ad_dc_no_ntlm",
 706             ])),
 707             ("lcov", LCOV_CMD),
 708             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 709         ],
 710     },
 711
 712     "samba-addc-mit-4a": {
 713         "dependency": "samba-mit-build",
 714         "sequence": [
 715             ("random-sleep", random_sleep(1, 1)),
 716             ("test", make_test(include_envs=[
 717             "fl2000dc",
 718             "ad_member_oneway",
 719             "fl2003dc",
 720             ])),
 721             ("lcov", LCOV_CMD),
 722             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 723         ],
 724     },
 725     "samba-addc-mit-4b": {
 726         "dependency": "samba-mit-build",
 727         "sequence": [
 728             ("random-sleep", random_sleep(1, 1)),
 729             ("test", make_test(include_envs=[
 730             "fl2008dc",
 731             "fl2008r2dc",
 732             ])),
 733             ("lcov", LCOV_CMD),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735         ],
 736     },
 737
 738     "samba-test-only": {
 739         "sequence": [
 740             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 741             ("make", "make -j"),
 742             ("test", make_test(TESTS="${TESTS}")),
 743             ("lcov", LCOV_CMD),
 744         ],
 745     },
 746
 747     # Test cross-compile infrastructure
 748     "samba-xc": {
 749         "sequence": [
 750             ("random-sleep", random_sleep(900, 1500)),
 751             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 752             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 753             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 754             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 755             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 756             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 757             ("compare-results", "script/compare_cc_results.py "
 758             "./bin/c4che/default{} "
 759             "./bin-xe/c4che/default{} "
 760             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 761             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 762             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 763             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 764             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 765             " = \"'1234'\"".format(CACHE_SUFFIX)),
 766             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 767             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 768             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 769             " ; test $? -ne 0"),
 770         ],
 771     },
 772
 773     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 774     "samba-o3": {
 775         "sequence": [
 776             ("random-sleep", random_sleep(300, 900)),
 777             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 778             ("make", "make -j"),
 779             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 780             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 781             ("lcov", LCOV_CMD),
 782             ("install", "make install"),
 783             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 784             ("clean", "make clean"),
 785         ],
 786     },
 787
 788     "samba-ctdb": {
 789         "sequence": [
 790             ("random-sleep", random_sleep(900, 1500)),
 791
 792         # make sure we have tdb around:
 793             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 794             ("tdb-make", "cd lib/tdb && make"),
 795             ("tdb-install", "cd lib/tdb && make install"),
 796
 797         # build samba with cluster support (also building ctdb):
 798             ("samba-configure",
 799          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 800          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 801          "./configure.developer ${PREFIX} "
 802          "--with-selftest-prefix=./bin/ab "
 803          "--with-cluster-support "
 804          "--without-ad-dc "
 805          "--bundled-libraries=!tdb"),
 806             ("samba-make", "make"),
 807             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 808             ("samba-install", "make install"),
 809             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 810
 811             ("test", make_test(
 812                 cmd='make test',
 813                 INJECT_SELFTEST_PREFIX=0,
 814                 include_envs=["clusteredmember"])
 815             ),
 816
 817         # clean up:
 818             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 819             ("clean", "make clean"),
 820             ("ctdb-clean", "cd ./ctdb && make clean"),
 821         ],
 822     },
 823
 824     "samba-libs": {
 825         "sequence": [
 826             ("random-sleep", random_sleep(300, 900)),
 827             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 828             ("talloc-make", "cd lib/talloc && make"),
 829             ("talloc-install", "cd lib/talloc && make install"),
 830
 831             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 832             ("tdb-make", "cd lib/tdb && make"),
 833             ("tdb-install", "cd lib/tdb && make install"),
 834
 835             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 836             ("tevent-make", "cd lib/tevent && make"),
 837             ("tevent-install", "cd lib/tevent && make install"),
 838
 839             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 840             ("ldb-make", "cd lib/ldb && make"),
 841             ("ldb-install", "cd lib/ldb && make install"),
 842
 843             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 844             ("nondevel-make", "make -j"),
 845             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 846             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 847             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 848             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 849             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 850             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 851             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 852             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 853             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 854             ("nondevel-no-public-nss_winbind",
 855                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 856             ("nondevel-no-public-nss_wins",
 857                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 858             ("nondevel-no-public-libwbclient",
 859                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 860             ("nondevel-no-public-pam_winbind",
 861                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 862             ("nondevel-no-public-winbind_krb5_locator",
 863                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 864             ("nondevel-no-public-async_dns_krb5_locator",
 865                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 866             ("nondevel-install", "make -j install"),
 867             ("nondevel-dist", "make dist"),
 868
 869             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 870             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 871             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 872             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 873             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 874             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 875             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 876             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 877             ("prefix-no-public-nss_winbind",
 878                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 879             ("prefix-no-public-nss_wins",
 880                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 881             ("prefix-no-public-libwbclient",
 882                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 883             ("prefix-no-public-pam_winbind",
 884                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 885             ("prefix-no-public-winbind_krb5_locator",
 886                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 887                               "service_locator")),
 888             ("prefix-no-public-async_dns_krb5_locator",
 889                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 890                               "service_locator")),
 891
 892             # retry with all modules shared
 893             ("allshared-distclean", "make distclean"),
 894             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 895             ("allshared-make", "make -j"),
 896             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 897             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 898             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 899             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 900             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 901             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 902             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 903             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 904             ("allshared-no-public-nss_winbind",
 905                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 906             ("allshared-no-public-nss_wins",
 907                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 908             ("allshared-no-public-libwbclient",
 909                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 910             ("allshared-no-public-pam_winbind",
 911                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 912             ("allshared-no-public-winbind_krb5_locator",
 913                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 914             ("allshared-no-public-async_dns_krb5_locator",
 915                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 916         ],
 917     },
 918
 919     "samba-fuzz": {
 920         "sequence": [
 921         # build the fuzzers (static) via the oss-fuzz script
 922             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 923             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 924         ],
 925     },
 926
 927     # * Test smbd and smbtorture can build semi-static
 928     #
 929     # * Test Samba without python still builds.
 930     #
 931     # When this test fails due to more use of Python, the expectations
 932     # is that the newly failing part of the code should be disabled
 933     # when --disable-python is set (rather than major work being done
 934     # to support this environment).
 935     #
 936     # The target here is for vendors shipping a minimal smbd.
 937     "samba-minimal-smbd": {
 938         "sequence": [
 939             ("random-sleep", random_sleep(300, 900)),
 940
 941         # build with all modules static
 942             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 943             ("allstatic-make", "make -j"),
 944             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 945             ("allstatic-lcov", LCOV_CMD),
 946
 947         # retry with nonshared smbd and smbtorture
 948             ("nonshared-distclean", "make distclean"),
 949             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 950             ("nonshared-make", "make -j"),
 951             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 952             # TODO ("nonshared-lcov", LCOV_CMD),
 953
 954             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 955             ("clean", "make clean"),
 956         ],
 957     },
 958
 959     "samba-nopython": {
 960         "sequence": [
 961             ("random-sleep", random_sleep(300, 900)),
 962
 963             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 964             ("make", "make -j"),
 965             ("find-python", "script/find_python.sh ${PREFIX}"),
 966             ("test", "make test-nopython"),
 967             ("lcov", LCOV_CMD),
 968             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 969             ("clean", "make clean"),
 970
 971             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 972             ("talloc-make", "cd lib/talloc && make"),
 973             ("talloc-install", "cd lib/talloc && make install"),
 974
 975             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 976             ("tdb-make", "cd lib/tdb && make"),
 977             ("tdb-install", "cd lib/tdb && make install"),
 978
 979             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 980             ("tevent-make", "cd lib/tevent && make"),
 981             ("tevent-install", "cd lib/tevent && make install"),
 982
 983             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 984             ("ldb-make", "cd lib/ldb && make"),
 985             ("ldb-install", "cd lib/ldb && make install"),
 986
 987         # retry against installed library packages, but no required modules
 988             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 989             ("libs-make", "make -j"),
 990             ("libs-install", "make install"),
 991             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 992             ("libs-clean", "make clean"),
 993
 994         ],
 995     },
 996
 997     "ldb": {
 998         "sequence": [
 999             ("random-sleep", random_sleep(60, 600)),
1000             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1001             ("make", "make"),
1002             ("install", "make install"),
1003             ("test", "make test"),
1004             ("lcov", LCOV_CMD),
1005             ("clean", "make clean"),
1006             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1007             ("make-no-lmdb", "make"),
1008             ("test-no-lmdb", "make test"),
1009             ("lcov-no-lmdb", LCOV_CMD),
1010             ("install-no-lmdb", "make install"),
1011             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1012             ("distcheck", "make distcheck"),
1013             ("clean", "make clean"),
1014         ],
1015     },
1016
1017     "tdb": {
1018         "sequence": [
1019             ("random-sleep", random_sleep(60, 600)),
1020             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1021             ("make", "make"),
1022             ("install", "make install"),
1023             ("test", "make test"),
1024             ("lcov", LCOV_CMD),
1025             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1026             ("distcheck", "make distcheck"),
1027             ("clean", "make clean"),
1028         ],
1029     },
1030
1031     "talloc": {
1032         "sequence": [
1033             ("random-sleep", random_sleep(60, 600)),
1034             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1035             ("make", "make"),
1036             ("install", "make install"),
1037             ("test", "make test"),
1038             ("lcov", LCOV_CMD),
1039             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1040             ("distcheck", "make distcheck"),
1041             ("clean", "make clean"),
1042         ],
1043     },
1044
1045     "replace": {
1046         "sequence": [
1047             ("random-sleep", random_sleep(60, 600)),
1048             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1049             ("make", "make"),
1050             ("install", "make install"),
1051             ("test", "make test"),
1052             ("lcov", LCOV_CMD),
1053             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1054             ("distcheck", "make distcheck"),
1055             ("clean", "make clean"),
1056         ],
1057     },
1058
1059     "tevent": {
1060         "sequence": [
1061             ("random-sleep", random_sleep(60, 600)),
1062             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1063             ("make", "make"),
1064             ("install", "make install"),
1065             ("test", "make test"),
1066             ("lcov", LCOV_CMD),
1067             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1068             ("distcheck", "make distcheck"),
1069             ("clean", "make clean"),
1070         ],
1071     },
1072
1073     "pidl": {
1074         "git-clone-required": True,
1075         "sequence": [
1076             ("random-sleep", random_sleep(60, 600)),
1077             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1078             ("touch", "touch *.yp"),
1079             ("make", "make"),
1080             ("test", "make test"),
1081             ("install", "make install"),
1082             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1083             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1084             ("clean", "make clean"),
1085         ],
1086     },
1087
1088     # these are useful for debugging autobuild
1089     "pass": {
1090         "sequence": [
1091             ("pass", 'echo passing && /bin/true'),
1092         ],
1093     },
1094     "fail": {
1095         "sequence": [
1096             ("fail", 'echo failing && /bin/false'),
1097         ],
1098     },
1099 }
1100
1101 defaulttasks = list(tasks.keys())
1102
1103 defaulttasks.remove("pass")
1104 defaulttasks.remove("fail")
1105
1106 # The build tasks will be brought in by the test tasks as needed
1107 defaulttasks.remove("samba-def-build")
1108 defaulttasks.remove("samba-nt4-build")
1109 defaulttasks.remove("samba-mit-build")
1110 defaulttasks.remove("samba-h5l-build")
1111 defaulttasks.remove("samba-no-opath-build")
1112
1113 # This is not a normal test, but a task to support manually running
1114 # one test under autobuild
1115 defaulttasks.remove("samba-test-only")
1116
1117 # Only built on GitLab CI and not in the default autobuild because it
1118 # uses too much space (4GB of semi-static binaries)
1119 defaulttasks.remove("samba-fuzz")
1120
1121 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1122 # container where a simulated FIPS mode is possible.
1123 defaulttasks.remove("samba-fips")
1124
1125 # The MIT build runs on a current Fedora where an up to date MIT KDC
1126 # is already packaged.  This avoids needing to backport a current MIT
1127 # to the default Ubuntu 18.04, particularly during development, and
1128 # the need to install on the shared sn-devel-184.
1129
1130 defaulttasks.remove("samba-mitkrb5")
1131 defaulttasks.remove("samba-admem-mit")
1132 defaulttasks.remove("samba-addc-mit-1")
1133 defaulttasks.remove("samba-addc-mit-4a")
1134 defaulttasks.remove("samba-addc-mit-4b")
1135
1136 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1137     defaulttasks.remove("samba-o3")
1138
1139
1140 def do_print(msg):
1141     print("%s" % msg)
1142     sys.stdout.flush()
1143     sys.stderr.flush()
1144
1145
1146 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1147     if show is None:
1148         show = options.verbose
1149     if show:
1150         do_print("Running: '%s' in '%s'" % (cmd, dir))
1151     if output:
1152         out = check_output([cmd], shell=True, cwd=dir)
1153         return out.decode(encoding='utf-8', errors='backslashreplace')
1154     elif checkfail:
1155         return check_call(cmd, shell=True, cwd=dir)
1156     else:
1157         return call(cmd, shell=True, cwd=dir)
1158
1159 def rmdir_force(dirname, re_raise=True):
1160     try:
1161         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1162                 dirname, dirname, dirname), output=True, show=True)
1163     except CalledProcessError as e:
1164         do_print("Failed: '%s'" % (str(e)))
1165         run_cmd("tree %s" % dirname, output=True, show=True)
1166         if re_raise:
1167             raise
1168         return False
1169     return True
1170
1171 class builder(object):
1172     '''handle build of one directory'''
1173
1174     def __init__(self, name, definition):
1175         self.name = name
1176         self.dir = builddirs.get(name, '.')
1177         self.tag = self.name.replace('/', '_')
1178         self.definition = definition
1179         self.sequence = definition["sequence"]
1180         self.git_clone_required = False
1181         if "git-clone-required" in definition:
1182             self.git_clone_required = bool(definition["git-clone-required"])
1183         self.proc = None
1184         self.done = False
1185         self.next = 0
1186         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1187         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1188         if options.verbose:
1189             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1190             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1191         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1192         self.stdout = open(self.stdout_path, 'w')
1193         self.stderr = open(self.stderr_path, 'w')
1194         self.stdin  = open("/dev/null", 'r')
1195         self.builder_dir = "%s/%s" % (testbase, self.tag)
1196         self.test_source_dir = self.builder_dir
1197         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1198         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1199         self.prefix = "%s/%s" % (test_prefix, self.tag)
1200         self.consumers = []
1201         self.producer = None
1202
1203         if self.git_clone_required:
1204             assert "dependency" not in definition
1205
1206     def mark_existing(self):
1207         do_print('%s: Mark as existing dependency' % self.name)
1208         self.next = len(self.sequence)
1209         self.done = True
1210
1211     def add_consumer(self, consumer):
1212         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1213         consumer.producer = self
1214         consumer.test_source_dir = self.test_source_dir
1215         self.consumers.append(consumer)
1216
1217     def start_next(self):
1218         if self.producer is not None:
1219             if not self.producer.done:
1220                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1221                 return
1222
1223         if self.next == 0:
1224             rmdir_force(self.builder_dir)
1225             rmdir_force(self.prefix)
1226             if self.producer is not None:
1227                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1228             elif not self.git_clone_required:
1229                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1230             else:
1231                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1232
1233         if self.next == len(self.sequence):
1234             if not self.done:
1235                 do_print('%s: Completed OK' % self.name)
1236                 self.done = True
1237             if not options.nocleanup and len(self.consumers) == 0:
1238                 do_print('%s: Cleaning up' % self.name)
1239                 rmdir_force(self.builder_dir)
1240                 rmdir_force(self.prefix)
1241             for consumer in self.consumers:
1242                 if consumer.next != 0:
1243                     continue
1244                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1245                 consumer.start_next()
1246             if self.producer is not None:
1247                 self.producer.consumers.remove(self)
1248                 assert self.producer.done
1249                 self.producer.start_next()
1250             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1251             return
1252         (self.stage, self.cmd) = self.sequence[self.next]
1253         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1254         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1255         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1256         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1257         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1258         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1259         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1260         self.cmd = self.cmd.replace("${NAME}", self.name)
1261         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1262         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1263         self.proc = Popen(self.cmd, shell=True,
1264                           close_fds=True, cwd=self.cwd,
1265                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1266         self.next += 1
1267
1268 def expand_dependencies(n):
1269     deps = list()
1270     if "dependency" in tasks[n]:
1271         depname = tasks[n]["dependency"]
1272         assert depname in tasks
1273         sdeps = expand_dependencies(depname)
1274         assert n not in sdeps
1275         for sdep in sdeps:
1276             deps.append(sdep)
1277         deps.append(depname)
1278     return deps
1279
1280
1281 class buildlist(object):
1282     '''handle build of multiple directories'''
1283
1284     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1285         self.tail_proc = None
1286         self.retry = None
1287         if not tasknames:
1288             if options.restrict_tests:
1289                 tasknames = ["samba-test-only"]
1290             else:
1291                 tasknames = defaulttasks
1292
1293         given_tasknames = tasknames.copy()
1294         implicit_tasknames = []
1295         for n in given_tasknames:
1296             deps = expand_dependencies(n)
1297             for dep in deps:
1298                 if dep in given_tasknames:
1299                     continue
1300                 if dep in implicit_tasknames:
1301                     continue
1302                 implicit_tasknames.append(dep)
1303
1304         tasknames = implicit_tasknames.copy()
1305         tasknames.extend(given_tasknames)
1306         do_print("given_tasknames: %s" % given_tasknames)
1307         do_print("implicit_tasknames: %s" % implicit_tasknames)
1308         do_print("tasknames: %s" % tasknames)
1309         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1310
1311         if options.retry:
1312             rebase_remote = "rebaseon"
1313             retry_task = {
1314                     "git-clone-required": True,
1315                     "sequence": [
1316                             ("retry",
1317                             '''set -e
1318                             git remote add -t %s %s %s
1319                             git fetch %s
1320                             while :; do
1321                               sleep 60
1322                               git describe %s/%s > old_remote_branch.desc
1323                               git fetch %s
1324                               git describe %s/%s > remote_branch.desc
1325                               diff old_remote_branch.desc remote_branch.desc
1326                             done
1327                            ''' % (
1328                                rebase_branch, rebase_remote, rebase_url,
1329                                rebase_remote,
1330                                rebase_remote, rebase_branch,
1331                                rebase_remote,
1332                                rebase_remote, rebase_branch
1333                             ))]}
1334
1335             self.retry = builder('retry', retry_task)
1336             self.need_retry = False
1337
1338         if options.skip_dependencies:
1339             for b in self.tlist:
1340                 if b.name in implicit_tasknames:
1341                     b.mark_existing()
1342
1343         for b in self.tlist:
1344             do_print("b.name=%s" % b.name)
1345             if "dependency" not in b.definition:
1346                 continue
1347             depname = b.definition["dependency"]
1348             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1349             for p in self.tlist:
1350                 if p.name == depname:
1351                     p.add_consumer(b)
1352
1353     def kill_kids(self):
1354         if self.tail_proc is not None:
1355             self.tail_proc.terminate()
1356             self.tail_proc.wait()
1357             self.tail_proc = None
1358         if self.retry is not None:
1359             self.retry.proc.terminate()
1360             self.retry.proc.wait()
1361             self.retry = None
1362         for b in self.tlist:
1363             if b.proc is not None:
1364                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1365                 b.proc.terminate()
1366                 b.proc.wait()
1367                 b.proc = None
1368
1369     def wait_one(self):
1370         while True:
1371             none_running = True
1372             for b in self.tlist:
1373                 if b.proc is None:
1374                     continue
1375                 none_running = False
1376                 b.status = b.proc.poll()
1377                 if b.status is None:
1378                     continue
1379                 b.proc = None
1380                 return b
1381             if options.retry:
1382                 ret = self.retry.proc.poll()
1383                 if ret is not None:
1384                     self.need_retry = True
1385                     self.retry = None
1386                     return None
1387             if none_running:
1388                 return None
1389             time.sleep(0.1)
1390
1391     def run(self):
1392         for b in self.tlist:
1393             b.start_next()
1394         if options.retry:
1395             self.retry.start_next()
1396         while True:
1397             b = self.wait_one()
1398             if options.retry and self.need_retry:
1399                 self.kill_kids()
1400                 do_print("retry needed")
1401                 return (0, None, None, None, "retry")
1402             if b is None:
1403                 break
1404             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1405                 self.kill_kids()
1406                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1407             b.start_next()
1408         self.kill_kids()
1409         return (0, None, None, None, "All OK")
1410
1411     def write_system_info(self, filename):
1412         with open(filename, 'w') as f:
1413             for cmd in ['uname -a',
1414                         'lsb_release -a',
1415                         'free',
1416                         'mount',
1417                         'cat /proc/cpuinfo',
1418                         'cc --version',
1419                         'df -m .',
1420                         'df -m %s' % testbase]:
1421                 try:
1422                     out = run_cmd(cmd, output=True, checkfail=False)
1423                 except CalledProcessError as e:
1424                     out = "<failed: %s>" % str(e)
1425                 print('### %s' % cmd, file=f)
1426                 print(out, file=f)
1427                 print(file=f)
1428
1429     def tarlogs(self, fname):
1430         with tarfile.open(fname, "w:gz") as tar:
1431             for b in self.tlist:
1432                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1433                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1434             if os.path.exists("autobuild.log"):
1435                 tar.add("autobuild.log")
1436             filename = 'system-info.txt'
1437             self.write_system_info(filename)
1438             tar.add(filename)
1439
1440     def remove_logs(self):
1441         for b in self.tlist:
1442             os.unlink(b.stdout_path)
1443             os.unlink(b.stderr_path)
1444
1445     def start_tail(self):
1446         cmd = ["tail", "-f"]
1447         for b in self.tlist:
1448             cmd.append(b.stdout_path)
1449             cmd.append(b.stderr_path)
1450         self.tail_proc = Popen(cmd, close_fds=True)
1451
1452
1453 def cleanup(do_raise=False):
1454     if options.nocleanup:
1455         return
1456     run_cmd("stat %s || true" % test_tmpdir, show=True)
1457     run_cmd("stat %s" % testbase, show=True)
1458     do_print("Cleaning up %r" % cleanup_list)
1459     for d in cleanup_list:
1460         ok = rmdir_force(d, re_raise=False)
1461         if ok:
1462             continue
1463         if os.path.isdir(d):
1464             do_print("Killing, waiting and retry")
1465             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1466         else:
1467             do_print("Waiting and retry")
1468         time.sleep(1)
1469         rmdir_force(d, re_raise=do_raise)
1470
1471
1472 def daemonize(logfile):
1473     pid = os.fork()
1474     if pid == 0:  # Parent
1475         os.setsid()
1476         pid = os.fork()
1477         if pid != 0:  # Actual daemon
1478             os._exit(0)
1479     else:  # Grandparent
1480         os._exit(0)
1481
1482     import resource      # Resource usage information.
1483     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1484     if maxfd == resource.RLIM_INFINITY:
1485         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1486     for fd in range(0, maxfd):
1487         try:
1488             os.close(fd)
1489         except OSError:
1490             pass
1491     os.open(logfile, os.O_RDWR | os.O_CREAT)
1492     os.dup2(0, 1)
1493     os.dup2(0, 2)
1494
1495
1496 def write_pidfile(fname):
1497     '''write a pid file, cleanup on exit'''
1498     with open(fname, mode='w') as f:
1499         f.write("%u\n" % os.getpid())
1500
1501
1502 def rebase_tree(rebase_url, rebase_branch="master"):
1503     rebase_remote = "rebaseon"
1504     do_print("Rebasing on %s" % rebase_url)
1505     run_cmd("git describe HEAD", show=True, dir=test_master)
1506     run_cmd("git remote add -t %s %s %s" %
1507             (rebase_branch, rebase_remote, rebase_url),
1508             show=True, dir=test_master)
1509     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1510     if options.fix_whitespace:
1511         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1512                 (rebase_remote, rebase_branch),
1513                 show=True, dir=test_master)
1514     else:
1515         run_cmd("git rebase --force-rebase %s/%s" %
1516                 (rebase_remote, rebase_branch),
1517                 show=True, dir=test_master)
1518     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1519                    (rebase_remote, rebase_branch),
1520                    dir=test_master, output=True)
1521     if diff == '':
1522         do_print("No differences between HEAD and %s/%s - exiting" %
1523                  (rebase_remote, rebase_branch))
1524         sys.exit(0)
1525     run_cmd("git describe %s/%s" %
1526             (rebase_remote, rebase_branch),
1527             show=True, dir=test_master)
1528     run_cmd("git describe HEAD", show=True, dir=test_master)
1529     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1530             (rebase_remote, rebase_branch),
1531             show=True, dir=test_master)
1532
1533
1534 def push_to(push_url, push_branch="master"):
1535     push_remote = "pushto"
1536     do_print("Pushing to %s" % push_url)
1537     if options.mark:
1538         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1539         run_cmd("git commit --amend -c HEAD", dir=test_master)
1540         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1541         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1542     run_cmd("git remote add -t %s %s %s" %
1543             (push_branch, push_remote, push_url),
1544             show=True, dir=test_master)
1545     run_cmd("git push %s +HEAD:%s" %
1546             (push_remote, push_branch),
1547             show=True, dir=test_master)
1548
1549
1550 def send_email(subject, text, log_tar):
1551     if options.email is None:
1552         do_print("not sending email because the recipient is not set")
1553         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1554                  (subject, text))
1555         return
1556     outer = MIMEMultipart()
1557     outer['Subject'] = subject
1558     outer['To'] = options.email
1559     outer['From'] = options.email_from
1560     outer['Date'] = email.utils.formatdate(localtime=True)
1561     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1562     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1563     if options.attach_logs:
1564         with open(log_tar, 'rb') as fp:
1565             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1566         # Set the filename parameter
1567         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1568         outer.attach(msg)
1569     content = outer.as_string()
1570     s = smtplib.SMTP(options.email_server)
1571     email_user = os.getenv('SMTP_USERNAME')
1572     email_password = os.getenv('SMTP_PASSWORD')
1573     if email_user is not None:
1574         s.starttls()
1575         s.login(email_user, email_password)
1576
1577     s.sendmail(options.email_from, [options.email], content)
1578     s.set_debuglevel(1)
1579     s.quit()
1580
1581
1582 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1583                   elapsed_time, log_base=None, add_log_tail=True):
1584     '''send an email to options.email about the failure'''
1585     elapsed_minutes = elapsed_time / 60.0
1586     if log_base is None:
1587         log_base = gitroot
1588     text = '''
1589 Dear Developer,
1590
1591 Your autobuild on %s failed after %.1f minutes
1592 when trying to test %s with the following error:
1593
1594    %s
1595
1596 the autobuild has been abandoned. Please fix the error and resubmit.
1597
1598 A summary of the autobuild process is here:
1599
1600   %s/autobuild.log
1601 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1602
1603     if options.restrict_tests:
1604         text += """
1605 The build was restricted to tests matching %s\n""" % options.restrict_tests
1606
1607     if failed_task != 'rebase':
1608         text += '''
1609 You can see logs of the failed task here:
1610
1611   %s/%s.stdout
1612   %s/%s.stderr
1613
1614 or you can get full logs of all tasks in this job here:
1615
1616   %s/logs.tar.gz
1617
1618 The top commit for the tree that was built was:
1619
1620 %s
1621
1622 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1623
1624     if add_log_tail:
1625         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1626         lines = f.readlines()
1627         log_tail = "".join(lines[-50:])
1628         num_lines = len(lines)
1629         if num_lines < 50:
1630             # Also include stderr (compile failures) if < 50 lines of stdout
1631             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1632             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1633
1634         text += '''
1635 The last 50 lines of log messages:
1636
1637 %s
1638     ''' % log_tail
1639         f.close()
1640
1641     logs = os.path.join(gitroot, 'logs.tar.gz')
1642     send_email('autobuild[%s] failure on %s for task %s during %s'
1643                % (options.branch, platform.node(), failed_task, failed_stage),
1644                text, logs)
1645
1646
1647 def email_success(elapsed_time, log_base=None):
1648     '''send an email to options.email about a successful build'''
1649     if log_base is None:
1650         log_base = gitroot
1651     text = '''
1652 Dear Developer,
1653
1654 Your autobuild on %s has succeeded after %.1f minutes.
1655
1656 ''' % (platform.node(), elapsed_time / 60.)
1657
1658     if options.restrict_tests:
1659         text += """
1660 The build was restricted to tests matching %s\n""" % options.restrict_tests
1661
1662     if options.keeplogs:
1663         text += '''
1664
1665 you can get full logs of all tasks in this job here:
1666
1667   %s/logs.tar.gz
1668
1669 ''' % log_base
1670
1671     text += '''
1672 The top commit for the tree that was built was:
1673
1674 %s
1675 ''' % top_commit_msg
1676
1677     logs = os.path.join(gitroot, 'logs.tar.gz')
1678     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1679                text, logs)
1680
1681
1682 # get the top commit message, for emails
1683 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1684
1685 try:
1686     if options.skip_dependencies:
1687         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1688     else:
1689         os.makedirs(testbase)
1690 except Exception as reason:
1691     raise Exception("Unable to create %s : %s" % (testbase, reason))
1692 cleanup_list.append(testbase)
1693
1694 if options.daemon:
1695     logfile = os.path.join(testbase, "log")
1696     do_print("Forking into the background, writing progress to %s" % logfile)
1697     daemonize(logfile)
1698
1699 write_pidfile(gitroot + "/autobuild.pid")
1700
1701 start_time = time.time()
1702
1703 while True:
1704     try:
1705         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1706         os.makedirs(test_tmpdir)
1707         # The waf uninstall code removes empty directories all the way
1708         # up the tree.  Creating a file in test_tmpdir stops it from
1709         # being removed.
1710         run_cmd("touch %s" % os.path.join(test_tmpdir,
1711                                           ".directory-is-not-empty"), show=True)
1712         run_cmd("stat %s" % test_tmpdir, show=True)
1713         run_cmd("stat %s" % testbase, show=True)
1714         if options.skip_dependencies:
1715             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1716         else:
1717             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1718     except Exception:
1719         cleanup()
1720         raise
1721
1722     try:
1723         if options.rebase is not None:
1724             rebase_tree(options.rebase, rebase_branch=options.branch)
1725     except Exception:
1726         cleanup_list.append(gitroot + "/autobuild.pid")
1727         cleanup()
1728         elapsed_time = time.time() - start_time
1729         email_failure(-1, 'rebase', 'rebase', 'rebase',
1730                       'rebase on %s failed' % options.branch,
1731                       elapsed_time, log_base=options.log_base)
1732         sys.exit(1)
1733
1734     try:
1735         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1736         if options.tail:
1737             blist.start_tail()
1738         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1739         if status != 0 or errstr != "retry":
1740             break
1741         cleanup(do_raise=True)
1742     except Exception:
1743         cleanup()
1744         raise
1745
1746 cleanup_list.append(gitroot + "/autobuild.pid")
1747
1748 do_print(errstr)
1749
1750 blist.kill_kids()
1751 if options.tail:
1752     do_print("waiting for tail to flush")
1753     time.sleep(1)
1754
1755 elapsed_time = time.time() - start_time
1756 if status == 0:
1757     if options.passcmd is not None:
1758         do_print("Running passcmd: %s" % options.passcmd)
1759         run_cmd(options.passcmd, dir=test_master)
1760     if options.pushto is not None:
1761         push_to(options.pushto, push_branch=options.branch)
1762     if options.keeplogs or options.attach_logs:
1763         blist.tarlogs("logs.tar.gz")
1764         do_print("Logs in logs.tar.gz")
1765     if options.always_email:
1766         email_success(elapsed_time, log_base=options.log_base)
1767     blist.remove_logs()
1768     cleanup()
1769     do_print(errstr)
1770     sys.exit(0)
1771
1772 # something failed, gather a tar of the logs
1773 blist.tarlogs("logs.tar.gz")
1774
1775 if options.email is not None:
1776     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1777                   elapsed_time, log_base=options.log_base)
1778 else:
1779     elapsed_minutes = elapsed_time / 60.0
1780     print('''
1781
1782 ####################################################################
1783
1784 AUTOBUILD FAILURE
1785
1786 Your autobuild[%s] on %s failed after %.1f minutes
1787 when trying to test %s with the following error:
1788
1789    %s
1790
1791 the autobuild has been abandoned. Please fix the error and resubmit.
1792
1793 ####################################################################
1794
1795 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1796
1797 cleanup()
1798 do_print(errstr)
1799 do_print("Logs in logs.tar.gz")
1800 sys.exit(status)