script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34
  35 def find_git_root():
  36     '''get to the top of the git repo'''
  37     p = os.getcwd()
  38     while p != '/':
  39         if os.path.exists(os.path.join(p, ".git")):
  40             return p
  41         p = os.path.abspath(os.path.join(p, '..'))
  42     return None
  43
  44
  45 gitroot = find_git_root()
  46 if gitroot is None:
  47     raise Exception("Failed to find git root")
  48
  49
  50 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  51
  52 parser = OptionParser()
  53 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  54 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  55 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  56 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  57 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  58                   default=def_testbase)
  59 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  60                   default=None)
  61 parser.add_option("--passcmd", help="command to run on success", default=None)
  62 parser.add_option("--verbose", help="show all commands as they are run",
  63                   default=False, action="store_true")
  64 parser.add_option("--rebase", help="rebase on the given tree before testing",
  65                   default=None, type='str')
  66 parser.add_option("--pushto", help="push to a git url on success",
  67                   default=None, type='str')
  68 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  69                   default=False, action="store_true")
  70 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  71                   default=False, action="store_true")
  72 parser.add_option("--retry", help="automatically retry if master changes",
  73                   default=False, action="store_true")
  74 parser.add_option("--email", help="send email to the given address on failure",
  75                   type='str', default=None)
  76 parser.add_option("--email-from", help="send email from the given address",
  77                   type='str', default="autobuild@samba.org")
  78 parser.add_option("--email-server", help="send email via the given server",
  79                   type='str', default='localhost')
  80 parser.add_option("--always-email", help="always send email, even on success",
  81                   action="store_true")
  82 parser.add_option("--daemon", help="daemonize after initial setup",
  83                   action="store_true")
  84 parser.add_option("--branch", help="the branch to work on (default=master)",
  85                   default="master", type='str')
  86 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  87                   default=gitroot, type='str')
  88 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  89                   default=False, action="store_true")
  90 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  91                   default='')
  92 parser.add_option("--enable-coverage", dest='enable_coverage',
  93                   action="store_const", const='--enable-coverage', default='',
  94                   help="Add --enable-coverage option while configure")
  95
  96 (options, args) = parser.parse_args()
  97
  98 if options.retry:
  99     if options.rebase is None:
 100         raise Exception('You can only use --retry if you also rebase')
 101
 102 if options.full_testbase is not None:
 103     testbase = options.full_testbase
 104 else:
 105     testbase = "%s/b%u" % (options.testbase, os.getpid())
 106 test_master = "%s/master" % testbase
 107 test_prefix = "%s/prefix" % testbase
 108 test_tmpdir = "%s/tmp" % testbase
 109 os.environ['TMPDIR'] = test_tmpdir
 110
 111 if options.enable_coverage:
 112     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 113 else:
 114     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 115
 116 if options.enable_coverage:
 117     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 118 else:
 119     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 120
 121 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 122
 123 def nm_grep_symbols(sofile, expected_symbols=""):
 124     return "nm " + sofile + " | " + \
 125            "egrep -v ' (__bss_start|_edata|_init|_fini|_end)' | " + \
 126            "egrep -v '" + expected_symbols + "' |" + \
 127            "egrep ' [BDGTRVWS] ' && exit 1; exit 0;"
 128
 129 if args:
 130     # If we are only running specific test,
 131     # do not sleep randomly to wait for it to start
 132     def random_sleep(low, high):
 133         return 'sleep 1'
 134 else:
 135     def random_sleep(low, high):
 136         return 'sleep {}'.format(random.randint(low, high))
 137
 138 cleanup_list = []
 139
 140 builddirs = {
 141     "ctdb": "ctdb",
 142     "ldb": "lib/ldb",
 143     "tdb": "lib/tdb",
 144     "talloc": "lib/talloc",
 145     "replace": "lib/replace",
 146     "tevent": "lib/tevent",
 147     "pidl": "pidl",
 148     "docs-xml": "docs-xml"
 149 }
 150
 151 ctdb_configure_params = " --enable-developer ${PREFIX}"
 152 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 153
 154 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 155 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 156 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 157 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 158 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 159 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 160 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 161
 162
 163 def format_option(name, value=None):
 164     """Format option as str list."""
 165     if value is None:  # boolean option
 166         return [name]
 167     if not isinstance(value, list):  # single value option
 168         value = [value]
 169     # repeatable option
 170     return ['{}={}'.format(name, item) for item in value]
 171
 172
 173 def make_test(
 174         cmd='make testonly',
 175         INJECT_SELFTEST_PREFIX=1,
 176         TESTS='',
 177         include_envs=None,
 178         exclude_envs=None):
 179
 180     test_options = []
 181     if include_envs:
 182         test_options = format_option('--include-env', include_envs)
 183     if exclude_envs:
 184         test_options = format_option('--exclude-env', exclude_envs)
 185     if test_options:
 186         # join envs options to original test options
 187         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 188
 189     _options = []
 190
 191     # Allow getting a full CI with
 192     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 193
 194     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 195
 196     if int(FAIL_IMMEDIATELY):
 197         _options.append('FAIL_IMMEDIATELY=1')
 198     if TESTS:
 199         _options.append("TESTS='{}'".format(TESTS))
 200
 201     if INJECT_SELFTEST_PREFIX:
 202         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 203         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 204
 205     return ' '.join([cmd] + _options)
 206
 207
 208 # When updating this list, also update .gitlab-ci.yml to add the job
 209 # and to make it a dependency of 'page' for the coverage report.
 210
 211 tasks = {
 212     "ctdb": {
 213         "sequence": [
 214             ("random-sleep", random_sleep(300, 900)),
 215             ("configure", "./configure " + ctdb_configure_params),
 216             ("make", "make all"),
 217             ("install", "make install"),
 218             ("test", "make autotest"),
 219             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 220             ("clean", "make clean"),
 221         ],
 222     },
 223     "docs-xml": {
 224         "sequence": [
 225             ("random-sleep", random_sleep(300, 900)),
 226             ("autoconf", "autoconf"),
 227             ("configure", "./configure"),
 228             ("make", "make html htmlman"),
 229             ("publish-docs", PUBLISH_DOCS),
 230             ("clean", "make clean"),
 231         ],
 232     },
 233
 234     "samba-def-build": {
 235         "git-clone-required": True,
 236         "sequence": [
 237             ("configure", "./configure.developer" + samba_configure_params),
 238             ("make", "make -j"),
 239             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 240             ("chmod-R-a-w", "chmod -R a-w ."),
 241         ],
 242     },
 243
 244     "samba-mit-build": {
 245         "git-clone-required": True,
 246         "sequence": [
 247             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 248             ("make", "make -j"),
 249             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 250             ("chmod-R-a-w", "chmod -R a-w ."),
 251         ],
 252     },
 253
 254     "samba-nt4-build": {
 255         "git-clone-required": True,
 256         "sequence": [
 257             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 258             ("make", "make -j"),
 259             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 260             ("chmod-R-a-w", "chmod -R a-w ."),
 261         ],
 262     },
 263
 264     "samba-h5l-build": {
 265         "git-clone-required": True,
 266         "sequence": [
 267             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 268             ("make", "make -j"),
 269             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 270             ("chmod-R-a-w", "chmod -R a-w ."),
 271         ],
 272     },
 273
 274     "samba-no-opath-build": {
 275         "git-clone-required": True,
 276         "sequence": [
 277             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 278             ("make", "make -j"),
 279             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 280             ("chmod-R-a-w", "chmod -R a-w ."),
 281         ],
 282     },
 283
 284     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 285     "samba": {
 286         "sequence": [
 287             ("random-sleep", random_sleep(300, 900)),
 288             ("configure", "./configure.developer" + samba_configure_params),
 289             ("make", "make -j"),
 290             ("test", make_test(exclude_envs=[
 291             "none",
 292             "nt4_dc",
 293             "nt4_dc_smb1",
 294             "nt4_dc_smb1_done",
 295             "nt4_dc_schannel",
 296             "nt4_member",
 297             "ad_dc",
 298             "ad_dc_smb1",
 299             "ad_dc_smb1_done",
 300             "ad_dc_backup",
 301             "ad_dc_ntvfs",
 302             "ad_dc_default",
 303             "ad_dc_default_smb1",
 304             "ad_dc_slowtests",
 305             "ad_dc_no_nss",
 306             "ad_dc_no_ntlm",
 307             "fl2003dc",
 308             "fl2008dc",
 309             "fl2008r2dc",
 310             "ad_member",
 311             "ad_member_idmap_rid",
 312             "ad_member_idmap_ad",
 313             "ad_member_rfc2307",
 314             "ad_member_oneway",
 315             "chgdcpass",
 316             "vampire_2000_dc",
 317             "fl2000dc",
 318             "fileserver",
 319             "fileserver_smb1",
 320             "fileserver_smb1_done",
 321             "maptoguest",
 322             "simpleserver",
 323             "backupfromdc",
 324             "restoredc",
 325             "renamedc",
 326             "offlinebackupdc",
 327             "labdc",
 328             "preforkrestartdc",
 329             "proclimitdc",
 330             "promoted_dc",
 331             "vampire_dc",
 332             "rodc",
 333             "ad_dc_default",
 334             "ad_dc_default_smb1",
 335             "ad_dc_default_smb1_done",
 336             "ad_dc_slowtests",
 337             "schema_pair_dc",
 338             "schema_dc",
 339             "clusteredmember",
 340             "ad_dc_fips",
 341             "ad_member_fips",
 342             ])),
 343             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 344             ("lcov", LCOV_CMD),
 345             ("install", "make install"),
 346             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 347             ("clean", "make clean"),
 348         ],
 349     },
 350
 351     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 352     "samba-mitkrb5": {
 353         "sequence": [
 354             ("random-sleep", random_sleep(300, 900)),
 355             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 356             ("make", "make -j"),
 357             ("test", make_test(exclude_envs=[
 358             "none",
 359             "nt4_dc",
 360             "nt4_dc_smb1",
 361             "nt4_dc_smb1_done",
 362             "nt4_dc_schannel",
 363             "nt4_member",
 364             "ad_dc",
 365             "ad_dc_smb1",
 366             "ad_dc_smb1_done",
 367             "ad_dc_backup",
 368             "ad_dc_ntvfs",
 369             "ad_dc_default",
 370             "ad_dc_default_smb1",
 371             "ad_dc_default_smb1_done",
 372             "ad_dc_slowtests",
 373             "ad_dc_no_nss",
 374             "ad_dc_no_ntlm",
 375             "fl2003dc",
 376             "fl2008dc",
 377             "fl2008r2dc",
 378             "ad_member",
 379             "ad_member_idmap_rid",
 380             "ad_member_idmap_ad",
 381             "ad_member_rfc2307",
 382             "ad_member_oneway",
 383             "chgdcpass",
 384             "vampire_2000_dc",
 385             "fl2000dc",
 386             "fileserver",
 387             "fileserver_smb1",
 388             "fileserver_smb1_done",
 389             "maptoguest",
 390             "simpleserver",
 391             "backupfromdc",
 392             "restoredc",
 393             "renamedc",
 394             "offlinebackupdc",
 395             "labdc",
 396             "preforkrestartdc",
 397             "proclimitdc",
 398             "promoted_dc",
 399             "vampire_dc",
 400             "rodc",
 401             "ad_dc_default",
 402             "ad_dc_default_smb1",
 403             "ad_dc_default_smb1_done",
 404             "ad_dc_slowtests",
 405             "schema_pair_dc",
 406             "schema_dc",
 407             "clusteredmember",
 408             "ad_dc_fips",
 409             "ad_member_fips",
 410             ])),
 411             ("lcov", LCOV_CMD),
 412             ("install", "make install"),
 413             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 414             ("clean", "make clean"),
 415         ],
 416     },
 417
 418     "samba-nt4": {
 419         "dependency": "samba-nt4-build",
 420         "sequence": [
 421             ("random-sleep", random_sleep(300, 900)),
 422             ("test", make_test(include_envs=[
 423             "nt4_dc",
 424             "nt4_dc_smb1",
 425             "nt4_dc_smb1_done",
 426             "nt4_dc_schannel",
 427             "nt4_member",
 428             "simpleserver",
 429             ])),
 430             ("lcov", LCOV_CMD),
 431             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 432         ],
 433     },
 434
 435     "samba-fileserver": {
 436         "dependency": "samba-h5l-build",
 437         "sequence": [
 438             ("random-sleep", random_sleep(300, 900)),
 439             ("test", make_test(include_envs=[
 440             "fileserver",
 441             "fileserver_smb1",
 442             "fileserver_smb1_done",
 443             "maptoguest",
 444             "ktest", # ktest is also tested in samba-ktest-mit samba
 445                      # and samba-mitkrb5 but is tested here against
 446                      # a system Heimdal
 447             ])),
 448             ("lcov", LCOV_CMD),
 449             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 450         ],
 451     },
 452
 453     # This is a full build without the AD DC so we test the build with
 454     # MIT Kerberos from the current system.  Runtime behaviour is
 455     # confirmed via the ktest (static ccache and keytab) environment
 456
 457     "samba-ktest-mit": {
 458         "sequence": [
 459             ("random-sleep", random_sleep(300, 900)),
 460             ("configure", "./configure.developer --without-ad-dc --with-system-mitkrb5 " + samba_configure_params),
 461             ("make", "make -j"),
 462             ("test", make_test(include_envs=[
 463             "ktest", # ktest is also tested in fileserver, samba and
 464                      # samba-mitkrb5 but is tested here against a
 465                      # system MIT krb5
 466             ])),
 467             ("lcov", LCOV_CMD),
 468             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 469         ],
 470     },
 471
 472     "samba-admem": {
 473         "dependency": "samba-def-build",
 474         "sequence": [
 475             ("random-sleep", random_sleep(300, 900)),
 476             ("test", make_test(include_envs=[
 477             "ad_member",
 478             "ad_member_idmap_rid",
 479             "ad_member_idmap_ad",
 480             "ad_member_rfc2307",
 481             "ad_member_offlogon",
 482             ])),
 483             ("lcov", LCOV_CMD),
 484             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 485         ],
 486     },
 487
 488     "samba-no-opath1": {
 489         "dependency": "samba-no-opath-build",
 490         "sequence": [
 491             ("random-sleep", random_sleep(300, 900)),
 492             ("test", make_test(
 493                 cmd="make testonly DISABLE_OPATH=1",
 494                 include_envs=[
 495                 "nt4_dc",
 496                 "nt4_dc_smb1",
 497                 "nt4_dc_smb1_done",
 498                 "nt4_dc_schannel",
 499                 "nt4_member",
 500                 "simpleserver",
 501                 ])),
 502             ("lcov", LCOV_CMD),
 503             ("check-clean-tree", "script/clean-source-tree.sh"),
 504         ],
 505     },
 506
 507     "samba-no-opath2": {
 508         "dependency": "samba-no-opath-build",
 509         "sequence": [
 510             ("random-sleep", random_sleep(300, 900)),
 511             ("test", make_test(
 512                 cmd="make testonly DISABLE_OPATH=1",
 513                 include_envs=[
 514                 "fileserver",
 515                 "fileserver_smb1",
 516                 "fileserver_smb1_done",
 517                 ])),
 518             ("lcov", LCOV_CMD),
 519             ("check-clean-tree", "script/clean-source-tree.sh"),
 520         ],
 521     },
 522
 523     "samba-ad-dc-1": {
 524         "dependency": "samba-def-build",
 525         "sequence": [
 526             ("random-sleep", random_sleep(1, 1)),
 527             ("test", make_test(include_envs=[
 528             "ad_dc",
 529             "ad_dc_smb1",
 530             "ad_dc_smb1_done",
 531             "ad_dc_no_nss",
 532             "ad_dc_no_ntlm",
 533             ])),
 534             ("lcov", LCOV_CMD),
 535             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 536         ],
 537     },
 538
 539     "samba-ad-dc-2": {
 540         "dependency": "samba-def-build",
 541         "sequence": [
 542             ("random-sleep", random_sleep(1, 1)),
 543             ("test", make_test(include_envs=[
 544             "vampire_dc",
 545             "vampire_2000_dc",
 546             "rodc",
 547             ])),
 548             ("lcov", LCOV_CMD),
 549             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 550         ],
 551     },
 552
 553     "samba-ad-dc-3": {
 554         "dependency": "samba-def-build",
 555         "sequence": [
 556             ("random-sleep", random_sleep(1, 1)),
 557             ("test", make_test(include_envs=[
 558             "promoted_dc",
 559             "chgdcpass",
 560             "preforkrestartdc",
 561             "proclimitdc",
 562             ])),
 563             ("lcov", LCOV_CMD),
 564             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 565         ],
 566     },
 567
 568     "samba-ad-dc-4a": {
 569         "dependency": "samba-def-build",
 570         "sequence": [
 571             ("random-sleep", random_sleep(1, 1)),
 572             ("test", make_test(include_envs=[
 573             "fl2000dc",
 574             "ad_member_oneway",
 575             "fl2003dc",
 576             ])),
 577             ("lcov", LCOV_CMD),
 578             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 579         ],
 580     },
 581     "samba-ad-dc-4b": {
 582         "dependency": "samba-def-build",
 583         "sequence": [
 584             ("random-sleep", random_sleep(1, 1)),
 585             ("test", make_test(include_envs=[
 586             "fl2008dc",
 587             "fl2008r2dc",
 588             ])),
 589             ("lcov", LCOV_CMD),
 590             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 591         ],
 592     },
 593
 594     "samba-ad-dc-5": {
 595         "dependency": "samba-def-build",
 596         "sequence": [
 597             ("random-sleep", random_sleep(1, 1)),
 598             ("test", make_test(include_envs=[
 599             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 600             ("lcov", LCOV_CMD),
 601             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 602         ],
 603     },
 604
 605     "samba-ad-dc-6": {
 606         "dependency": "samba-def-build",
 607         "sequence": [
 608             ("random-sleep", random_sleep(1, 1)),
 609             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 610             ("lcov", LCOV_CMD),
 611             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 612         ],
 613     },
 614
 615     "samba-schemaupgrade": {
 616         "dependency": "samba-def-build",
 617         "sequence": [
 618             ("random-sleep", random_sleep(1, 1)),
 619             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 620             ("lcov", LCOV_CMD),
 621             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 622         ],
 623     },
 624
 625     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 626     # This is currently the longest task, so we don't randomly delay it.
 627     "samba-ad-dc-ntvfs": {
 628         "dependency": "samba-def-build",
 629         "sequence": [
 630             ("random-sleep", random_sleep(1, 1)),
 631             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 632             ("lcov", LCOV_CMD),
 633             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 634         ],
 635     },
 636
 637     # Test fips compliance
 638     "samba-fips": {
 639         "dependency": "samba-mit-build",
 640         "sequence": [
 641             ("random-sleep", random_sleep(1, 1)),
 642             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 643             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 644             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 645         ],
 646     },
 647
 648     # run the backup/restore testenvs separately as they're fairly standalone
 649     # (and CI seems to max out at ~3 different DCs running at once)
 650     "samba-ad-back1": {
 651         "dependency": "samba-def-build",
 652         "sequence": [
 653             ("random-sleep", random_sleep(300, 900)),
 654             ("test", make_test(include_envs=[
 655             "backupfromdc",
 656             "restoredc",
 657             "renamedc",
 658             ])),
 659             ("lcov", LCOV_CMD),
 660             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 661         ],
 662     },
 663     "samba-ad-back2": {
 664         "dependency": "samba-def-build",
 665         "sequence": [
 666             ("random-sleep", random_sleep(300, 900)),
 667             ("test", make_test(include_envs=[
 668             "backupfromdc",
 669             "offlinebackupdc",
 670             "labdc",
 671             ])),
 672             ("lcov", LCOV_CMD),
 673             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 674         ],
 675     },
 676
 677     "samba-admem-mit": {
 678         "dependency": "samba-mit-build",
 679         "sequence": [
 680             ("random-sleep", random_sleep(1, 1)),
 681             ("test", make_test(include_envs=[
 682             "ad_member",
 683             "ad_member_idmap_rid",
 684             "ad_member_idmap_ad",
 685             "ad_member_rfc2307",
 686             "ad_member_offlogon",
 687             ])),
 688             ("lcov", LCOV_CMD),
 689             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 690         ],
 691     },
 692
 693     "samba-addc-mit-1": {
 694         "dependency": "samba-mit-build",
 695         "sequence": [
 696             ("random-sleep", random_sleep(1, 1)),
 697             ("test", make_test(include_envs=[
 698             "ad_dc",
 699             "ad_dc_smb1",
 700             "ad_dc_smb1_done",
 701             "ad_dc_no_nss",
 702             "ad_dc_no_ntlm",
 703             ])),
 704             ("lcov", LCOV_CMD),
 705             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 706         ],
 707     },
 708
 709     "samba-addc-mit-4a": {
 710         "dependency": "samba-mit-build",
 711         "sequence": [
 712             ("random-sleep", random_sleep(1, 1)),
 713             ("test", make_test(include_envs=[
 714             "fl2000dc",
 715             "ad_member_oneway",
 716             "fl2003dc",
 717             ])),
 718             ("lcov", LCOV_CMD),
 719             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 720         ],
 721     },
 722     "samba-addc-mit-4b": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "fl2008dc",
 728             "fl2008r2dc",
 729             ])),
 730             ("lcov", LCOV_CMD),
 731             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 732         ],
 733     },
 734
 735     "samba-test-only": {
 736         "sequence": [
 737             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 738             ("make", "make -j"),
 739             ("test", make_test(TESTS="${TESTS}")),
 740             ("lcov", LCOV_CMD),
 741         ],
 742     },
 743
 744     # Test cross-compile infrastructure
 745     "samba-xc": {
 746         "sequence": [
 747             ("random-sleep", random_sleep(900, 1500)),
 748             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 749             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 750             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 751             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 752             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 753             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 754             ("compare-results", "script/compare_cc_results.py "
 755             "./bin/c4che/default{} "
 756             "./bin-xe/c4che/default{} "
 757             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 758             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 759             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 760             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 761             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 762             " = \"'1234'\"".format(CACHE_SUFFIX)),
 763             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 764             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 765             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 766             " ; test $? -ne 0"),
 767         ],
 768     },
 769
 770     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 771     "samba-o3": {
 772         "sequence": [
 773             ("random-sleep", random_sleep(300, 900)),
 774             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 775             ("make", "make -j"),
 776             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 777             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 778             ("lcov", LCOV_CMD),
 779             ("install", "make install"),
 780             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 781             ("clean", "make clean"),
 782         ],
 783     },
 784
 785     "samba-ctdb": {
 786         "sequence": [
 787             ("random-sleep", random_sleep(900, 1500)),
 788
 789         # make sure we have tdb around:
 790             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 791             ("tdb-make", "cd lib/tdb && make"),
 792             ("tdb-install", "cd lib/tdb && make install"),
 793
 794         # build samba with cluster support (also building ctdb):
 795             ("samba-configure",
 796          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 797          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 798          "./configure.developer ${PREFIX} "
 799          "--with-selftest-prefix=./bin/ab "
 800          "--with-cluster-support "
 801          "--without-ad-dc "
 802          "--bundled-libraries=!tdb"),
 803             ("samba-make", "make"),
 804             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 805             ("samba-install", "make install"),
 806             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 807
 808             ("test", make_test(
 809                 cmd='make test',
 810                 INJECT_SELFTEST_PREFIX=0,
 811                 include_envs=["clusteredmember"])
 812             ),
 813
 814         # clean up:
 815             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 816             ("clean", "make clean"),
 817             ("ctdb-clean", "cd ./ctdb && make clean"),
 818         ],
 819     },
 820
 821     "samba-libs": {
 822         "sequence": [
 823             ("random-sleep", random_sleep(300, 900)),
 824             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 825             ("talloc-make", "cd lib/talloc && make"),
 826             ("talloc-install", "cd lib/talloc && make install"),
 827
 828             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 829             ("tdb-make", "cd lib/tdb && make"),
 830             ("tdb-install", "cd lib/tdb && make install"),
 831
 832             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 833             ("tevent-make", "cd lib/tevent && make"),
 834             ("tevent-install", "cd lib/tevent && make install"),
 835
 836             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 837             ("ldb-make", "cd lib/ldb && make"),
 838             ("ldb-install", "cd lib/ldb && make install"),
 839
 840             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 841             ("nondevel-make", "make -j"),
 842             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 843             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 844             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 845             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 846             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 847             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 848             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 849             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 850             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 851             ("nondevel-no-public-nss_winbind",
 852                 nm_grep_symbols("./bin/plugins/libnss_winbind.so.2", " T _nss_winbind_")),
 853             ("nondevel-no-public-nss_wins",
 854                 nm_grep_symbols("./bin/plugins/libnss_wins.so.2", " T _nss_wins_")),
 855             ("nondevel-no-public-libwbclient",
 856                 nm_grep_symbols("./bin/shared/libwbclient.so.0", " T wbc")),
 857             ("nondevel-no-public-pam_winbind",
 858                 nm_grep_symbols("./bin/plugins/pam_winbind.so", "T pam_sm_")),
 859             ("nondevel-no-public-winbind_krb5_locator",
 860                 nm_grep_symbols("./bin/plugins/winbind_krb5_locator.so", " D resolve\>")),
 861             ("nondevel-no-public-async_dns_krb5_locator",
 862                 nm_grep_symbols("./bin/plugins/async_dns_krb5_locator.so", " D resolve\>")),
 863             ("nondevel-install", "make -j install"),
 864             ("nondevel-dist", "make dist"),
 865
 866             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 867             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 868             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 869             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 870             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 871             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 872             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 873             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 874             ("prefix-no-public-nss_winbind",
 875                 nm_grep_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", " T _nss_winbind_")),
 876             ("prefix-no-public-nss_wins",
 877                 nm_grep_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", " T _nss_wins_")),
 878             ("prefix-no-public-libwbclient",
 879                 nm_grep_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", " T wbc")),
 880             ("prefix-no-public-pam_winbind",
 881                 nm_grep_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "T pam_sm_")),
 882             ("prefix-no-public-winbind_krb5_locator",
 883                 nm_grep_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so", " D resolve\>")),
 884             ("prefix-no-public-async_dns_krb5_locator",
 885                 nm_grep_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so", " D resolve\>")),
 886
 887             # retry with all modules shared
 888             ("allshared-distclean", "make distclean"),
 889             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 890             ("allshared-make", "make -j"),
 891             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 892             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 893             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 894             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 895             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 896             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 897             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 898             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 899             ("allshared-no-public-nss_winbind",
 900                 nm_grep_symbols("./bin/plugins/libnss_winbind.so.2", " T _nss_winbind_")),
 901             ("allshared-no-public-nss_wins",
 902                 nm_grep_symbols("./bin/plugins/libnss_wins.so.2", " T _nss_wins_")),
 903             ("allshared-no-public-libwbclient",
 904                 nm_grep_symbols("./bin/shared/libwbclient.so.0", " T wbc")),
 905             ("allshared-no-public-pam_winbind",
 906                 nm_grep_symbols("./bin/plugins/pam_winbind.so", "T pam_sm_")),
 907             ("allshared-no-public-winbind_krb5_locator",
 908                 nm_grep_symbols("./bin/plugins/winbind_krb5_locator.so", " D resolve\>")),
 909             ("allshared-no-public-async_dns_krb5_locator",
 910                 nm_grep_symbols("./bin/plugins/async_dns_krb5_locator.so", " D resolve\>")),
 911         ],
 912     },
 913
 914     "samba-fuzz": {
 915         "sequence": [
 916         # build the fuzzers (static) via the oss-fuzz script
 917             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 918             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 919         ],
 920     },
 921
 922     # * Test smbd and smbtorture can build semi-static
 923     #
 924     # * Test Samba without python still builds.
 925     #
 926     # When this test fails due to more use of Python, the expectations
 927     # is that the newly failing part of the code should be disabled
 928     # when --disable-python is set (rather than major work being done
 929     # to support this environment).
 930     #
 931     # The target here is for vendors shipping a minimal smbd.
 932     "samba-minimal-smbd": {
 933         "sequence": [
 934             ("random-sleep", random_sleep(300, 900)),
 935
 936         # build with all modules static
 937             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 938             ("allstatic-make", "make -j"),
 939             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 940             ("allstatic-lcov", LCOV_CMD),
 941
 942         # retry with nonshared smbd and smbtorture
 943             ("nonshared-distclean", "make distclean"),
 944             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 945             ("nonshared-make", "make -j"),
 946             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 947             # TODO ("nonshared-lcov", LCOV_CMD),
 948
 949             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 950             ("clean", "make clean"),
 951         ],
 952     },
 953
 954     "samba-nopython": {
 955         "sequence": [
 956             ("random-sleep", random_sleep(300, 900)),
 957
 958             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
 959             ("make", "make -j"),
 960             ("find-python", "script/find_python.sh ${PREFIX}"),
 961             ("test", "make test-nopython"),
 962             ("lcov", LCOV_CMD),
 963             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 964             ("clean", "make clean"),
 965
 966             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 967             ("talloc-make", "cd lib/talloc && make"),
 968             ("talloc-install", "cd lib/talloc && make install"),
 969
 970             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 971             ("tdb-make", "cd lib/tdb && make"),
 972             ("tdb-install", "cd lib/tdb && make install"),
 973
 974             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 975             ("tevent-make", "cd lib/tevent && make"),
 976             ("tevent-install", "cd lib/tevent && make install"),
 977
 978             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
 979             ("ldb-make", "cd lib/ldb && make"),
 980             ("ldb-install", "cd lib/ldb && make install"),
 981
 982         # retry against installed library packages, but no required modules
 983             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
 984             ("libs-make", "make -j"),
 985             ("libs-install", "make install"),
 986             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 987             ("libs-clean", "make clean"),
 988
 989         ],
 990     },
 991
 992     "ldb": {
 993         "sequence": [
 994             ("random-sleep", random_sleep(60, 600)),
 995             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
 996             ("make", "make"),
 997             ("install", "make install"),
 998             ("test", "make test"),
 999             ("lcov", LCOV_CMD),
1000             ("clean", "make clean"),
1001             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1002             ("make-no-lmdb", "make"),
1003             ("test-no-lmdb", "make test"),
1004             ("lcov-no-lmdb", LCOV_CMD),
1005             ("install-no-lmdb", "make install"),
1006             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1007             ("distcheck", "make distcheck"),
1008             ("clean", "make clean"),
1009         ],
1010     },
1011
1012     "tdb": {
1013         "sequence": [
1014             ("random-sleep", random_sleep(60, 600)),
1015             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1016             ("make", "make"),
1017             ("install", "make install"),
1018             ("test", "make test"),
1019             ("lcov", LCOV_CMD),
1020             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1021             ("distcheck", "make distcheck"),
1022             ("clean", "make clean"),
1023         ],
1024     },
1025
1026     "talloc": {
1027         "sequence": [
1028             ("random-sleep", random_sleep(60, 600)),
1029             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1030             ("make", "make"),
1031             ("install", "make install"),
1032             ("test", "make test"),
1033             ("lcov", LCOV_CMD),
1034             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1035             ("distcheck", "make distcheck"),
1036             ("clean", "make clean"),
1037         ],
1038     },
1039
1040     "replace": {
1041         "sequence": [
1042             ("random-sleep", random_sleep(60, 600)),
1043             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1044             ("make", "make"),
1045             ("install", "make install"),
1046             ("test", "make test"),
1047             ("lcov", LCOV_CMD),
1048             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1049             ("distcheck", "make distcheck"),
1050             ("clean", "make clean"),
1051         ],
1052     },
1053
1054     "tevent": {
1055         "sequence": [
1056             ("random-sleep", random_sleep(60, 600)),
1057             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1058             ("make", "make"),
1059             ("install", "make install"),
1060             ("test", "make test"),
1061             ("lcov", LCOV_CMD),
1062             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1063             ("distcheck", "make distcheck"),
1064             ("clean", "make clean"),
1065         ],
1066     },
1067
1068     "pidl": {
1069         "git-clone-required": True,
1070         "sequence": [
1071             ("random-sleep", random_sleep(60, 600)),
1072             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1073             ("touch", "touch *.yp"),
1074             ("make", "make"),
1075             ("test", "make test"),
1076             ("install", "make install"),
1077             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1078             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1079             ("clean", "make clean"),
1080         ],
1081     },
1082
1083     # these are useful for debugging autobuild
1084     "pass": {
1085         "sequence": [
1086             ("pass", 'echo passing && /bin/true'),
1087         ],
1088     },
1089     "fail": {
1090         "sequence": [
1091             ("fail", 'echo failing && /bin/false'),
1092         ],
1093     },
1094 }
1095
1096 defaulttasks = list(tasks.keys())
1097
1098 defaulttasks.remove("pass")
1099 defaulttasks.remove("fail")
1100
1101 # The build tasks will be brought in by the test tasks as needed
1102 defaulttasks.remove("samba-def-build")
1103 defaulttasks.remove("samba-nt4-build")
1104 defaulttasks.remove("samba-mit-build")
1105 defaulttasks.remove("samba-h5l-build")
1106 defaulttasks.remove("samba-no-opath-build")
1107
1108 # This is not a normal test, but a task to support manually running
1109 # one test under autobuild
1110 defaulttasks.remove("samba-test-only")
1111
1112 # Only built on GitLab CI and not in the default autobuild because it
1113 # uses too much space (4GB of semi-static binaries)
1114 defaulttasks.remove("samba-fuzz")
1115
1116 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1117 # container where a simulated FIPS mode is possible.
1118 defaulttasks.remove("samba-fips")
1119
1120 # The MIT build runs on a current Fedora where an up to date MIT KDC
1121 # is already packaged.  This avoids needing to backport a current MIT
1122 # to the default Ubuntu 18.04, particularly during development, and
1123 # the need to install on the shared sn-devel-184.
1124
1125 defaulttasks.remove("samba-mitkrb5")
1126 defaulttasks.remove("samba-admem-mit")
1127 defaulttasks.remove("samba-addc-mit-1")
1128 defaulttasks.remove("samba-addc-mit-4a")
1129 defaulttasks.remove("samba-addc-mit-4b")
1130
1131 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1132     defaulttasks.remove("samba-o3")
1133
1134
1135 def do_print(msg):
1136     print("%s" % msg)
1137     sys.stdout.flush()
1138     sys.stderr.flush()
1139
1140
1141 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1142     if show is None:
1143         show = options.verbose
1144     if show:
1145         do_print("Running: '%s' in '%s'" % (cmd, dir))
1146     if output:
1147         out = check_output([cmd], shell=True, cwd=dir)
1148         return out.decode(encoding='utf-8', errors='backslashreplace')
1149     elif checkfail:
1150         return check_call(cmd, shell=True, cwd=dir)
1151     else:
1152         return call(cmd, shell=True, cwd=dir)
1153
1154 def rmdir_force(dirname, re_raise=True):
1155     try:
1156         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1157                 dirname, dirname, dirname), output=True, show=True)
1158     except CalledProcessError as e:
1159         do_print("Failed: '%s'" % (str(e)))
1160         run_cmd("tree %s" % dirname, output=True, show=True)
1161         if re_raise:
1162             raise
1163         return False
1164     return True
1165
1166 class builder(object):
1167     '''handle build of one directory'''
1168
1169     def __init__(self, name, definition):
1170         self.name = name
1171         self.dir = builddirs.get(name, '.')
1172         self.tag = self.name.replace('/', '_')
1173         self.definition = definition
1174         self.sequence = definition["sequence"]
1175         self.git_clone_required = False
1176         if "git-clone-required" in definition:
1177             self.git_clone_required = bool(definition["git-clone-required"])
1178         self.proc = None
1179         self.done = False
1180         self.next = 0
1181         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1182         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1183         if options.verbose:
1184             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1185             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1186         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1187         self.stdout = open(self.stdout_path, 'w')
1188         self.stderr = open(self.stderr_path, 'w')
1189         self.stdin  = open("/dev/null", 'r')
1190         self.builder_dir = "%s/%s" % (testbase, self.tag)
1191         self.test_source_dir = self.builder_dir
1192         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1193         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1194         self.prefix = "%s/%s" % (test_prefix, self.tag)
1195         self.consumers = []
1196         self.producer = None
1197
1198         if self.git_clone_required:
1199             assert "dependency" not in definition
1200
1201     def mark_existing(self):
1202         do_print('%s: Mark as existing dependency' % self.name)
1203         self.next = len(self.sequence)
1204         self.done = True
1205
1206     def add_consumer(self, consumer):
1207         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1208         consumer.producer = self
1209         consumer.test_source_dir = self.test_source_dir
1210         self.consumers.append(consumer)
1211
1212     def start_next(self):
1213         if self.producer is not None:
1214             if not self.producer.done:
1215                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1216                 return
1217
1218         if self.next == 0:
1219             rmdir_force(self.builder_dir)
1220             rmdir_force(self.prefix)
1221             if self.producer is not None:
1222                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1223             elif not self.git_clone_required:
1224                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1225             else:
1226                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1227
1228         if self.next == len(self.sequence):
1229             if not self.done:
1230                 do_print('%s: Completed OK' % self.name)
1231                 self.done = True
1232             if not options.nocleanup and len(self.consumers) == 0:
1233                 do_print('%s: Cleaning up' % self.name)
1234                 rmdir_force(self.builder_dir)
1235                 rmdir_force(self.prefix)
1236             for consumer in self.consumers:
1237                 if consumer.next != 0:
1238                     continue
1239                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1240                 consumer.start_next()
1241             if self.producer is not None:
1242                 self.producer.consumers.remove(self)
1243                 assert self.producer.done
1244                 self.producer.start_next()
1245             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1246             return
1247         (self.stage, self.cmd) = self.sequence[self.next]
1248         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1249         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1250         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1251         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1252         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1253         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1254         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1255         self.cmd = self.cmd.replace("${NAME}", self.name)
1256         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1257         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1258         self.proc = Popen(self.cmd, shell=True,
1259                           close_fds=True, cwd=self.cwd,
1260                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1261         self.next += 1
1262
1263 def expand_dependencies(n):
1264     deps = list()
1265     if "dependency" in tasks[n]:
1266         depname = tasks[n]["dependency"]
1267         assert depname in tasks
1268         sdeps = expand_dependencies(depname)
1269         assert n not in sdeps
1270         for sdep in sdeps:
1271             deps.append(sdep)
1272         deps.append(depname)
1273     return deps
1274
1275
1276 class buildlist(object):
1277     '''handle build of multiple directories'''
1278
1279     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1280         self.tail_proc = None
1281         self.retry = None
1282         if not tasknames:
1283             if options.restrict_tests:
1284                 tasknames = ["samba-test-only"]
1285             else:
1286                 tasknames = defaulttasks
1287
1288         given_tasknames = tasknames.copy()
1289         implicit_tasknames = []
1290         for n in given_tasknames:
1291             deps = expand_dependencies(n)
1292             for dep in deps:
1293                 if dep in given_tasknames:
1294                     continue
1295                 if dep in implicit_tasknames:
1296                     continue
1297                 implicit_tasknames.append(dep)
1298
1299         tasknames = implicit_tasknames.copy()
1300         tasknames.extend(given_tasknames)
1301         do_print("given_tasknames: %s" % given_tasknames)
1302         do_print("implicit_tasknames: %s" % implicit_tasknames)
1303         do_print("tasknames: %s" % tasknames)
1304         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1305
1306         if options.retry:
1307             rebase_remote = "rebaseon"
1308             retry_task = {
1309                     "git-clone-required": True,
1310                     "sequence": [
1311                             ("retry",
1312                             '''set -e
1313                             git remote add -t %s %s %s
1314                             git fetch %s
1315                             while :; do
1316                               sleep 60
1317                               git describe %s/%s > old_remote_branch.desc
1318                               git fetch %s
1319                               git describe %s/%s > remote_branch.desc
1320                               diff old_remote_branch.desc remote_branch.desc
1321                             done
1322                            ''' % (
1323                                rebase_branch, rebase_remote, rebase_url,
1324                                rebase_remote,
1325                                rebase_remote, rebase_branch,
1326                                rebase_remote,
1327                                rebase_remote, rebase_branch
1328                             ))]}
1329
1330             self.retry = builder('retry', retry_task)
1331             self.need_retry = False
1332
1333         if options.skip_dependencies:
1334             for b in self.tlist:
1335                 if b.name in implicit_tasknames:
1336                     b.mark_existing()
1337
1338         for b in self.tlist:
1339             do_print("b.name=%s" % b.name)
1340             if "dependency" not in b.definition:
1341                 continue
1342             depname = b.definition["dependency"]
1343             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1344             for p in self.tlist:
1345                 if p.name == depname:
1346                     p.add_consumer(b)
1347
1348     def kill_kids(self):
1349         if self.tail_proc is not None:
1350             self.tail_proc.terminate()
1351             self.tail_proc.wait()
1352             self.tail_proc = None
1353         if self.retry is not None:
1354             self.retry.proc.terminate()
1355             self.retry.proc.wait()
1356             self.retry = None
1357         for b in self.tlist:
1358             if b.proc is not None:
1359                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1360                 b.proc.terminate()
1361                 b.proc.wait()
1362                 b.proc = None
1363
1364     def wait_one(self):
1365         while True:
1366             none_running = True
1367             for b in self.tlist:
1368                 if b.proc is None:
1369                     continue
1370                 none_running = False
1371                 b.status = b.proc.poll()
1372                 if b.status is None:
1373                     continue
1374                 b.proc = None
1375                 return b
1376             if options.retry:
1377                 ret = self.retry.proc.poll()
1378                 if ret is not None:
1379                     self.need_retry = True
1380                     self.retry = None
1381                     return None
1382             if none_running:
1383                 return None
1384             time.sleep(0.1)
1385
1386     def run(self):
1387         for b in self.tlist:
1388             b.start_next()
1389         if options.retry:
1390             self.retry.start_next()
1391         while True:
1392             b = self.wait_one()
1393             if options.retry and self.need_retry:
1394                 self.kill_kids()
1395                 do_print("retry needed")
1396                 return (0, None, None, None, "retry")
1397             if b is None:
1398                 break
1399             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1400                 self.kill_kids()
1401                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1402             b.start_next()
1403         self.kill_kids()
1404         return (0, None, None, None, "All OK")
1405
1406     def write_system_info(self, filename):
1407         with open(filename, 'w') as f:
1408             for cmd in ['uname -a',
1409                         'lsb_release -a',
1410                         'free',
1411                         'mount',
1412                         'cat /proc/cpuinfo',
1413                         'cc --version',
1414                         'df -m .',
1415                         'df -m %s' % testbase]:
1416                 try:
1417                     out = run_cmd(cmd, output=True, checkfail=False)
1418                 except CalledProcessError as e:
1419                     out = "<failed: %s>" % str(e)
1420                 print('### %s' % cmd, file=f)
1421                 print(out, file=f)
1422                 print(file=f)
1423
1424     def tarlogs(self, fname):
1425         with tarfile.open(fname, "w:gz") as tar:
1426             for b in self.tlist:
1427                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1428                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1429             if os.path.exists("autobuild.log"):
1430                 tar.add("autobuild.log")
1431             filename = 'system-info.txt'
1432             self.write_system_info(filename)
1433             tar.add(filename)
1434
1435     def remove_logs(self):
1436         for b in self.tlist:
1437             os.unlink(b.stdout_path)
1438             os.unlink(b.stderr_path)
1439
1440     def start_tail(self):
1441         cmd = ["tail", "-f"]
1442         for b in self.tlist:
1443             cmd.append(b.stdout_path)
1444             cmd.append(b.stderr_path)
1445         self.tail_proc = Popen(cmd, close_fds=True)
1446
1447
1448 def cleanup(do_raise=False):
1449     if options.nocleanup:
1450         return
1451     run_cmd("stat %s || true" % test_tmpdir, show=True)
1452     run_cmd("stat %s" % testbase, show=True)
1453     do_print("Cleaning up %r" % cleanup_list)
1454     for d in cleanup_list:
1455         ok = rmdir_force(d, re_raise=False)
1456         if ok:
1457             continue
1458         if os.path.isdir(d):
1459             do_print("Killing, waiting and retry")
1460             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1461         else:
1462             do_print("Waiting and retry")
1463         time.sleep(1)
1464         rmdir_force(d, re_raise=do_raise)
1465
1466
1467 def daemonize(logfile):
1468     pid = os.fork()
1469     if pid == 0:  # Parent
1470         os.setsid()
1471         pid = os.fork()
1472         if pid != 0:  # Actual daemon
1473             os._exit(0)
1474     else:  # Grandparent
1475         os._exit(0)
1476
1477     import resource      # Resource usage information.
1478     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1479     if maxfd == resource.RLIM_INFINITY:
1480         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1481     for fd in range(0, maxfd):
1482         try:
1483             os.close(fd)
1484         except OSError:
1485             pass
1486     os.open(logfile, os.O_RDWR | os.O_CREAT)
1487     os.dup2(0, 1)
1488     os.dup2(0, 2)
1489
1490
1491 def write_pidfile(fname):
1492     '''write a pid file, cleanup on exit'''
1493     with open(fname, mode='w') as f:
1494         f.write("%u\n" % os.getpid())
1495
1496
1497 def rebase_tree(rebase_url, rebase_branch="master"):
1498     rebase_remote = "rebaseon"
1499     do_print("Rebasing on %s" % rebase_url)
1500     run_cmd("git describe HEAD", show=True, dir=test_master)
1501     run_cmd("git remote add -t %s %s %s" %
1502             (rebase_branch, rebase_remote, rebase_url),
1503             show=True, dir=test_master)
1504     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1505     if options.fix_whitespace:
1506         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1507                 (rebase_remote, rebase_branch),
1508                 show=True, dir=test_master)
1509     else:
1510         run_cmd("git rebase --force-rebase %s/%s" %
1511                 (rebase_remote, rebase_branch),
1512                 show=True, dir=test_master)
1513     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1514                    (rebase_remote, rebase_branch),
1515                    dir=test_master, output=True)
1516     if diff == '':
1517         do_print("No differences between HEAD and %s/%s - exiting" %
1518                  (rebase_remote, rebase_branch))
1519         sys.exit(0)
1520     run_cmd("git describe %s/%s" %
1521             (rebase_remote, rebase_branch),
1522             show=True, dir=test_master)
1523     run_cmd("git describe HEAD", show=True, dir=test_master)
1524     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1525             (rebase_remote, rebase_branch),
1526             show=True, dir=test_master)
1527
1528
1529 def push_to(push_url, push_branch="master"):
1530     push_remote = "pushto"
1531     do_print("Pushing to %s" % push_url)
1532     if options.mark:
1533         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1534         run_cmd("git commit --amend -c HEAD", dir=test_master)
1535         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1536         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1537     run_cmd("git remote add -t %s %s %s" %
1538             (push_branch, push_remote, push_url),
1539             show=True, dir=test_master)
1540     run_cmd("git push %s +HEAD:%s" %
1541             (push_remote, push_branch),
1542             show=True, dir=test_master)
1543
1544
1545 def send_email(subject, text, log_tar):
1546     if options.email is None:
1547         do_print("not sending email because the recipient is not set")
1548         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1549                  (subject, text))
1550         return
1551     outer = MIMEMultipart()
1552     outer['Subject'] = subject
1553     outer['To'] = options.email
1554     outer['From'] = options.email_from
1555     outer['Date'] = email.utils.formatdate(localtime=True)
1556     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1557     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1558     if options.attach_logs:
1559         with open(log_tar, 'rb') as fp:
1560             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1561         # Set the filename parameter
1562         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1563         outer.attach(msg)
1564     content = outer.as_string()
1565     s = smtplib.SMTP(options.email_server)
1566     email_user = os.getenv('SMTP_USERNAME')
1567     email_password = os.getenv('SMTP_PASSWORD')
1568     if email_user is not None:
1569         s.starttls()
1570         s.login(email_user, email_password)
1571
1572     s.sendmail(options.email_from, [options.email], content)
1573     s.set_debuglevel(1)
1574     s.quit()
1575
1576
1577 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1578                   elapsed_time, log_base=None, add_log_tail=True):
1579     '''send an email to options.email about the failure'''
1580     elapsed_minutes = elapsed_time / 60.0
1581     if log_base is None:
1582         log_base = gitroot
1583     text = '''
1584 Dear Developer,
1585
1586 Your autobuild on %s failed after %.1f minutes
1587 when trying to test %s with the following error:
1588
1589    %s
1590
1591 the autobuild has been abandoned. Please fix the error and resubmit.
1592
1593 A summary of the autobuild process is here:
1594
1595   %s/autobuild.log
1596 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1597
1598     if options.restrict_tests:
1599         text += """
1600 The build was restricted to tests matching %s\n""" % options.restrict_tests
1601
1602     if failed_task != 'rebase':
1603         text += '''
1604 You can see logs of the failed task here:
1605
1606   %s/%s.stdout
1607   %s/%s.stderr
1608
1609 or you can get full logs of all tasks in this job here:
1610
1611   %s/logs.tar.gz
1612
1613 The top commit for the tree that was built was:
1614
1615 %s
1616
1617 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1618
1619     if add_log_tail:
1620         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1621         lines = f.readlines()
1622         log_tail = "".join(lines[-50:])
1623         num_lines = len(lines)
1624         if num_lines < 50:
1625             # Also include stderr (compile failures) if < 50 lines of stdout
1626             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1627             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1628
1629         text += '''
1630 The last 50 lines of log messages:
1631
1632 %s
1633     ''' % log_tail
1634         f.close()
1635
1636     logs = os.path.join(gitroot, 'logs.tar.gz')
1637     send_email('autobuild[%s] failure on %s for task %s during %s'
1638                % (options.branch, platform.node(), failed_task, failed_stage),
1639                text, logs)
1640
1641
1642 def email_success(elapsed_time, log_base=None):
1643     '''send an email to options.email about a successful build'''
1644     if log_base is None:
1645         log_base = gitroot
1646     text = '''
1647 Dear Developer,
1648
1649 Your autobuild on %s has succeeded after %.1f minutes.
1650
1651 ''' % (platform.node(), elapsed_time / 60.)
1652
1653     if options.restrict_tests:
1654         text += """
1655 The build was restricted to tests matching %s\n""" % options.restrict_tests
1656
1657     if options.keeplogs:
1658         text += '''
1659
1660 you can get full logs of all tasks in this job here:
1661
1662   %s/logs.tar.gz
1663
1664 ''' % log_base
1665
1666     text += '''
1667 The top commit for the tree that was built was:
1668
1669 %s
1670 ''' % top_commit_msg
1671
1672     logs = os.path.join(gitroot, 'logs.tar.gz')
1673     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1674                text, logs)
1675
1676
1677 # get the top commit message, for emails
1678 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1679
1680 try:
1681     if options.skip_dependencies:
1682         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1683     else:
1684         os.makedirs(testbase)
1685 except Exception as reason:
1686     raise Exception("Unable to create %s : %s" % (testbase, reason))
1687 cleanup_list.append(testbase)
1688
1689 if options.daemon:
1690     logfile = os.path.join(testbase, "log")
1691     do_print("Forking into the background, writing progress to %s" % logfile)
1692     daemonize(logfile)
1693
1694 write_pidfile(gitroot + "/autobuild.pid")
1695
1696 start_time = time.time()
1697
1698 while True:
1699     try:
1700         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1701         os.makedirs(test_tmpdir)
1702         # The waf uninstall code removes empty directories all the way
1703         # up the tree.  Creating a file in test_tmpdir stops it from
1704         # being removed.
1705         run_cmd("touch %s" % os.path.join(test_tmpdir,
1706                                           ".directory-is-not-empty"), show=True)
1707         run_cmd("stat %s" % test_tmpdir, show=True)
1708         run_cmd("stat %s" % testbase, show=True)
1709         if options.skip_dependencies:
1710             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1711         else:
1712             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1713     except Exception:
1714         cleanup()
1715         raise
1716
1717     try:
1718         if options.rebase is not None:
1719             rebase_tree(options.rebase, rebase_branch=options.branch)
1720     except Exception:
1721         cleanup_list.append(gitroot + "/autobuild.pid")
1722         cleanup()
1723         elapsed_time = time.time() - start_time
1724         email_failure(-1, 'rebase', 'rebase', 'rebase',
1725                       'rebase on %s failed' % options.branch,
1726                       elapsed_time, log_base=options.log_base)
1727         sys.exit(1)
1728
1729     try:
1730         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1731         if options.tail:
1732             blist.start_tail()
1733         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1734         if status != 0 or errstr != "retry":
1735             break
1736         cleanup(do_raise=True)
1737     except Exception:
1738         cleanup()
1739         raise
1740
1741 cleanup_list.append(gitroot + "/autobuild.pid")
1742
1743 do_print(errstr)
1744
1745 blist.kill_kids()
1746 if options.tail:
1747     do_print("waiting for tail to flush")
1748     time.sleep(1)
1749
1750 elapsed_time = time.time() - start_time
1751 if status == 0:
1752     if options.passcmd is not None:
1753         do_print("Running passcmd: %s" % options.passcmd)
1754         run_cmd(options.passcmd, dir=test_master)
1755     if options.pushto is not None:
1756         push_to(options.pushto, push_branch=options.branch)
1757     if options.keeplogs or options.attach_logs:
1758         blist.tarlogs("logs.tar.gz")
1759         do_print("Logs in logs.tar.gz")
1760     if options.always_email:
1761         email_success(elapsed_time, log_base=options.log_base)
1762     blist.remove_logs()
1763     cleanup()
1764     do_print(errstr)
1765     sys.exit(0)
1766
1767 # something failed, gather a tar of the logs
1768 blist.tarlogs("logs.tar.gz")
1769
1770 if options.email is not None:
1771     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1772                   elapsed_time, log_base=options.log_base)
1773 else:
1774     elapsed_minutes = elapsed_time / 60.0
1775     print('''
1776
1777 ####################################################################
1778
1779 AUTOBUILD FAILURE
1780
1781 Your autobuild[%s] on %s failed after %.1f minutes
1782 when trying to test %s with the following error:
1783
1784    %s
1785
1786 the autobuild has been abandoned. Please fix the error and resubmit.
1787
1788 ####################################################################
1789
1790 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1791
1792 cleanup()
1793 do_print(errstr)
1794 do_print("Logs in logs.tar.gz")
1795 sys.exit(status)