script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from distutils.sysconfig import get_python_lib
  20 import platform
  21
  22 try:
  23     from waflib.Build import CACHE_SUFFIX
  24 except ImportError:
  25     sys.path.insert(0, "./third_party/waf")
  26     from waflib.Build import CACHE_SUFFIX
  27
  28
  29 os.environ["PYTHONUNBUFFERED"] = "1"
  30
  31 # This speeds up testing remarkably.
  32 os.environ['TDB_NO_FSYNC'] = '1'
  33
  34 # allow autobuild to run within git rebase -i
  35 if "GIT_DIR" in os.environ:
  36     del os.environ["GIT_DIR"]
  37 if "GIT_WORK_TREE" in os.environ:
  38     del os.environ["GIT_WORK_TREE"]
  39
  40 def find_git_root():
  41     '''get to the top of the git repo'''
  42     p = os.getcwd()
  43     while p != '/':
  44         if os.path.exists(os.path.join(p, ".git")):
  45             return p
  46         p = os.path.abspath(os.path.join(p, '..'))
  47     return None
  48
  49
  50 gitroot = find_git_root()
  51 if gitroot is None:
  52     raise Exception("Failed to find git root")
  53
  54
  55 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  56
  57 parser = OptionParser()
  58 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  59 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  60 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  61 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  62 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  63                   default=def_testbase)
  64 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  65                   default=None)
  66 parser.add_option("--passcmd", help="command to run on success", default=None)
  67 parser.add_option("--verbose", help="show all commands as they are run",
  68                   default=False, action="store_true")
  69 parser.add_option("--rebase", help="rebase on the given tree before testing",
  70                   default=None, type='str')
  71 parser.add_option("--pushto", help="push to a git url on success",
  72                   default=None, type='str')
  73 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  74                   default=False, action="store_true")
  75 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  76                   default=False, action="store_true")
  77 parser.add_option("--retry", help="automatically retry if master changes",
  78                   default=False, action="store_true")
  79 parser.add_option("--email", help="send email to the given address on failure",
  80                   type='str', default=None)
  81 parser.add_option("--email-from", help="send email from the given address",
  82                   type='str', default="autobuild@samba.org")
  83 parser.add_option("--email-server", help="send email via the given server",
  84                   type='str', default='localhost')
  85 parser.add_option("--always-email", help="always send email, even on success",
  86                   action="store_true")
  87 parser.add_option("--daemon", help="daemonize after initial setup",
  88                   action="store_true")
  89 parser.add_option("--branch", help="the branch to work on (default=master)",
  90                   default="master", type='str')
  91 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  92                   default=gitroot, type='str')
  93 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  94                   default=False, action="store_true")
  95 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
  96                   default='')
  97 parser.add_option("--enable-coverage", dest='enable_coverage',
  98                   action="store_const", const='--enable-coverage', default='',
  99                   help="Add --enable-coverage option while configure")
 100
 101 (options, args) = parser.parse_args()
 102
 103 if options.retry:
 104     if options.rebase is None:
 105         raise Exception('You can only use --retry if you also rebase')
 106
 107 if options.full_testbase is not None:
 108     testbase = options.full_testbase
 109 else:
 110     testbase = "%s/b%u" % (options.testbase, os.getpid())
 111 test_master = "%s/master" % testbase
 112 test_prefix = "%s/prefix" % testbase
 113 test_tmpdir = "%s/tmp" % testbase
 114 os.environ['TMPDIR'] = test_tmpdir
 115
 116 if options.enable_coverage:
 117     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 118 else:
 119     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 120
 121 if options.enable_coverage:
 122     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 123 else:
 124     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 125
 126 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 127
 128
 129 def check_symbols(sofile, expected_symbols=""):
 130     return "objdump --dynamic-syms " + sofile + " | " + \
 131            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 132
 133 if args:
 134     # If we are only running specific test,
 135     # do not sleep randomly to wait for it to start
 136     def random_sleep(low, high):
 137         return 'sleep 1'
 138 else:
 139     def random_sleep(low, high):
 140         return 'sleep {}'.format(random.randint(low, high))
 141
 142 cleanup_list = []
 143
 144 builddirs = {
 145     "ctdb": "ctdb",
 146     "ldb": "lib/ldb",
 147     "tdb": "lib/tdb",
 148     "talloc": "lib/talloc",
 149     "replace": "lib/replace",
 150     "tevent": "lib/tevent",
 151     "pidl": "pidl",
 152     "docs-xml": "docs-xml"
 153 }
 154
 155 ctdb_configure_params = " --enable-developer ${PREFIX}"
 156 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 157
 158 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 159 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 160 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 161 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 162 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 163 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt"
 164 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 165
 166
 167 def format_option(name, value=None):
 168     """Format option as str list."""
 169     if value is None:  # boolean option
 170         return [name]
 171     if not isinstance(value, list):  # single value option
 172         value = [value]
 173     # repeatable option
 174     return ['{}={}'.format(name, item) for item in value]
 175
 176
 177 def make_test(
 178         cmd='make testonly',
 179         INJECT_SELFTEST_PREFIX=1,
 180         TESTS='',
 181         include_envs=None,
 182         exclude_envs=None):
 183
 184     test_options = []
 185     if include_envs:
 186         test_options = format_option('--include-env', include_envs)
 187     if exclude_envs:
 188         test_options = format_option('--exclude-env', exclude_envs)
 189     if test_options:
 190         # join envs options to original test options
 191         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 192
 193     _options = []
 194
 195     # Allow getting a full CI with
 196     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 197
 198     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 199
 200     if int(FAIL_IMMEDIATELY):
 201         _options.append('FAIL_IMMEDIATELY=1')
 202     if TESTS:
 203         _options.append("TESTS='{}'".format(TESTS))
 204
 205     if INJECT_SELFTEST_PREFIX:
 206         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 207         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 208
 209     return ' '.join([cmd] + _options)
 210
 211
 212 # When updating this list, also update .gitlab-ci.yml to add the job
 213 # and to make it a dependency of 'page' for the coverage report.
 214
 215 tasks = {
 216     "ctdb": {
 217         "sequence": [
 218             ("random-sleep", random_sleep(300, 900)),
 219             ("configure", "./configure " + ctdb_configure_params),
 220             ("make", "make all"),
 221             ("install", "make install"),
 222             ("test", "make autotest"),
 223             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 224             ("clean", "make clean"),
 225         ],
 226     },
 227     "docs-xml": {
 228         "sequence": [
 229             ("random-sleep", random_sleep(300, 900)),
 230             ("autoconf", "autoconf"),
 231             ("configure", "./configure"),
 232             ("make", "make html htmlman"),
 233             ("publish-docs", PUBLISH_DOCS),
 234             ("clean", "make clean"),
 235         ],
 236     },
 237
 238     "samba-def-build": {
 239         "git-clone-required": True,
 240         "sequence": [
 241             ("configure", "./configure.developer" + samba_configure_params),
 242             ("make", "make -j"),
 243             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 244             ("chmod-R-a-w", "chmod -R a-w ."),
 245         ],
 246     },
 247
 248     "samba-mit-build": {
 249         "git-clone-required": True,
 250         "sequence": [
 251             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 252             ("make", "make -j"),
 253             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 254             ("chmod-R-a-w", "chmod -R a-w ."),
 255         ],
 256     },
 257
 258     "samba-nt4-build": {
 259         "git-clone-required": True,
 260         "sequence": [
 261             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 262             ("make", "make -j"),
 263             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 264             ("chmod-R-a-w", "chmod -R a-w ."),
 265         ],
 266     },
 267
 268     "samba-h5l-build": {
 269         "git-clone-required": True,
 270         "sequence": [
 271             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 272             ("make", "make -j"),
 273             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 274             ("chmod-R-a-w", "chmod -R a-w ."),
 275         ],
 276     },
 277
 278     "samba-without-smb1-build": {
 279         "git-clone-required": True,
 280         "sequence": [
 281             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 282             ("make", "make -j"),
 283             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 284             ("chmod-R-a-w", "chmod -R a-w ."),
 285         ],
 286     },
 287
 288     "samba-no-opath-build": {
 289         "git-clone-required": True,
 290         "sequence": [
 291             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 292             ("make", "make -j"),
 293             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 294             ("chmod-R-a-w", "chmod -R a-w ."),
 295         ],
 296     },
 297
 298     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 299     "samba": {
 300         "sequence": [
 301             ("random-sleep", random_sleep(300, 900)),
 302             ("configure", "./configure.developer" + samba_configure_params),
 303             ("make", "make -j"),
 304             ("test", make_test(exclude_envs=[
 305             "none",
 306             "nt4_dc",
 307             "nt4_dc_smb1",
 308             "nt4_dc_smb1_done",
 309             "nt4_dc_schannel",
 310             "nt4_member",
 311             "ad_dc",
 312             "ad_dc_smb1",
 313             "ad_dc_smb1_done",
 314             "ad_dc_backup",
 315             "ad_dc_ntvfs",
 316             "ad_dc_default",
 317             "ad_dc_default_smb1",
 318             "ad_dc_slowtests",
 319             "ad_dc_no_nss",
 320             "ad_dc_no_ntlm",
 321             "fl2003dc",
 322             "fl2008dc",
 323             "fl2008r2dc",
 324             "ad_member",
 325             "ad_member_idmap_rid",
 326             "admem_idmap_autorid",
 327             "ad_member_idmap_ad",
 328             "ad_member_rfc2307",
 329             "ad_member_oneway",
 330             "chgdcpass",
 331             "vampire_2000_dc",
 332             "fl2000dc",
 333             "fileserver",
 334             "fileserver_smb1",
 335             "fileserver_smb1_done",
 336             "maptoguest",
 337             "simpleserver",
 338             "backupfromdc",
 339             "restoredc",
 340             "renamedc",
 341             "offlinebackupdc",
 342             "labdc",
 343             "preforkrestartdc",
 344             "proclimitdc",
 345             "promoted_dc",
 346             "vampire_dc",
 347             "rodc",
 348             "ad_dc_default",
 349             "ad_dc_default_smb1",
 350             "ad_dc_default_smb1_done",
 351             "ad_dc_slowtests",
 352             "schema_pair_dc",
 353             "schema_dc",
 354             "clusteredmember",
 355             "ad_dc_fips",
 356             "ad_member_fips",
 357             ])),
 358             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 359             ("lcov", LCOV_CMD),
 360             ("install", "make install"),
 361             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 362             ("clean", "make clean"),
 363         ],
 364     },
 365
 366     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 367     "samba-mitkrb5": {
 368         "sequence": [
 369             ("random-sleep", random_sleep(300, 900)),
 370             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 371             ("make", "make -j"),
 372             ("test", make_test(exclude_envs=[
 373             "none",
 374             "nt4_dc",
 375             "nt4_dc_smb1",
 376             "nt4_dc_smb1_done",
 377             "nt4_dc_schannel",
 378             "nt4_member",
 379             "ad_dc",
 380             "ad_dc_smb1",
 381             "ad_dc_smb1_done",
 382             "ad_dc_backup",
 383             "ad_dc_ntvfs",
 384             "ad_dc_default",
 385             "ad_dc_default_smb1",
 386             "ad_dc_default_smb1_done",
 387             "ad_dc_slowtests",
 388             "ad_dc_no_nss",
 389             "ad_dc_no_ntlm",
 390             "fl2003dc",
 391             "fl2008dc",
 392             "fl2008r2dc",
 393             "ad_member",
 394             "ad_member_idmap_rid",
 395             "admem_idmap_autorid",
 396             "ad_member_idmap_ad",
 397             "ad_member_rfc2307",
 398             "ad_member_oneway",
 399             "chgdcpass",
 400             "vampire_2000_dc",
 401             "fl2000dc",
 402             "fileserver",
 403             "fileserver_smb1",
 404             "fileserver_smb1_done",
 405             "maptoguest",
 406             "simpleserver",
 407             "backupfromdc",
 408             "restoredc",
 409             "renamedc",
 410             "offlinebackupdc",
 411             "labdc",
 412             "preforkrestartdc",
 413             "proclimitdc",
 414             "promoted_dc",
 415             "vampire_dc",
 416             "rodc",
 417             "ad_dc_default",
 418             "ad_dc_default_smb1",
 419             "ad_dc_default_smb1_done",
 420             "ad_dc_slowtests",
 421             "schema_pair_dc",
 422             "schema_dc",
 423             "clusteredmember",
 424             "ad_dc_fips",
 425             "ad_member_fips",
 426             ])),
 427             ("lcov", LCOV_CMD),
 428             ("install", "make install"),
 429             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 430             ("clean", "make clean"),
 431         ],
 432     },
 433
 434     "samba-nt4": {
 435         "dependency": "samba-nt4-build",
 436         "sequence": [
 437             ("random-sleep", random_sleep(300, 900)),
 438             ("test", make_test(include_envs=[
 439             "nt4_dc",
 440             "nt4_dc_smb1",
 441             "nt4_dc_smb1_done",
 442             "nt4_dc_schannel",
 443             "nt4_member",
 444             "simpleserver",
 445             ])),
 446             ("lcov", LCOV_CMD),
 447             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 448         ],
 449     },
 450
 451     "samba-fileserver": {
 452         "dependency": "samba-h5l-build",
 453         "sequence": [
 454             ("random-sleep", random_sleep(300, 900)),
 455             ("test", make_test(include_envs=[
 456             "fileserver",
 457             "fileserver_smb1",
 458             "fileserver_smb1_done",
 459             "maptoguest",
 460             "ktest", # ktest is also tested in samba-ktest-mit samba
 461                      # and samba-mitkrb5 but is tested here against
 462                      # a system Heimdal
 463             ])),
 464             ("lcov", LCOV_CMD),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466         ],
 467     },
 468
 469     "samba-fileserver-without-smb1": {
 470         "dependency": "samba-without-smb1-build",
 471         "sequence": [
 472             ("random-sleep", random_sleep(300, 900)),
 473             ("test", make_test(include_envs=["fileserver"])),
 474             ("lcov", LCOV_CMD),
 475             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 476         ],
 477     },
 478
 479     # This is a full build without the AD DC so we test the build with
 480     # MIT Kerberos from the current system.  Runtime behaviour is
 481     # confirmed via the ktest (static ccache and keytab) environment
 482
 483     # This environment also used to confirm we can still build with --with-libunwind
 484     "samba-ktest-mit": {
 485         "sequence": [
 486             ("random-sleep", random_sleep(300, 900)),
 487             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 488             ("make", "make -j"),
 489             ("test", make_test(include_envs=[
 490             "ktest", # ktest is also tested in fileserver, samba and
 491                      # samba-mitkrb5 but is tested here against a
 492                      # system MIT krb5
 493             ])),
 494             ("lcov", LCOV_CMD),
 495             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 496         ],
 497     },
 498
 499     "samba-admem": {
 500         "dependency": "samba-def-build",
 501         "sequence": [
 502             ("random-sleep", random_sleep(300, 900)),
 503             ("test", make_test(include_envs=[
 504             "ad_member",
 505             "ad_member_idmap_rid",
 506             "admem_idmap_autorid",
 507             "ad_member_idmap_ad",
 508             "ad_member_rfc2307",
 509             "ad_member_offlogon",
 510             ])),
 511             ("lcov", LCOV_CMD),
 512             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 513         ],
 514     },
 515
 516     "samba-no-opath1": {
 517         "dependency": "samba-no-opath-build",
 518         "sequence": [
 519             ("random-sleep", random_sleep(300, 900)),
 520             ("test", make_test(
 521                 cmd="make testonly DISABLE_OPATH=1",
 522                 include_envs=[
 523                 "nt4_dc",
 524                 "nt4_dc_smb1",
 525                 "nt4_dc_smb1_done",
 526                 "nt4_dc_schannel",
 527                 "nt4_member",
 528                 "simpleserver",
 529                 ])),
 530             ("lcov", LCOV_CMD),
 531             ("check-clean-tree", "script/clean-source-tree.sh"),
 532         ],
 533     },
 534
 535     "samba-no-opath2": {
 536         "dependency": "samba-no-opath-build",
 537         "sequence": [
 538             ("random-sleep", random_sleep(300, 900)),
 539             ("test", make_test(
 540                 cmd="make testonly DISABLE_OPATH=1",
 541                 include_envs=[
 542                 "fileserver",
 543                 "fileserver_smb1",
 544                 "fileserver_smb1_done",
 545                 ])),
 546             ("lcov", LCOV_CMD),
 547             ("check-clean-tree", "script/clean-source-tree.sh"),
 548         ],
 549     },
 550
 551     "samba-ad-dc-1": {
 552         "dependency": "samba-def-build",
 553         "sequence": [
 554             ("random-sleep", random_sleep(1, 1)),
 555             ("test", make_test(include_envs=[
 556             "ad_dc",
 557             "ad_dc_smb1",
 558             "ad_dc_smb1_done",
 559             "ad_dc_no_nss",
 560             "ad_dc_no_ntlm",
 561             ])),
 562             ("lcov", LCOV_CMD),
 563             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 564         ],
 565     },
 566
 567     "samba-ad-dc-2": {
 568         "dependency": "samba-def-build",
 569         "sequence": [
 570             ("random-sleep", random_sleep(1, 1)),
 571             ("test", make_test(include_envs=[
 572             "vampire_dc",
 573             "vampire_2000_dc",
 574             "rodc",
 575             ])),
 576             ("lcov", LCOV_CMD),
 577             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 578         ],
 579     },
 580
 581     "samba-ad-dc-3": {
 582         "dependency": "samba-def-build",
 583         "sequence": [
 584             ("random-sleep", random_sleep(1, 1)),
 585             ("test", make_test(include_envs=[
 586             "promoted_dc",
 587             "chgdcpass",
 588             "preforkrestartdc",
 589             "proclimitdc",
 590             ])),
 591             ("lcov", LCOV_CMD),
 592             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 593         ],
 594     },
 595
 596     "samba-ad-dc-4a": {
 597         "dependency": "samba-def-build",
 598         "sequence": [
 599             ("random-sleep", random_sleep(1, 1)),
 600             ("test", make_test(include_envs=[
 601             "fl2000dc",
 602             "ad_member_oneway",
 603             "fl2003dc",
 604             ])),
 605             ("lcov", LCOV_CMD),
 606             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 607         ],
 608     },
 609     "samba-ad-dc-4b": {
 610         "dependency": "samba-def-build",
 611         "sequence": [
 612             ("random-sleep", random_sleep(1, 1)),
 613             ("test", make_test(include_envs=[
 614             "fl2008dc",
 615             "fl2008r2dc",
 616             ])),
 617             ("lcov", LCOV_CMD),
 618             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 619         ],
 620     },
 621
 622     "samba-ad-dc-5": {
 623         "dependency": "samba-def-build",
 624         "sequence": [
 625             ("random-sleep", random_sleep(1, 1)),
 626             ("test", make_test(include_envs=[
 627             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-6": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 638             ("lcov", LCOV_CMD),
 639             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 640         ],
 641     },
 642
 643     "samba-schemaupgrade": {
 644         "dependency": "samba-def-build",
 645         "sequence": [
 646             ("random-sleep", random_sleep(1, 1)),
 647             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 648             ("lcov", LCOV_CMD),
 649             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 650         ],
 651     },
 652
 653     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 654     # This is currently the longest task, so we don't randomly delay it.
 655     "samba-ad-dc-ntvfs": {
 656         "dependency": "samba-def-build",
 657         "sequence": [
 658             ("random-sleep", random_sleep(1, 1)),
 659             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 660             ("lcov", LCOV_CMD),
 661             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 662         ],
 663     },
 664
 665     # Test fips compliance
 666     "samba-fips": {
 667         "dependency": "samba-mit-build",
 668         "sequence": [
 669             ("random-sleep", random_sleep(1, 1)),
 670             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 671             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 672             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 673         ],
 674     },
 675
 676     # run the backup/restore testenvs separately as they're fairly standalone
 677     # (and CI seems to max out at ~3 different DCs running at once)
 678     "samba-ad-back1": {
 679         "dependency": "samba-def-build",
 680         "sequence": [
 681             ("random-sleep", random_sleep(300, 900)),
 682             ("test", make_test(include_envs=[
 683             "backupfromdc",
 684             "restoredc",
 685             "renamedc",
 686             ])),
 687             ("lcov", LCOV_CMD),
 688             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 689         ],
 690     },
 691     "samba-ad-back2": {
 692         "dependency": "samba-def-build",
 693         "sequence": [
 694             ("random-sleep", random_sleep(300, 900)),
 695             ("test", make_test(include_envs=[
 696             "backupfromdc",
 697             "offlinebackupdc",
 698             "labdc",
 699             ])),
 700             ("lcov", LCOV_CMD),
 701             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 702         ],
 703     },
 704
 705     "samba-admem-mit": {
 706         "dependency": "samba-mit-build",
 707         "sequence": [
 708             ("random-sleep", random_sleep(1, 1)),
 709             ("test", make_test(include_envs=[
 710             "ad_member",
 711             "ad_member_idmap_rid",
 712             "admem_idmap_autorid",
 713             "ad_member_idmap_ad",
 714             "ad_member_rfc2307",
 715             "ad_member_offlogon",
 716             ])),
 717             ("lcov", LCOV_CMD),
 718             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 719         ],
 720     },
 721
 722     "samba-addc-mit-1": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "ad_dc",
 728             "ad_dc_smb1",
 729             "ad_dc_smb1_done",
 730             "ad_dc_no_nss",
 731             "ad_dc_no_ntlm",
 732             ])),
 733             ("lcov", LCOV_CMD),
 734             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 735         ],
 736     },
 737
 738     "samba-addc-mit-4a": {
 739         "dependency": "samba-mit-build",
 740         "sequence": [
 741             ("random-sleep", random_sleep(1, 1)),
 742             ("test", make_test(include_envs=[
 743             "fl2000dc",
 744             "ad_member_oneway",
 745             "fl2003dc",
 746             ])),
 747             ("lcov", LCOV_CMD),
 748             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 749         ],
 750     },
 751     "samba-addc-mit-4b": {
 752         "dependency": "samba-mit-build",
 753         "sequence": [
 754             ("random-sleep", random_sleep(1, 1)),
 755             ("test", make_test(include_envs=[
 756             "fl2008dc",
 757             "fl2008r2dc",
 758             ])),
 759             ("lcov", LCOV_CMD),
 760             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 761         ],
 762     },
 763
 764     "samba-test-only": {
 765         "sequence": [
 766             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 767             ("make", "make -j"),
 768             ("test", make_test(TESTS="${TESTS}")),
 769             ("lcov", LCOV_CMD),
 770         ],
 771     },
 772
 773     # Test cross-compile infrastructure
 774     "samba-xc": {
 775         "sequence": [
 776             ("random-sleep", random_sleep(900, 1500)),
 777             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 778             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 779             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 780             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 781             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 782             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 783             ("compare-results", "script/compare_cc_results.py "
 784             "./bin/c4che/default{} "
 785             "./bin-xe/c4che/default{} "
 786             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 787             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 788             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 789             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 790             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 791             " = \"'1234'\"".format(CACHE_SUFFIX)),
 792             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 793             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 794             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 795             " ; test $? -ne 0"),
 796         ],
 797     },
 798
 799     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 800     "samba-o3": {
 801         "sequence": [
 802             ("random-sleep", random_sleep(300, 900)),
 803             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 804             ("make", "make -j"),
 805             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 806             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 807             ("lcov", LCOV_CMD),
 808             ("install", "make install"),
 809             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 810             ("clean", "make clean"),
 811         ],
 812     },
 813
 814     "samba-32bit": {
 815         "sequence": [
 816             ("random-sleep", random_sleep(300, 900)),
 817             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 818             ("make", "make -j"),
 819             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 820             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 821             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 822             ("install", "make install"),
 823             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 824             ("clean", "make clean"),
 825         ],
 826     },
 827
 828     "samba-ctdb": {
 829         "sequence": [
 830             ("random-sleep", random_sleep(900, 1500)),
 831
 832         # make sure we have tdb around:
 833             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 834             ("tdb-make", "cd lib/tdb && make"),
 835             ("tdb-install", "cd lib/tdb && make install"),
 836
 837         # build samba with cluster support (also building ctdb):
 838             ("samba-configure",
 839          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 840          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 841          "./configure.developer ${PREFIX} "
 842          "--with-selftest-prefix=./bin/ab "
 843          "--with-cluster-support "
 844          "--without-ad-dc "
 845          "--bundled-libraries=!tdb"),
 846             ("samba-make", "make"),
 847             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 848             ("samba-install", "make install"),
 849             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 850
 851             ("test", make_test(
 852                 cmd='make test',
 853                 INJECT_SELFTEST_PREFIX=0,
 854                 include_envs=["clusteredmember"])
 855             ),
 856
 857         # clean up:
 858             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 859             ("clean", "make clean"),
 860             ("ctdb-clean", "cd ./ctdb && make clean"),
 861         ],
 862     },
 863
 864     "samba-libs": {
 865         "sequence": [
 866             ("random-sleep", random_sleep(300, 900)),
 867             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 868             ("talloc-make", "cd lib/talloc && make"),
 869             ("talloc-install", "cd lib/talloc && make install"),
 870
 871             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 872             ("tdb-make", "cd lib/tdb && make"),
 873             ("tdb-install", "cd lib/tdb && make install"),
 874
 875             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 876             ("tevent-make", "cd lib/tevent && make"),
 877             ("tevent-install", "cd lib/tevent && make install"),
 878
 879             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs),
 880             ("ldb-make", "cd lib/ldb && make"),
 881             ("ldb-install", "cd lib/ldb && make install"),
 882
 883             ("nondevel-configure", samba_libs_envvars + " ./configure ${PREFIX}"),
 884             ("nondevel-make", "make -j"),
 885             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 886             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 887             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 888             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 889             ("nondevel-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 890             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 891             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 892             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 893             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 894             ("nondevel-no-public-nss_winbind",
 895                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 896             ("nondevel-no-public-nss_wins",
 897                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 898             ("nondevel-no-public-libwbclient",
 899                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 900             ("nondevel-no-public-pam_winbind",
 901                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 902             ("nondevel-no-public-winbind_krb5_locator",
 903                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 904             ("nondevel-no-public-async_dns_krb5_locator",
 905                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 906             ("nondevel-install", "make -j install"),
 907             ("nondevel-dist", "make dist"),
 908
 909             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 910             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 911             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 912             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb' && exit 1; exit 0"),
 913             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 914             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 915             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 916             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 917             ("prefix-no-public-nss_winbind",
 918                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 919             ("prefix-no-public-nss_wins",
 920                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 921             ("prefix-no-public-libwbclient",
 922                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 923             ("prefix-no-public-pam_winbind",
 924                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 925             ("prefix-no-public-winbind_krb5_locator",
 926                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 927                               "service_locator")),
 928             ("prefix-no-public-async_dns_krb5_locator",
 929                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 930                               "service_locator")),
 931
 932             # retry with all modules shared
 933             ("allshared-distclean", "make distclean"),
 934             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 935             ("allshared-make", "make -j"),
 936             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 937             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 938             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 939             ("allshared-no-libldb", "find ./bin | grep -v 'module' | grep -v 'libldbsamba' | grep 'libldb' && exit 1; exit 0"),
 940             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 941             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 942             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 943             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 944             ("allshared-no-public-nss_winbind",
 945                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 946             ("allshared-no-public-nss_wins",
 947                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 948             ("allshared-no-public-libwbclient",
 949                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 950             ("allshared-no-public-pam_winbind",
 951                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 952             ("allshared-no-public-winbind_krb5_locator",
 953                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 954             ("allshared-no-public-async_dns_krb5_locator",
 955                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 956         ],
 957     },
 958
 959     "samba-fuzz": {
 960         "sequence": [
 961         # build the fuzzers (static) via the oss-fuzz script
 962             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
 963             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
 964         ],
 965     },
 966
 967     # * Test smbd and smbtorture can build semi-static
 968     #
 969     # * Test Samba without python still builds.
 970     #
 971     # When this test fails due to more use of Python, the expectations
 972     # is that the newly failing part of the code should be disabled
 973     # when --disable-python is set (rather than major work being done
 974     # to support this environment).
 975     #
 976     # The target here is for vendors shipping a minimal smbd.
 977     "samba-minimal-smbd": {
 978         "sequence": [
 979             ("random-sleep", random_sleep(300, 900)),
 980
 981         # build with all modules static
 982             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
 983             ("allstatic-make", "make -j"),
 984             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 985             ("allstatic-lcov", LCOV_CMD),
 986
 987         # retry with nonshared smbd and smbtorture
 988             ("nonshared-distclean", "make distclean"),
 989             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
 990             ("nonshared-make", "make -j"),
 991             # TODO ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
 992             # TODO ("nonshared-lcov", LCOV_CMD),
 993
 994             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 995             ("clean", "make clean"),
 996         ],
 997     },
 998
 999     "samba-nopython": {
1000         "sequence": [
1001             ("random-sleep", random_sleep(300, 900)),
1002
1003             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1004             ("make", "make -j"),
1005             ("find-python", "script/find_python.sh ${PREFIX}"),
1006             ("test", "make test-nopython"),
1007             ("lcov", LCOV_CMD),
1008             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1009             ("clean", "make clean"),
1010
1011             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1012             ("talloc-make", "cd lib/talloc && make"),
1013             ("talloc-install", "cd lib/talloc && make install"),
1014
1015             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1016             ("tdb-make", "cd lib/tdb && make"),
1017             ("tdb-install", "cd lib/tdb && make install"),
1018
1019             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1020             ("tevent-make", "cd lib/tevent && make"),
1021             ("tevent-install", "cd lib/tevent && make install"),
1022
1023             ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1024             ("ldb-make", "cd lib/ldb && make"),
1025             ("ldb-install", "cd lib/ldb && make install"),
1026
1027         # retry against installed library packages, but no required modules
1028             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1029             ("libs-make", "make -j"),
1030             ("libs-install", "make install"),
1031             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1032             ("libs-clean", "make clean"),
1033
1034         ],
1035     },
1036
1037     "samba-shellcheck": {
1038         "sequence": [
1039             ("run", "script/check-shell-scripts.sh ."),
1040         ],
1041     },
1042
1043     "ldb": {
1044         "sequence": [
1045             ("random-sleep", random_sleep(60, 600)),
1046             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1047             ("make", "make"),
1048             ("install", "make install"),
1049             ("test", "make test"),
1050             ("lcov", LCOV_CMD),
1051             ("clean", "make clean"),
1052             ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"),
1053             ("make-no-lmdb", "make"),
1054             ("test-no-lmdb", "make test"),
1055             ("lcov-no-lmdb", LCOV_CMD),
1056             ("install-no-lmdb", "make install"),
1057             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1058             ("distcheck", "make distcheck"),
1059             ("clean", "make clean"),
1060         ],
1061     },
1062
1063     "tdb": {
1064         "sequence": [
1065             ("random-sleep", random_sleep(60, 600)),
1066             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1067             ("make", "make"),
1068             ("install", "make install"),
1069             ("test", "make test"),
1070             ("lcov", LCOV_CMD),
1071             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1072             ("distcheck", "make distcheck"),
1073             ("clean", "make clean"),
1074         ],
1075     },
1076
1077     "talloc": {
1078         "sequence": [
1079             ("random-sleep", random_sleep(60, 600)),
1080             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1081             ("make", "make"),
1082             ("install", "make install"),
1083             ("test", "make test"),
1084             ("lcov", LCOV_CMD),
1085             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1086             ("distcheck", "make distcheck"),
1087             ("clean", "make clean"),
1088         ],
1089     },
1090
1091     "replace": {
1092         "sequence": [
1093             ("random-sleep", random_sleep(60, 600)),
1094             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1095             ("make", "make"),
1096             ("install", "make install"),
1097             ("test", "make test"),
1098             ("lcov", LCOV_CMD),
1099             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1100             ("distcheck", "make distcheck"),
1101             ("clean", "make clean"),
1102         ],
1103     },
1104
1105     "tevent": {
1106         "sequence": [
1107             ("random-sleep", random_sleep(60, 600)),
1108             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1109             ("make", "make"),
1110             ("install", "make install"),
1111             ("test", "make test"),
1112             ("lcov", LCOV_CMD),
1113             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1114             ("distcheck", "make distcheck"),
1115             ("clean", "make clean"),
1116         ],
1117     },
1118
1119     "pidl": {
1120         "git-clone-required": True,
1121         "sequence": [
1122             ("random-sleep", random_sleep(60, 600)),
1123             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1124             ("touch", "touch *.yp"),
1125             ("make", "make"),
1126             ("test", "make test"),
1127             ("install", "make install"),
1128             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1129             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1130             ("clean", "make clean"),
1131         ],
1132     },
1133
1134     # these are useful for debugging autobuild
1135     "pass": {
1136         "sequence": [
1137             ("pass", 'echo passing && /bin/true'),
1138         ],
1139     },
1140     "fail": {
1141         "sequence": [
1142             ("fail", 'echo failing && /bin/false'),
1143         ],
1144     },
1145 }
1146
1147 defaulttasks = list(tasks.keys())
1148
1149 defaulttasks.remove("pass")
1150 defaulttasks.remove("fail")
1151
1152 # The build tasks will be brought in by the test tasks as needed
1153 defaulttasks.remove("samba-def-build")
1154 defaulttasks.remove("samba-nt4-build")
1155 defaulttasks.remove("samba-mit-build")
1156 defaulttasks.remove("samba-h5l-build")
1157 defaulttasks.remove("samba-no-opath-build")
1158
1159 # This is not a normal test, but a task to support manually running
1160 # one test under autobuild
1161 defaulttasks.remove("samba-test-only")
1162
1163 # Only built on GitLab CI and not in the default autobuild because it
1164 # uses too much space (4GB of semi-static binaries)
1165 defaulttasks.remove("samba-fuzz")
1166
1167 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1168 # container where a simulated FIPS mode is possible.
1169 defaulttasks.remove("samba-fips")
1170
1171 # The MIT build runs on a current Fedora where an up to date MIT KDC
1172 # is already packaged.  This avoids needing to backport a current MIT
1173 # to the default Ubuntu 18.04, particularly during development, and
1174 # the need to install on the shared sn-devel-184.
1175
1176 defaulttasks.remove("samba-mitkrb5")
1177 defaulttasks.remove("samba-admem-mit")
1178 defaulttasks.remove("samba-addc-mit-1")
1179 defaulttasks.remove("samba-addc-mit-4a")
1180 defaulttasks.remove("samba-addc-mit-4b")
1181
1182 defaulttasks.remove("samba-32bit")
1183
1184 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1185     defaulttasks.remove("samba-o3")
1186
1187
1188 def do_print(msg):
1189     print("%s" % msg)
1190     sys.stdout.flush()
1191     sys.stderr.flush()
1192
1193
1194 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1195     if show is None:
1196         show = options.verbose
1197     if show:
1198         do_print("Running: '%s' in '%s'" % (cmd, dir))
1199     if output:
1200         out = check_output([cmd], shell=True, cwd=dir)
1201         return out.decode(encoding='utf-8', errors='backslashreplace')
1202     elif checkfail:
1203         return check_call(cmd, shell=True, cwd=dir)
1204     else:
1205         return call(cmd, shell=True, cwd=dir)
1206
1207 def rmdir_force(dirname, re_raise=True):
1208     try:
1209         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1210                 dirname, dirname, dirname), output=True, show=True)
1211     except CalledProcessError as e:
1212         do_print("Failed: '%s'" % (str(e)))
1213         run_cmd("tree %s" % dirname, output=True, show=True)
1214         if re_raise:
1215             raise
1216         return False
1217     return True
1218
1219 class builder(object):
1220     '''handle build of one directory'''
1221
1222     def __init__(self, name, definition):
1223         self.name = name
1224         self.dir = builddirs.get(name, '.')
1225         self.tag = self.name.replace('/', '_')
1226         self.definition = definition
1227         self.sequence = definition["sequence"]
1228         self.git_clone_required = False
1229         if "git-clone-required" in definition:
1230             self.git_clone_required = bool(definition["git-clone-required"])
1231         self.proc = None
1232         self.done = False
1233         self.next = 0
1234         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1235         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1236         if options.verbose:
1237             do_print("stdout for %s in %s" % (self.name, self.stdout_path))
1238             do_print("stderr for %s in %s" % (self.name, self.stderr_path))
1239         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1240         self.stdout = open(self.stdout_path, 'w')
1241         self.stderr = open(self.stderr_path, 'w')
1242         self.stdin  = open("/dev/null", 'r')
1243         self.builder_dir = "%s/%s" % (testbase, self.tag)
1244         self.test_source_dir = self.builder_dir
1245         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1246         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1247         self.prefix = "%s/%s" % (test_prefix, self.tag)
1248         self.consumers = []
1249         self.producer = None
1250
1251         if self.git_clone_required:
1252             assert "dependency" not in definition
1253
1254     def mark_existing(self):
1255         do_print('%s: Mark as existing dependency' % self.name)
1256         self.next = len(self.sequence)
1257         self.done = True
1258
1259     def add_consumer(self, consumer):
1260         do_print("%s: add consumer: %s" % (self.name, consumer.name))
1261         consumer.producer = self
1262         consumer.test_source_dir = self.test_source_dir
1263         self.consumers.append(consumer)
1264
1265     def start_next(self):
1266         if self.producer is not None:
1267             if not self.producer.done:
1268                 do_print("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1269                 return
1270
1271         if self.next == 0:
1272             rmdir_force(self.builder_dir)
1273             rmdir_force(self.prefix)
1274             if self.producer is not None:
1275                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1276             elif not self.git_clone_required:
1277                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1278             else:
1279                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1280
1281         if self.next == len(self.sequence):
1282             if not self.done:
1283                 do_print('%s: Completed OK' % self.name)
1284                 self.done = True
1285             if not options.nocleanup and len(self.consumers) == 0:
1286                 do_print('%s: Cleaning up' % self.name)
1287                 rmdir_force(self.builder_dir)
1288                 rmdir_force(self.prefix)
1289             for consumer in self.consumers:
1290                 if consumer.next != 0:
1291                     continue
1292                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1293                 consumer.start_next()
1294             if self.producer is not None:
1295                 self.producer.consumers.remove(self)
1296                 assert self.producer.done
1297                 self.producer.start_next()
1298             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1299             return
1300         (self.stage, self.cmd) = self.sequence[self.next]
1301         self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix))
1302         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1303         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1304         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1305         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1306         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1307         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1308         self.cmd = self.cmd.replace("${NAME}", self.name)
1309         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1310         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1311         self.proc = Popen(self.cmd, shell=True,
1312                           close_fds=True, cwd=self.cwd,
1313                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1314         self.next += 1
1315
1316 def expand_dependencies(n):
1317     deps = list()
1318     if "dependency" in tasks[n]:
1319         depname = tasks[n]["dependency"]
1320         assert depname in tasks
1321         sdeps = expand_dependencies(depname)
1322         assert n not in sdeps
1323         for sdep in sdeps:
1324             deps.append(sdep)
1325         deps.append(depname)
1326     return deps
1327
1328
1329 class buildlist(object):
1330     '''handle build of multiple directories'''
1331
1332     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1333         self.tail_proc = None
1334         self.retry = None
1335         if not tasknames:
1336             if options.restrict_tests:
1337                 tasknames = ["samba-test-only"]
1338             else:
1339                 tasknames = defaulttasks
1340
1341         given_tasknames = tasknames.copy()
1342         implicit_tasknames = []
1343         for n in given_tasknames:
1344             deps = expand_dependencies(n)
1345             for dep in deps:
1346                 if dep in given_tasknames:
1347                     continue
1348                 if dep in implicit_tasknames:
1349                     continue
1350                 implicit_tasknames.append(dep)
1351
1352         tasknames = implicit_tasknames.copy()
1353         tasknames.extend(given_tasknames)
1354         do_print("given_tasknames: %s" % given_tasknames)
1355         do_print("implicit_tasknames: %s" % implicit_tasknames)
1356         do_print("tasknames: %s" % tasknames)
1357         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1358
1359         if options.retry:
1360             rebase_remote = "rebaseon"
1361             retry_task = {
1362                     "git-clone-required": True,
1363                     "sequence": [
1364                             ("retry",
1365                             '''set -e
1366                             git remote add -t %s %s %s
1367                             git fetch %s
1368                             while :; do
1369                               sleep 60
1370                               git describe %s/%s > old_remote_branch.desc
1371                               git fetch %s
1372                               git describe %s/%s > remote_branch.desc
1373                               diff old_remote_branch.desc remote_branch.desc
1374                             done
1375                            ''' % (
1376                                rebase_branch, rebase_remote, rebase_url,
1377                                rebase_remote,
1378                                rebase_remote, rebase_branch,
1379                                rebase_remote,
1380                                rebase_remote, rebase_branch
1381                             ))]}
1382
1383             self.retry = builder('retry', retry_task)
1384             self.need_retry = False
1385
1386         if options.skip_dependencies:
1387             for b in self.tlist:
1388                 if b.name in implicit_tasknames:
1389                     b.mark_existing()
1390
1391         for b in self.tlist:
1392             do_print("b.name=%s" % b.name)
1393             if "dependency" not in b.definition:
1394                 continue
1395             depname = b.definition["dependency"]
1396             do_print("b.name=%s: dependency:%s" % (b.name, depname))
1397             for p in self.tlist:
1398                 if p.name == depname:
1399                     p.add_consumer(b)
1400
1401     def kill_kids(self):
1402         if self.tail_proc is not None:
1403             self.tail_proc.terminate()
1404             self.tail_proc.wait()
1405             self.tail_proc = None
1406         if self.retry is not None:
1407             self.retry.proc.terminate()
1408             self.retry.proc.wait()
1409             self.retry = None
1410         for b in self.tlist:
1411             if b.proc is not None:
1412                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1413                 b.proc.terminate()
1414                 b.proc.wait()
1415                 b.proc = None
1416
1417     def wait_one(self):
1418         while True:
1419             none_running = True
1420             for b in self.tlist:
1421                 if b.proc is None:
1422                     continue
1423                 none_running = False
1424                 b.status = b.proc.poll()
1425                 if b.status is None:
1426                     continue
1427                 b.proc = None
1428                 return b
1429             if options.retry:
1430                 ret = self.retry.proc.poll()
1431                 if ret is not None:
1432                     self.need_retry = True
1433                     self.retry = None
1434                     return None
1435             if none_running:
1436                 return None
1437             time.sleep(0.1)
1438
1439     def run(self):
1440         for b in self.tlist:
1441             b.start_next()
1442         if options.retry:
1443             self.retry.start_next()
1444         while True:
1445             b = self.wait_one()
1446             if options.retry and self.need_retry:
1447                 self.kill_kids()
1448                 do_print("retry needed")
1449                 return (0, None, None, None, "retry")
1450             if b is None:
1451                 break
1452             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1453                 self.kill_kids()
1454                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1455             b.start_next()
1456         self.kill_kids()
1457         return (0, None, None, None, "All OK")
1458
1459     def write_system_info(self, filename):
1460         with open(filename, 'w') as f:
1461             for cmd in ['uname -a',
1462                         'lsb_release -a',
1463                         'free',
1464                         'mount',
1465                         'cat /proc/cpuinfo',
1466                         'cc --version',
1467                         'df -m .',
1468                         'df -m %s' % testbase]:
1469                 try:
1470                     out = run_cmd(cmd, output=True, checkfail=False)
1471                 except CalledProcessError as e:
1472                     out = "<failed: %s>" % str(e)
1473                 print('### %s' % cmd, file=f)
1474                 print(out, file=f)
1475                 print(file=f)
1476
1477     def tarlogs(self, fname):
1478         with tarfile.open(fname, "w:gz") as tar:
1479             for b in self.tlist:
1480                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1481                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1482             if os.path.exists("autobuild.log"):
1483                 tar.add("autobuild.log")
1484             filename = 'system-info.txt'
1485             self.write_system_info(filename)
1486             tar.add(filename)
1487
1488     def remove_logs(self):
1489         for b in self.tlist:
1490             os.unlink(b.stdout_path)
1491             os.unlink(b.stderr_path)
1492
1493     def start_tail(self):
1494         cmd = ["tail", "-f"]
1495         for b in self.tlist:
1496             cmd.append(b.stdout_path)
1497             cmd.append(b.stderr_path)
1498         self.tail_proc = Popen(cmd, close_fds=True)
1499
1500
1501 def cleanup(do_raise=False):
1502     if options.nocleanup:
1503         return
1504     run_cmd("stat %s || true" % test_tmpdir, show=True)
1505     run_cmd("stat %s" % testbase, show=True)
1506     do_print("Cleaning up %r" % cleanup_list)
1507     for d in cleanup_list:
1508         ok = rmdir_force(d, re_raise=False)
1509         if ok:
1510             continue
1511         if os.path.isdir(d):
1512             do_print("Killing, waiting and retry")
1513             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1514         else:
1515             do_print("Waiting and retry")
1516         time.sleep(1)
1517         rmdir_force(d, re_raise=do_raise)
1518
1519
1520 def daemonize(logfile):
1521     pid = os.fork()
1522     if pid == 0:  # Parent
1523         os.setsid()
1524         pid = os.fork()
1525         if pid != 0:  # Actual daemon
1526             os._exit(0)
1527     else:  # Grandparent
1528         os._exit(0)
1529
1530     import resource      # Resource usage information.
1531     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1532     if maxfd == resource.RLIM_INFINITY:
1533         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1534     for fd in range(0, maxfd):
1535         try:
1536             os.close(fd)
1537         except OSError:
1538             pass
1539     os.open(logfile, os.O_RDWR | os.O_CREAT)
1540     os.dup2(0, 1)
1541     os.dup2(0, 2)
1542
1543
1544 def write_pidfile(fname):
1545     '''write a pid file, cleanup on exit'''
1546     with open(fname, mode='w') as f:
1547         f.write("%u\n" % os.getpid())
1548
1549
1550 def rebase_tree(rebase_url, rebase_branch="master"):
1551     rebase_remote = "rebaseon"
1552     do_print("Rebasing on %s" % rebase_url)
1553     run_cmd("git describe HEAD", show=True, dir=test_master)
1554     run_cmd("git remote add -t %s %s %s" %
1555             (rebase_branch, rebase_remote, rebase_url),
1556             show=True, dir=test_master)
1557     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1558     if options.fix_whitespace:
1559         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1560                 (rebase_remote, rebase_branch),
1561                 show=True, dir=test_master)
1562     else:
1563         run_cmd("git rebase --force-rebase %s/%s" %
1564                 (rebase_remote, rebase_branch),
1565                 show=True, dir=test_master)
1566     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1567                    (rebase_remote, rebase_branch),
1568                    dir=test_master, output=True)
1569     if diff == '':
1570         do_print("No differences between HEAD and %s/%s - exiting" %
1571                  (rebase_remote, rebase_branch))
1572         sys.exit(0)
1573     run_cmd("git describe %s/%s" %
1574             (rebase_remote, rebase_branch),
1575             show=True, dir=test_master)
1576     run_cmd("git describe HEAD", show=True, dir=test_master)
1577     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1578             (rebase_remote, rebase_branch),
1579             show=True, dir=test_master)
1580
1581
1582 def push_to(push_url, push_branch="master"):
1583     push_remote = "pushto"
1584     do_print("Pushing to %s" % push_url)
1585     if options.mark:
1586         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1587         run_cmd("git commit --amend -c HEAD", dir=test_master)
1588         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1589         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1590     run_cmd("git remote add -t %s %s %s" %
1591             (push_branch, push_remote, push_url),
1592             show=True, dir=test_master)
1593     run_cmd("git push %s +HEAD:%s" %
1594             (push_remote, push_branch),
1595             show=True, dir=test_master)
1596
1597
1598 def send_email(subject, text, log_tar):
1599     if options.email is None:
1600         do_print("not sending email because the recipient is not set")
1601         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1602                  (subject, text))
1603         return
1604     outer = MIMEMultipart()
1605     outer['Subject'] = subject
1606     outer['To'] = options.email
1607     outer['From'] = options.email_from
1608     outer['Date'] = email.utils.formatdate(localtime=True)
1609     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1610     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1611     if options.attach_logs:
1612         with open(log_tar, 'rb') as fp:
1613             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1614         # Set the filename parameter
1615         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1616         outer.attach(msg)
1617     content = outer.as_string()
1618     s = smtplib.SMTP(options.email_server)
1619     email_user = os.getenv('SMTP_USERNAME')
1620     email_password = os.getenv('SMTP_PASSWORD')
1621     if email_user is not None:
1622         s.starttls()
1623         s.login(email_user, email_password)
1624
1625     s.sendmail(options.email_from, [options.email], content)
1626     s.set_debuglevel(1)
1627     s.quit()
1628
1629
1630 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1631                   elapsed_time, log_base=None, add_log_tail=True):
1632     '''send an email to options.email about the failure'''
1633     elapsed_minutes = elapsed_time / 60.0
1634     if log_base is None:
1635         log_base = gitroot
1636     text = '''
1637 Dear Developer,
1638
1639 Your autobuild on %s failed after %.1f minutes
1640 when trying to test %s with the following error:
1641
1642    %s
1643
1644 the autobuild has been abandoned. Please fix the error and resubmit.
1645
1646 A summary of the autobuild process is here:
1647
1648   %s/autobuild.log
1649 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1650
1651     if options.restrict_tests:
1652         text += """
1653 The build was restricted to tests matching %s\n""" % options.restrict_tests
1654
1655     if failed_task != 'rebase':
1656         text += '''
1657 You can see logs of the failed task here:
1658
1659   %s/%s.stdout
1660   %s/%s.stderr
1661
1662 or you can get full logs of all tasks in this job here:
1663
1664   %s/logs.tar.gz
1665
1666 The top commit for the tree that was built was:
1667
1668 %s
1669
1670 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1671
1672     if add_log_tail:
1673         f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r')
1674         lines = f.readlines()
1675         log_tail = "".join(lines[-50:])
1676         num_lines = len(lines)
1677         if num_lines < 50:
1678             # Also include stderr (compile failures) if < 50 lines of stdout
1679             f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r')
1680             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1681
1682         text += '''
1683 The last 50 lines of log messages:
1684
1685 %s
1686     ''' % log_tail
1687         f.close()
1688
1689     logs = os.path.join(gitroot, 'logs.tar.gz')
1690     send_email('autobuild[%s] failure on %s for task %s during %s'
1691                % (options.branch, platform.node(), failed_task, failed_stage),
1692                text, logs)
1693
1694
1695 def email_success(elapsed_time, log_base=None):
1696     '''send an email to options.email about a successful build'''
1697     if log_base is None:
1698         log_base = gitroot
1699     text = '''
1700 Dear Developer,
1701
1702 Your autobuild on %s has succeeded after %.1f minutes.
1703
1704 ''' % (platform.node(), elapsed_time / 60.)
1705
1706     if options.restrict_tests:
1707         text += """
1708 The build was restricted to tests matching %s\n""" % options.restrict_tests
1709
1710     if options.keeplogs:
1711         text += '''
1712
1713 you can get full logs of all tasks in this job here:
1714
1715   %s/logs.tar.gz
1716
1717 ''' % log_base
1718
1719     text += '''
1720 The top commit for the tree that was built was:
1721
1722 %s
1723 ''' % top_commit_msg
1724
1725     logs = os.path.join(gitroot, 'logs.tar.gz')
1726     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1727                text, logs)
1728
1729
1730 # get the top commit message, for emails
1731 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1732
1733 try:
1734     if options.skip_dependencies:
1735         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1736     else:
1737         os.makedirs(testbase)
1738 except Exception as reason:
1739     raise Exception("Unable to create %s : %s" % (testbase, reason))
1740 cleanup_list.append(testbase)
1741
1742 if options.daemon:
1743     logfile = os.path.join(testbase, "log")
1744     do_print("Forking into the background, writing progress to %s" % logfile)
1745     daemonize(logfile)
1746
1747 write_pidfile(gitroot + "/autobuild.pid")
1748
1749 start_time = time.time()
1750
1751 while True:
1752     try:
1753         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1754         os.makedirs(test_tmpdir)
1755         # The waf uninstall code removes empty directories all the way
1756         # up the tree.  Creating a file in test_tmpdir stops it from
1757         # being removed.
1758         run_cmd("touch %s" % os.path.join(test_tmpdir,
1759                                           ".directory-is-not-empty"), show=True)
1760         run_cmd("stat %s" % test_tmpdir, show=True)
1761         run_cmd("stat %s" % testbase, show=True)
1762         if options.skip_dependencies:
1763             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1764         else:
1765             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1766     except Exception:
1767         cleanup()
1768         raise
1769
1770     try:
1771         if options.rebase is not None:
1772             rebase_tree(options.rebase, rebase_branch=options.branch)
1773     except Exception:
1774         cleanup_list.append(gitroot + "/autobuild.pid")
1775         cleanup()
1776         elapsed_time = time.time() - start_time
1777         email_failure(-1, 'rebase', 'rebase', 'rebase',
1778                       'rebase on %s failed' % options.branch,
1779                       elapsed_time, log_base=options.log_base)
1780         sys.exit(1)
1781
1782     try:
1783         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1784         if options.tail:
1785             blist.start_tail()
1786         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1787         if status != 0 or errstr != "retry":
1788             break
1789         cleanup(do_raise=True)
1790     except Exception:
1791         cleanup()
1792         raise
1793
1794 cleanup_list.append(gitroot + "/autobuild.pid")
1795
1796 do_print(errstr)
1797
1798 blist.kill_kids()
1799 if options.tail:
1800     do_print("waiting for tail to flush")
1801     time.sleep(1)
1802
1803 elapsed_time = time.time() - start_time
1804 if status == 0:
1805     if options.passcmd is not None:
1806         do_print("Running passcmd: %s" % options.passcmd)
1807         run_cmd(options.passcmd, dir=test_master)
1808     if options.pushto is not None:
1809         push_to(options.pushto, push_branch=options.branch)
1810     if options.keeplogs or options.attach_logs:
1811         blist.tarlogs("logs.tar.gz")
1812         do_print("Logs in logs.tar.gz")
1813     if options.always_email:
1814         email_success(elapsed_time, log_base=options.log_base)
1815     blist.remove_logs()
1816     cleanup()
1817     do_print(errstr)
1818     sys.exit(0)
1819
1820 # something failed, gather a tar of the logs
1821 blist.tarlogs("logs.tar.gz")
1822
1823 if options.email is not None:
1824     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1825                   elapsed_time, log_base=options.log_base)
1826 else:
1827     elapsed_minutes = elapsed_time / 60.0
1828     print('''
1829
1830 ####################################################################
1831
1832 AUTOBUILD FAILURE
1833
1834 Your autobuild[%s] on %s failed after %.1f minutes
1835 when trying to test %s with the following error:
1836
1837    %s
1838
1839 the autobuild has been abandoned. Please fix the error and resubmit.
1840
1841 ####################################################################
1842
1843 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1844
1845 cleanup()
1846 do_print(errstr)
1847 do_print("Logs in logs.tar.gz")
1848 sys.exit(status)